From 8c006b38bdc304201d646c35c993156abcc2b845 Mon Sep 17 00:00:00 2001 From: Ti Chi Robot Date: Mon, 20 Nov 2023 11:46:12 +0800 Subject: [PATCH] metrics(ticdc): fix grafana configuration of ticdc (#10083) (#10111) close pingcap/tiflow#10055 --- metrics/grafana/ticdc.json | 9063 ++++++++++++++++++------------------ 1 file changed, 4639 insertions(+), 4424 deletions(-) diff --git a/metrics/grafana/ticdc.json b/metrics/grafana/ticdc.json index 407c341a652..95795f7cd67 100644 --- a/metrics/grafana/ticdc.json +++ b/metrics/grafana/ticdc.json @@ -131,7 +131,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1694492203479, + "iteration": 1699845194714, "links": [], "panels": [ { @@ -8585,15 +8585,15 @@ "type": "row" }, { - "id": 10000, "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 18 + "y": 7 }, + "id": 713, "panels": [ { "aliasColors": {}, @@ -8608,13 +8608,13 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 9, + "h": 8, "w": 12, "x": 0, - "y": 1 + "y": 8 }, "hiddenSeries": false, - "id": 10001, + "id": 714, "legend": { "avg": false, "current": false, @@ -8626,7 +8626,7 @@ }, "lines": true, "linewidth": 1, - "nullPointMode": "null as zero", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -8637,24 +8637,15 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2", - "format": "time_series", - "interval": "", - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" - }, - { - "exemplar": true, - "expr": "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}", - "format": "time_series", + "expr": "sum(rate(ticdc_sink_cloud_storage_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace, changefeed, instance)", + "hide": false, "interval": "", - "legendFormat": "", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", "queryType": "randomWalk", "refId": "A" } @@ -8663,7 +8654,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pulsar published DDL schema count", + "title": "Write Bytes/s", "tooltip": { "shared": true, "sort": 0, @@ -8679,7 +8670,6 @@ }, "yaxes": [ { - "$$hashKey": "object:97", "format": "short", "label": null, "logBase": 1, @@ -8688,7 +8678,6 @@ "show": true }, { - "$$hashKey": "object:98", "format": "short", "label": null, "logBase": 1, @@ -8702,7 +8691,6 @@ "alignLevel": null } }, - { "aliasColors": {}, "bars": false, @@ -8716,13 +8704,13 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 9, + "h": 8, "w": 12, "x": 12, - "y": 1 + "y": 8 }, "hiddenSeries": false, - "id": 10002, + "id": 715, "legend": { "avg": false, "current": false, @@ -8734,7 +8722,7 @@ }, "lines": true, "linewidth": 1, - "nullPointMode": "null as zero", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -8745,24 +8733,14 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2", - "format": "time_series", - "interval": "", - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" - }, - { - "exemplar": true, - "expr": "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}", - "format": "time_series", + "expr": "sum(ticdc_sink_cloud_storage_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, instance)", "interval": "", - "legendFormat": "", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", "queryType": "randomWalk", "refId": "A" } @@ -8771,7 +8749,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pulsar published DDL schema success", + "title": "File Count", "tooltip": { "shared": true, "sort": 0, @@ -8787,7 +8765,6 @@ }, "yaxes": [ { - "$$hashKey": "object:97", "format": "short", "label": null, "logBase": 1, @@ -8796,7 +8773,6 @@ "show": true }, { - "$$hashKey": "object:98", "format": "short", "label": null, "logBase": 1, @@ -8810,12 +8786,165 @@ "alignLevel": null } }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The latency distributions of write storage by a cloud storage sink", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 10015, + "legend": { + "show": true + }, + "pluginVersion": "7.5.11", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "max(rate(ticdc_sink_cloud_storage_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", + "format": "heatmap", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Write duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The latency distributions of flush storage by a cloud storage sink", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 10017, + "legend": { + "show": true + }, + "pluginVersion": "7.5.11", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "max(rate(ticdc_sink_cloud_storage_flush_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", + "format": "heatmap", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Flush duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "Busy ratio (X ms in 1s) for cloud storage sink dml worker", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8823,26 +8952,25 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 9, + "h": 8, "w": 12, "x": 0, - "y": 10 + "y": 24 }, "hiddenSeries": false, - - "id": 10003, + "id": 10019, "legend": { - "avg": false, - "current": false, + "avg": true, + "current": true, "max": false, "min": false, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, - "nullPointMode": "null as zero", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -8853,24 +8981,15 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2\r", - "format": "time_series", - "interval": "", - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" - }, - { - "exemplar": true, - "expr": "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}", - "format": "time_series", + "expr": "sum(rate(ticdc_sink_cloud_storage_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])/10) by (namespace,changefeed,id,instance)", + "hide": false, "interval": "", - "legendFormat": "", + "legendFormat": "{{namespace}}-{{changefeed}}-{{id}}-{{instance}}", "queryType": "randomWalk", "refId": "A" } @@ -8879,7 +8998,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pulsar published DDL schema fail", + "title": "Worker Busy Ratio", "tooltip": { "shared": true, "sort": 0, @@ -8895,8 +9014,7 @@ }, "yaxes": [ { - "$$hashKey": "object:97", - "format": "short", + "format": "percent", "label": null, "logBase": 1, "max": null, @@ -8904,56 +9022,76 @@ "show": true }, { - "$$hashKey": "object:98", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { "align": false, "alignLevel": null } - }, + } + ], + "title": "Sink - Cloud Storage Sink", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 528, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.", "fieldConfig": { - "defaults": {}, + "defaults": { + "links": [] + }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 9, + "h": 4, "w": 12, - "x": 12, - "y": 10 + "x": 0, + "y": 9 }, "hiddenSeries": false, - - "id": 10004, + "id": 571, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, - "nullPointMode": "null as zero", + "links": [], + "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, @@ -8961,16 +9099,16 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "increase(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",\r\nschema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2", + "expr": "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", "format": "time_series", "interval": "", - "legendFormat": "", - "queryType": "randomWalk", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}", "refId": "A" } ], @@ -8978,7 +9116,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pulsar published DML schema count", + "title": "Changefeed checkpoint lag", "tooltip": { "shared": true, "sort": 0, @@ -8994,16 +9132,14 @@ }, "yaxes": [ { - "$$hashKey": "object:97", - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "$$hashKey": "object:98", "format": "short", "label": null, "logBase": 1, @@ -9023,36 +9159,42 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The lag between changefeed resolved ts and the latest ts of upstream TiDB.", "fieldConfig": { - "defaults": {}, + "defaults": { + "links": [] + }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 9, + "h": 4, "w": 12, - "x": 0, - "y": 19 + "x": 12, + "y": 9 }, "hiddenSeries": false, - - "id": 10005, + "id": 589, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, - "nullPointMode": "null as zero", + "links": [], + "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, @@ -9060,24 +9202,24 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2", + "expr": "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", "format": "time_series", "interval": "", - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-resolvedts", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pulsar published DML schema success", + "title": "Changefeed resolved ts lag", "tooltip": { "shared": true, "sort": 0, @@ -9093,16 +9235,14 @@ }, "yaxes": [ { - "$$hashKey": "object:97", - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "$$hashKey": "object:98", "format": "short", "label": null, "logBase": 1, @@ -9122,62 +9262,78 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "description": "The total number of tables in different replication states.\n\n0: ReplicationSetStateUnknown means the replication state is unknown, it should not happen.\n\n1: ReplicationSetStateAbsent means there is no one replicates or prepares it.\n\n2: ReplicationSetStatePrepare means it needs to add a secondary.\n\n3: ReplicationSetStateCommit means it needs to promote secondary to primary.\n\n4: ReplicationSetStateReplicating means there is exactly one capture that is replicating the table.\n\n5: ReplicationSetStateRemoving means all captures need to stop replication eventually.\n\n", + "editable": true, + "error": false, "fieldConfig": { - "defaults": {}, + "defaults": { + "unit": "none" + }, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { - "h": 9, + "h": 6, "w": 12, - "x": 12, - "y": 19 + "x": 0, + "y": 13 }, "hiddenSeries": false, - - "id": 10006, + "id": 546, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": null, "total": false, "values": false }, "lines": true, "linewidth": 1, - "nullPointMode": "null as zero", + "links": [], + "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, + "pointradius": 1, + "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2", + "expr": "sum(ticdc_scheduler_table_replication_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, state)", "format": "time_series", + "hide": false, "interval": "", - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "{{namespace}}-{{changefeed}}-{{state}}", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pulsar published DML schema fail", + "title": "Table replication state", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -9192,22 +9348,20 @@ }, "yaxes": [ { - "$$hashKey": "object:97", - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "$$hashKey": "object:98", - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -9221,63 +9375,78 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "description": "The total number of accepted schedule tasks.", + "editable": true, + "error": false, "fieldConfig": { - "defaults": {}, + "defaults": { + "unit": "none" + }, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { - "h": 9, + "h": 6, "w": 12, - "x": 0, - "y": 28 + "x": 12, + "y": 13 }, "hiddenSeries": false, - - "id": 10007, + "id": 549, "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, - "current": true, + "current": false, "max": false, "min": false, + "rightSide": false, "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": null, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "nullPointMode": "null as zero", + "links": [], + "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, + "pointradius": 1, + "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "sum(pulsar_client_bytes_published{changefeed=~\"$changefeed\"}) by (changefeed, instance)", + "expr": "sum(ticdc_scheduler_task_accept{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)", "format": "time_series", + "hide": false, "interval": "", - "legendFormat": "{{changefeed}}-{{instance}}", - "queryType": "randomWalk", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "{{namespace}}-{{changefeed}}-{{task}}", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pulsar client bytes published ", + "title": "Schedule tasks accept", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -9292,22 +9461,20 @@ }, "yaxes": [ { - "$$hashKey": "object:97", - "format": "bytes", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "$$hashKey": "object:98", - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -9321,65 +9488,78 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "description": "The total number of different schedule tasks.", + "editable": true, + "error": false, "fieldConfig": { - "defaults": {}, + "defaults": { + "unit": "none" + }, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { - "h": 9, + "h": 6, "w": 12, - "x": 12, - "y": 28 + "x": 0, + "y": 19 }, "hiddenSeries": false, - - "id": 10008, + "id": 548, "legend": { "alignAsTable": false, "avg": false, - "current": true, + "current": false, "max": false, "min": false, + "rightSide": false, "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": null, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "nullPointMode": "null as zero", + "links": [], + "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, + "pointradius": 1, + "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "sum(pulsar_client_connections_opened{changefeed=~\"$changefeed\"}) by (changefeed, instance)", + "expr": "sum(ticdc_scheduler_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)", "format": "time_series", - "instant": false, + "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "{{changefeed}}-{{instance}}", - "queryType": "randomWalk", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "{{namespace}}-{{changefeed}}-{{scheduler}}-{{task}}", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pulsar client connections opened", + "title": "Schedule tasks", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -9394,22 +9574,20 @@ }, "yaxes": [ { - "$$hashKey": "object:97", "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "$$hashKey": "object:98", - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -9423,65 +9601,78 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "description": "The total number of running schedule tasks.", + "editable": true, + "error": false, "fieldConfig": { - "defaults": {}, + "defaults": { + "unit": "none" + }, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { - "h": 9, + "h": 6, "w": 12, - "x": 0, - "y": 37 + "x": 12, + "y": 19 }, "hiddenSeries": false, - - "id": 10009, + "id": 588, "legend": { "alignAsTable": false, "avg": false, - "current": true, + "current": false, "max": false, "min": false, + "rightSide": false, "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": null, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "nullPointMode": "null as zero", + "links": [], + "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, + "pointradius": 1, + "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "sum(pulsar_client_rpc_count{changefeed=~\"$changefeed\"}) by (changefeed, instance)", + "expr": "sum(ticdc_scheduler_task_running{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)", "format": "time_series", - "instant": false, + "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "{{changefeed}}-{{instance}}", - "queryType": "randomWalk", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "{{namespace}}-{{changefeed}}-{{task}}", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pulsar client rpc count", + "title": "Schedule tasks running", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -9496,22 +9687,20 @@ }, "yaxes": [ { - "$$hashKey": "object:97", "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "$$hashKey": "object:98", - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -9519,139 +9708,52 @@ "alignLevel": null } }, - { - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 37 - }, - "id": 10010, - "options": { - "graph": {}, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltipOptions": { - "mode": "single" - } - }, - "pluginVersion": "7.5.11", - "targets": [ - { - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=~\"$changefeed\"}[1m])) by (le,changefeed,instance))\r", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{changefeed}}-{{instance}}-P999", - "queryType": "randomWalk", - "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[1m])) by (changefeed,instance) / \r\nsum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[1m])) by (changefeed,instance)", - "hide": false, - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pulsar client producer latency ", - "type": "timeseries" - }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The total number of tables", "fieldConfig": { "defaults": { - "unit": "s" + "links": [] }, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, "gridPos": { - "h": 9, + "h": 6, "w": 12, "x": 0, - "y": 46 + "y": 25 }, "hiddenSeries": false, - - "id": 10011, + "id": 545, "legend": { + "alignAsTable": false, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, + "rightSide": false, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, - "nullPointMode": "null as zero", + "links": [], + "nullPointMode": "null", "options": { - "alertThreshold": true + "alertThreshold": false }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, - "points": false, + "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -9660,13 +9762,11 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(pulsar_client_producer_rpc_latency_seconds_bucket{changefeed=~\"$changefeed\"}[1m])) by (le,changefeed,instance))\r", + "expr": "sum(ticdc_scheduler_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", "format": "time_series", - "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{changefeed}}-{{instance}}", - "queryType": "randomWalk", + "legendFormat": "{{namespace}}-{{changefeed}}", "refId": "A" } ], @@ -9674,7 +9774,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pulsar client producer rpc latency", + "title": "Table count", "tooltip": { "shared": true, "sort": 0, @@ -9690,8 +9790,7 @@ }, "yaxes": [ { - "$$hashKey": "object:809", - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -9699,7 +9798,6 @@ "show": true }, { - "$$hashKey": "object:810", "format": "short", "label": null, "logBase": 1, @@ -9719,43 +9817,53 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", + "decimals": 1, + "description": "The number of tables in different capture.", + "editable": true, + "error": false, "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { - "h": 9, + "h": 6, "w": 12, "x": 12, - "y": 46 + "y": 25 }, "hiddenSeries": false, - - "id": 10012, + "id": 547, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": null, "total": false, "values": false }, "lines": true, "linewidth": 1, - "nullPointMode": "null as zero", + "links": [], + "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, + "pointradius": 1, + "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -9763,23 +9871,25 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(pulsar_client_producer_pending_messages{changefeed=~\"$changefeed\"}) by (changefeed, instance)\r", + "exemplar": false, + "expr": "sum(ticdc_scheduler_capture_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, addr)", "format": "time_series", + "hide": false, "instant": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "{{changefeed}}-{{instance}}", - "queryType": "randomWalk", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "{{namespace}}-{{changefeed}}-{{addr}}", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pulsar client producer pending messages", + "title": "Capture tables count", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -9794,22 +9904,20 @@ }, "yaxes": [ { - "$$hashKey": "object:809", "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "$$hashKey": "object:810", - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -9823,43 +9931,51 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", + "decimals": 1, + "description": "The ID of the slowest table", + "editable": true, + "error": false, "fieldConfig": { - "defaults": { - "unit": "none" - }, + "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { - "h": 9, + "h": 6, "w": 12, - "x": 12, - "y": 46 + "x": 0, + "y": 31 }, "hiddenSeries": false, - - "id": 10013, + "id": 543, "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": false, "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": null, "total": false, - "values": false + "values": true }, - "lines": true, + "lines": false, "linewidth": 1, - "nullPointMode": "null as zero", + "links": [], + "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, + "pointradius": 1, + "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -9868,22 +9984,23 @@ "targets": [ { "exemplar": true, - "expr": "sum(published_message_type_resolved_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\"}) by (changefeed)", + "expr": "sum(ticdc_scheduler_slow_table_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", "format": "time_series", - "instant": false, + "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "{{changefeed}}-{{topic}}", - "queryType": "randomWalk", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "{{namespace}}-{{changefeed}}", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Pulsar client producer pending messages", + "title": "Slowest table ID", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -9898,83 +10015,80 @@ }, "yaxes": [ { - "$$hashKey": "object:809", "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "$$hashKey": "object:810", - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { "align": false, "alignLevel": null } - } - ], - "title": "Pulsar Sink", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 713, - "panels": [ + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "description": "The state of the slowest table.\n\n0: ReplicationSetStateUnknown means the replication state is unknown, it should not happen.\n\n1: ReplicationSetStateAbsent means there is no one replicates or prepares it.\n\n2: ReplicationSetStatePrepare means it needs to add a secondary.\n\n3: ReplicationSetStateCommit means it needs to promote secondary to primary.\n\n4: ReplicationSetStateReplicating means there is exactly one capture that is replicating the table.\n\n5: ReplicationSetStateRemoving means all captures need to stop replication eventually.\n\n", + "editable": true, + "error": false, "fieldConfig": { - "defaults": {}, + "defaults": { + "unit": "none" + }, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { - "h": 8, + "h": 6, "w": 12, - "x": 0, - "y": 8 + "x": 12, + "y": 31 }, "hiddenSeries": false, - "id": 714, + "id": 544, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": null, "total": false, "values": false }, - "lines": true, + "lines": false, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, + "pointradius": 1, + "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -9983,19 +10097,23 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sink_cloud_storage_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)\n", + "expr": "sum(ticdc_scheduler_slow_table_replication_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", + "format": "time_series", + "hide": false, "interval": "", - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", - "queryType": "randomWalk", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "{{namespace}}-{{changefeed}}", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Write Bytes/s", + "title": "Slowest table replication state", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -10010,20 +10128,20 @@ }, "yaxes": [ { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -10033,35 +10151,43 @@ }, { "aliasColors": {}, - "bars": false, + "bars": true, + "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The checkpoint ts of the slowest table.", "fieldConfig": { - "defaults": {}, + "defaults": { + "unit": "dateTimeAsIso" + }, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, "gridPos": { - "h": 8, + "h": 6, "w": 12, - "x": 12, - "y": 8 + "x": 0, + "y": 37 }, "hiddenSeries": false, - "id": 715, + "id": 541, "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": false, "show": true, + "sideWidth": null, "total": false, - "values": false + "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, + "links": [], "nullPointMode": "null", "options": { "alertThreshold": true @@ -10071,25 +10197,42 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/approximate current time.*/", + "bars": false + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "sum(ticdc_sink_cloud_storage_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, instance)", + "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", + "format": "time_series", + "hide": false, "interval": "", - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", - "queryType": "randomWalk", + "intervalFactor": 1, + "legendFormat": "approximate current time (s)", "refId": "A" + }, + { + "exemplar": true, + "expr": "max(ticdc_scheduler_slow_table_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "File Count", + "title": "Slowest table checkpoint", "tooltip": { "shared": true, "sort": 0, @@ -10098,6 +10241,7 @@ "type": "graph", "xaxis": { "buckets": null, + "max": null, "mode": "time", "name": null, "show": true, @@ -10105,7 +10249,7 @@ }, "yaxes": [ { - "format": "short", + "format": "dateTimeAsIso", "label": null, "logBase": 1, "max": null, @@ -10118,96 +10262,96 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { "align": false, "alignLevel": null } - } - ], - "title": "Sink - Cloud Storage Sink", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 528, - "panels": [ + }, { "aliasColors": {}, - "bars": false, + "bars": true, + "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.", + "description": "The resolved ts of the slowest table.", "fieldConfig": { - "defaults": { - "links": [] - }, + "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, "gridPos": { - "h": 4, + "h": 6, "w": 12, - "x": 0, - "y": 9 + "x": 12, + "y": 37 }, "hiddenSeries": false, - "id": 571, + "id": 590, "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, "rightSide": false, "show": true, + "sideWidth": null, "total": false, - "values": false + "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/approximate current time.*/", + "bars": false + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", + "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", "format": "time_series", + "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}", + "legendFormat": "approximate current time (s)", "refId": "A" + }, + { + "exemplar": true, + "expr": "max(ticdc_scheduler_slow_table_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Changefeed checkpoint lag", + "title": "Slowest table resolved ts", "tooltip": { "shared": true, "sort": 0, @@ -10216,6 +10360,7 @@ "type": "graph", "xaxis": { "buckets": null, + "max": null, "mode": "time", "name": null, "show": true, @@ -10223,11 +10368,11 @@ }, "yaxes": [ { - "format": "s", + "format": "dateTimeAsIso", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -10236,7 +10381,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -10250,31 +10395,35 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The lag between changefeed resolved ts and the latest ts of upstream TiDB.", + "decimals": 1, + "description": "The number of incremental scan tasks per minute", "fieldConfig": { - "defaults": { - "links": [] - }, + "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 4, + "h": 6, "w": 12, - "x": 12, - "y": 9 + "x": 0, + "y": 43 }, "hiddenSeries": false, - "id": 589, + "id": 560, "legend": { "alignAsTable": false, "avg": false, "current": false, + "hideEmpty": false, + "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": false }, @@ -10288,7 +10437,7 @@ "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -10298,19 +10447,20 @@ "targets": [ { "exemplar": true, - "expr": "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", + "expr": "sum(increase(tikv_cdc_scan_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (type, instance)", "format": "time_series", + "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-resolvedts", - "refId": "C" + "legendFormat": "{{instance}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Changefeed resolved ts lag", + "title": "TiKV scan tasks/m", "tooltip": { "shared": true, "sort": 0, @@ -10326,15 +10476,15 @@ }, "yaxes": [ { - "format": "s", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -10346,7 +10496,22 @@ "align": false, "alignLevel": null } - }, + } + ], + "title": "Scheduler", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 677, + "panels": [ { "aliasColors": {}, "bars": false, @@ -10354,13 +10519,11 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The total number of tables in different replication states.\n\n0: ReplicationSetStateUnknown means the replication state is unknown, it should not happen.\n\n1: ReplicationSetStateAbsent means there is no one replicates or prepares it.\n\n2: ReplicationSetStatePrepare means it needs to add a secondary.\n\n3: ReplicationSetStateCommit means it needs to promote secondary to primary.\n\n4: ReplicationSetStateReplicating means there is exactly one capture that is replicating the table.\n\n5: ReplicationSetStateRemoving means all captures need to stop replication eventually.\n\n", + "description": "The ID of the slowest table", "editable": true, "error": false, "fieldConfig": { - "defaults": { - "unit": "none" - }, + "defaults": {}, "overrides": [] }, "fill": 0, @@ -10370,14 +10533,14 @@ "h": 6, "w": 12, "x": 0, - "y": 13 + "y": 10 }, "hiddenSeries": false, - "id": 546, + "id": 689, "legend": { "alignAsTable": false, "avg": false, - "current": false, + "current": true, "max": false, "min": false, "rightSide": false, @@ -10386,9 +10549,9 @@ "sort": null, "sortDesc": null, "total": false, - "values": false + "values": true }, - "lines": true, + "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null", @@ -10408,12 +10571,12 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_scheduler_table_replication_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, state)", + "expr": "sum(ticdc_scheduler_slow_table_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, - "legendFormat": "{{namespace}}-{{changefeed}}-{{state}}", + "legendFormat": "{{namespace}}-{{changefeed}}", "refId": "A", "step": 10 } @@ -10422,7 +10585,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Table replication state", + "title": "Slowest table ID", "tooltip": { "msResolution": false, "shared": true, @@ -10467,13 +10630,11 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The total number of accepted schedule tasks.", + "description": "The number of regions captured by the slowest table.", "editable": true, "error": false, "fieldConfig": { - "defaults": { - "unit": "none" - }, + "defaults": {}, "overrides": [] }, "fill": 0, @@ -10483,14 +10644,14 @@ "h": 6, "w": 12, "x": 12, - "y": 13 + "y": 10 }, "hiddenSeries": false, - "id": 549, + "id": 691, "legend": { "alignAsTable": false, "avg": false, - "current": false, + "current": true, "max": false, "min": false, "rightSide": false, @@ -10499,9 +10660,9 @@ "sort": null, "sortDesc": null, "total": false, - "values": false + "values": true }, - "lines": true, + "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null", @@ -10521,12 +10682,12 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_scheduler_task_accept{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)", + "expr": "sum(ticdc_scheduler_slow_table_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, - "legendFormat": "{{namespace}}-{{changefeed}}-{{task}}", + "legendFormat": "{{namespace}}-{{changefeed}}", "refId": "A", "step": 10 } @@ -10535,7 +10696,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Schedule tasks accept", + "title": "Slowest table region count", "tooltip": { "msResolution": false, "shared": true, @@ -10576,41 +10737,33 @@ { "aliasColors": {}, "bars": false, + "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The total number of different schedule tasks.", - "editable": true, - "error": false, + "description": "Checkpoint ts of the slowest table each stage.", "fieldConfig": { "defaults": { - "unit": "none" + "unit": "dateTimeAsIso" }, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, - "grid": {}, "gridPos": { "h": 6, "w": 12, "x": 0, - "y": 19 + "y": 16 }, "hiddenSeries": false, - "id": 548, + "id": 693, "legend": { - "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, "show": true, - "sideWidth": null, - "sort": null, - "sortDesc": null, "total": false, "values": false }, @@ -10621,11 +10774,10 @@ "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 1, - "points": true, + "pointradius": 2, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -10634,23 +10786,31 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_scheduler_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)", + "expr": "max(ticdc_owner_current_pd_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", "format": "time_series", "hide": false, "interval": "", - "intervalFactor": 2, - "legendFormat": "{{namespace}}-{{changefeed}}-{{scheduler}}-{{task}}", - "refId": "A", - "step": 10 + "intervalFactor": 1, + "legendFormat": "approximate current time (s)", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(ticdc_scheduler_slow_table_stage_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Schedule tasks", + "title": "Slow table checkpoint ts", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -10665,20 +10825,20 @@ }, "yaxes": [ { - "format": "none", + "format": "dateTimeAsIso", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -10689,41 +10849,33 @@ { "aliasColors": {}, "bars": false, + "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The total number of running schedule tasks.", - "editable": true, - "error": false, + "description": "Resolved ts of the slowest table each stage.", "fieldConfig": { "defaults": { - "unit": "none" + "unit": "dateTimeAsIso" }, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, - "grid": {}, "gridPos": { "h": 6, "w": 12, "x": 12, - "y": 19 + "y": 16 }, "hiddenSeries": false, - "id": 588, + "id": 695, "legend": { - "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, "show": true, - "sideWidth": null, - "sort": null, - "sortDesc": null, "total": false, "values": false }, @@ -10734,11 +10886,10 @@ "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 1, - "points": true, + "pointradius": 2, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -10747,23 +10898,31 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_scheduler_task_running{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)", + "expr": "max(ticdc_owner_current_pd_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", "format": "time_series", "hide": false, "interval": "", - "intervalFactor": 2, - "legendFormat": "{{namespace}}-{{changefeed}}-{{task}}", - "refId": "A", - "step": 10 + "intervalFactor": 1, + "legendFormat": "approximate current time (s)", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(ticdc_scheduler_slow_table_stage_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Schedule tasks running", + "title": "Slow table resolved ts", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -10778,20 +10937,20 @@ }, "yaxes": [ { - "format": "none", + "format": "dateTimeAsIso", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -10805,28 +10964,28 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The total number of tables", + "description": "The details of changefeed lag including resolved ts and checkpoint ts in different stages.", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, - "w": 12, + "w": 24, "x": 0, - "y": 25 + "y": 22 }, "hiddenSeries": false, - "id": 545, + "id": 699, "legend": { "alignAsTable": false, "avg": false, "current": true, - "max": true, + "max": false, "min": false, "rightSide": false, "show": true, @@ -10838,13 +10997,13 @@ "links": [], "nullPointMode": "null", "options": { - "alertThreshold": false + "alertThreshold": true }, "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, - "points": true, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -10853,19 +11012,35 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_scheduler_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", + "expr": "max(ticdc_scheduler_slow_table_stage_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}", + "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "max(tikv_cdc_min_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000)", + "hide": false, + "interval": "", + "legendFormat": "tikv", + "refId": "B" + }, + { + "exemplar": true, + "expr": "max(ticdc_scheduler_slow_table_stage_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", stage=\"sink\"}) by (namespace,changefeed)", + "hide": false, + "interval": "", + "legendFormat": "sink-checkpoint", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Table count", + "title": "Slow table lag details", "tooltip": { "shared": true, "sort": 0, @@ -10881,11 +11056,11 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -10908,53 +11083,42 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The number of tables in different capture.", - "editable": true, - "error": false, + "description": "Lag distribution of the slowest table in different stages.", "fieldConfig": { - "defaults": { - "unit": "none" - }, + "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, - "grid": {}, "gridPos": { "h": 6, - "w": 12, - "x": 12, - "y": 25 + "w": 24, + "x": 0, + "y": 28 }, "hiddenSeries": false, - "id": 547, + "id": 703, "legend": { "alignAsTable": false, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": false, "show": true, - "sideWidth": null, - "sort": null, - "sortDesc": null, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 1, - "points": true, + "pointradius": 2, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -10962,25 +11126,28 @@ "steppedLine": false, "targets": [ { - "exemplar": false, - "expr": "sum(ticdc_scheduler_capture_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, addr)", - "format": "time_series", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_scheduler_slow_table_stage_resolved_ts_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[100s])) by (le,namespace,changefeed,stage))", "hide": false, - "instant": false, "interval": "", - "intervalFactor": 2, - "legendFormat": "{{namespace}}-{{changefeed}}-{{addr}}", - "refId": "A", - "step": 10 + "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}-p99", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_scheduler_slow_table_stage_checkpoint_ts_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", stage=\"sink\"}[100s])) by (le,namespace,changefeed,stage))", + "hide": false, + "interval": "", + "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}-checkpoint-p99", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Capture tables count", + "title": "Slow table lag duration percentile", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -10995,11 +11162,11 @@ }, "yaxes": [ { - "format": "none", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -11008,65 +11175,68 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { "align": false, "alignLevel": null } - }, + } + ], + "title": "Slow Table", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 269, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The ID of the slowest table", - "editable": true, - "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, - "grid": {}, "gridPos": { "h": 6, - "w": 12, + "w": 8, "x": 0, - "y": 31 + "y": 11 }, "hiddenSeries": false, - "id": 543, + "id": 271, "legend": { - "alignAsTable": true, "avg": false, - "current": true, + "current": false, "max": false, "min": false, - "rightSide": false, "show": true, - "sideWidth": null, - "sort": null, - "sortDesc": null, "total": false, - "values": true + "values": false }, - "lines": false, + "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 1, - "points": true, + "pointradius": 2, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -11074,24 +11244,19 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(ticdc_scheduler_slow_table_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", + "expr": "sum(ticdc_sorter_on_disk_data_size_gauge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{namespace}}-{{changefeed}}", - "refId": "A", - "step": 10 + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Slowest table ID", + "title": "On disk data size", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -11106,7 +11271,7 @@ }, "yaxes": [ { - "format": "none", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -11114,12 +11279,12 @@ "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -11133,78 +11298,72 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The state of the slowest table.\n\n0: ReplicationSetStateUnknown means the replication state is unknown, it should not happen.\n\n1: ReplicationSetStateAbsent means there is no one replicates or prepares it.\n\n2: ReplicationSetStatePrepare means it needs to add a secondary.\n\n3: ReplicationSetStateCommit means it needs to promote secondary to primary.\n\n4: ReplicationSetStateReplicating means there is exactly one capture that is replicating the table.\n\n5: ReplicationSetStateRemoving means all captures need to stop replication eventually.\n\n", - "editable": true, - "error": false, "fieldConfig": { - "defaults": { - "unit": "none" - }, + "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, - "grid": {}, "gridPos": { "h": 6, - "w": 12, - "x": 12, - "y": 31 + "w": 8, + "x": 8, + "y": 11 }, "hiddenSeries": false, - "id": 544, + "id": 273, "legend": { - "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, "show": true, - "sideWidth": null, - "sort": null, - "sortDesc": null, "total": false, "values": false }, - "lines": false, + "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 1, - "points": true, + "pointradius": 2, + "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/cache-hit.*/", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(ticdc_scheduler_slow_table_replication_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", + "expr": "sum(ticdc_sorter_in_memory_data_size_gauge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{namespace}}-{{changefeed}}", - "refId": "A", - "step": 10 + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + }, + { + "expr": "sum(ticdc_db_block_cache_access_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"hit\"}) by (instance) / sum(ticdc_db_block_cache_access_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "cache-hit-{{instance}}", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Slowest table replication state", + "title": "In-memory data size", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -11219,7 +11378,7 @@ }, "yaxes": [ { - "format": "none", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -11227,12 +11386,12 @@ "show": true }, { - "format": "none", + "format": "percentunit", "label": null, "logBase": 1, "max": null, - "min": null, - "show": false + "min": "0", + "show": true } ], "yaxis": { @@ -11242,88 +11401,65 @@ }, { "aliasColors": {}, - "bars": true, - "cacheTimeout": null, + "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The checkpoint ts of the slowest table.", + "description": "The count of files of each levels", "fieldConfig": { - "defaults": { - "unit": "dateTimeAsIso" - }, + "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, - "w": 12, - "x": 0, - "y": 37 + "w": 8, + "x": 16, + "y": 11 }, "hiddenSeries": false, - "id": 541, + "id": 272, "legend": { - "alignAsTable": true, "avg": false, "current": true, - "max": false, + "max": true, "min": false, - "rightSide": false, "show": true, - "sideWidth": null, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/approximate current time.*/", - "bars": false - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", + "expr": "sum(ticdc_db_level_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, level)", "format": "time_series", - "hide": false, - "interval": "", "intervalFactor": 1, - "legendFormat": "approximate current time (s)", + "legendFormat": "{{instance}}-{{level}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "max(ticdc_scheduler_slow_table_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Slowest table checkpoint", + "title": "Level files", "tooltip": { "shared": true, "sort": 0, @@ -11332,7 +11468,6 @@ "type": "graph", "xaxis": { "buckets": null, - "max": null, "mode": "time", "name": null, "show": true, @@ -11340,7 +11475,7 @@ }, "yaxes": [ { - "format": "dateTimeAsIso", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -11353,7 +11488,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -11362,41 +11497,122 @@ } }, { - "aliasColors": {}, - "bars": true, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "The resolved ts of the slowest table.", + "description": "The time of sorter write", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 0, - "fillGradient": 0, "gridPos": { "h": 6, - "w": 12, - "x": 12, - "y": 37 + "w": 8, + "x": 0, + "y": 17 }, - "hiddenSeries": false, - "id": 590, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 274, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "maxPerRow": 3, + "repeatDirection": "h", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sorter_db_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Write duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 17 + }, + "hiddenSeries": false, + "id": 288, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, "min": false, "rightSide": false, "show": true, - "sideWidth": null, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null", "options": { @@ -11409,8 +11625,8 @@ "renderer": "flot", "seriesOverrides": [ { - "alias": "/approximate current time.*/", - "bars": false + "alias": "/.*amplification/", + "yaxis": 2 } ], "spaceLength": 10, @@ -11419,30 +11635,40 @@ "targets": [ { "exemplar": true, - "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", + "expr": "sum(rate(ticdc_sorter_db_write_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "approximate current time (s)", + "legendFormat": "{{instance}}-sorter", "refId": "A" }, { "exemplar": true, - "expr": "max(ticdc_scheduler_slow_table_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", + "expr": "sum(rate(ticdc_sorter_db_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_sorter_db_write_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}", + "legendFormat": "{{instance}}-amplification", "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_sorter_db_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}-disk", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Slowest table resolved ts", + "title": "Write speed", "tooltip": { "shared": true, "sort": 0, @@ -11451,7 +11677,6 @@ "type": "graph", "xaxis": { "buckets": null, - "max": null, "mode": "time", "name": null, "show": true, @@ -11459,20 +11684,20 @@ }, "yaxes": [ { - "format": "dateTimeAsIso", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "short", + "format": "percentunit", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -11486,37 +11711,34 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The number of incremental scan tasks per minute", + "description": "CPU usage of LevelDB sorter", "fieldConfig": { - "defaults": {}, + "defaults": { + "links": [] + }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, - "w": 12, - "x": 0, - "y": 43 + "w": 8, + "x": 16, + "y": 17 }, "hiddenSeries": false, - "id": 560, + "id": 286, "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": true, - "max": false, + "current": true, + "hideEmpty": true, + "max": true, "min": false, "rightSide": false, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -11528,22 +11750,24 @@ "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/.*sorter-[0-9]+/", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(increase(tikv_cdc_scan_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (type, instance)", + "expr": "sum(rate(ticdc_actor_worker_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", name=~\"sorter.*\"}[1m])) by (name, instance)", "format": "time_series", - "hide": false, - "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}", + "legendFormat": "{{instance}}-{{name}}", "refId": "A" } ], @@ -11551,7 +11775,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "TiKV scan tasks/m", + "title": "CPU usage", "tooltip": { "shared": true, "sort": 0, @@ -11567,7 +11791,7 @@ }, "yaxes": [ { - "format": "none", + "format": "percentunit", "label": null, "logBase": 1, "max": null, @@ -11575,10 +11799,10 @@ "show": true }, { - "format": "none", + "format": "percentunit", "label": null, "logBase": 1, - "max": null, + "max": "1.2", "min": null, "show": true } @@ -11587,132 +11811,90 @@ "align": false, "alignLevel": null } - } - ], - "title": "Scheduler", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 677, - "panels": [ + }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The ID of the slowest table", - "editable": true, - "error": false, + "description": "The time of sorter write", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 0, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 6, - "w": 12, + "w": 8, "x": 0, - "y": 10 + "y": 23 }, - "hiddenSeries": false, - "id": 689, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 277, "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, - "max": false, + "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, - "sideWidth": null, - "sort": null, - "sortDesc": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, - "lines": false, - "linewidth": 1, "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 1, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "maxPerRow": 3, + "repeatDirection": "h", + "reverseYBuckets": false, "targets": [ { "exemplar": true, - "expr": "sum(ticdc_scheduler_slow_table_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", - "format": "time_series", - "hide": false, + "expr": "sum(rate(ticdc_sorter_db_write_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, "interval": "", "intervalFactor": 2, - "legendFormat": "{{namespace}}-{{changefeed}}", - "refId": "A", - "step": 10 + "legendFormat": "{{le}}", + "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Slowest table ID", + "title": "Write bytes", "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" + "show": true, + "showHistogram": true }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null }, { "aliasColors": {}, @@ -11720,51 +11902,42 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The number of regions captured by the slowest table.", - "editable": true, - "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, - "grid": {}, "gridPos": { "h": 6, - "w": 12, - "x": 12, - "y": 10 + "w": 8, + "x": 8, + "y": 23 }, "hiddenSeries": false, - "id": 691, + "id": 279, "legend": { - "alignAsTable": false, - "avg": false, + "alignAsTable": true, + "avg": true, "current": true, - "max": false, + "max": true, "min": false, "rightSide": false, "show": true, - "sideWidth": null, - "sort": null, - "sortDesc": null, "total": false, "values": true }, - "lines": false, + "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 1, - "points": true, + "pointradius": 2, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -11773,23 +11946,21 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_scheduler_slow_table_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", + "expr": "sum(rate(ticdc_sorter_db_write_bytes_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "format": "time_series", "hide": false, "interval": "", - "intervalFactor": 2, - "legendFormat": "{{namespace}}-{{changefeed}}", - "refId": "A", - "step": 10 + "intervalFactor": 1, + "legendFormat": "{{instance}}-sorter", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Slowest table region count", + "title": "Write OPS", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -11804,7 +11975,7 @@ }, "yaxes": [ { - "format": "none", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -11812,12 +11983,12 @@ "show": true }, { - "format": "none", + "format": "percentunit", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -11828,14 +11999,13 @@ { "aliasColors": {}, "bars": false, - "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Checkpoint ts of the slowest table each stage.", + "description": "The count and duration of write delay", "fieldConfig": { "defaults": { - "unit": "dateTimeAsIso" + "links": [] }, "overrides": [] }, @@ -11843,20 +12013,22 @@ "fillGradient": 0, "gridPos": { "h": 6, - "w": 12, - "x": 0, - "y": 16 + "w": 8, + "x": 16, + "y": 23 }, "hiddenSeries": false, - "id": 693, + "id": 275, "legend": { + "alignAsTable": false, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, + "rightSide": false, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -11865,34 +12037,39 @@ "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/.*count.*/", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "max(ticdc_owner_current_pd_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", + "expr": "sum(rate(ticdc_db_write_delay_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "format": "time_series", - "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "approximate current time (s)", + "legendFormat": "{{instance}}-duration", "refId": "A" }, { "exemplar": true, - "expr": "sum(ticdc_scheduler_slow_table_stage_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)", + "expr": "sum(rate(ticdc_db_write_delay_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}", + "legendFormat": "{{instance}}-count", "refId": "B" } ], @@ -11900,7 +12077,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Slow table checkpoint ts", + "title": "Write delay", "tooltip": { "shared": true, "sort": 0, @@ -11916,7 +12093,7 @@ }, "yaxes": [ { - "format": "dateTimeAsIso", + "format": "dtdurations", "label": null, "logBase": 1, "max": null, @@ -11938,116 +12115,256 @@ } }, { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "Resolved ts of the slowest table each stage.", + "description": "The time of sorter iterator read", "fieldConfig": { - "defaults": { - "unit": "dateTimeAsIso" - }, + "defaults": {}, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 6, - "w": 12, - "x": 12, - "y": 16 + "w": 8, + "x": 0, + "y": 29 }, - "hiddenSeries": false, - "id": 695, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 278, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, + "rightSide": true, "show": true, + "sort": "current", + "sortDesc": true, "total": false, - "values": false + "values": true }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "maxPerRow": 3, + "repeatDirection": "h", + "reverseYBuckets": false, "targets": [ { "exemplar": true, - "expr": "max(ticdc_owner_current_pd_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", - "format": "time_series", - "hide": false, + "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "approximate current time (s)", + "intervalFactor": 2, + "legendFormat": "{{le}}", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(ticdc_scheduler_slow_table_stage_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}", - "refId": "B" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Slow table resolved ts", + "title": "Read duration - First", "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, "show": true, - "values": [] + "showHistogram": true }, - "yaxes": [ + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The time of sorter iterator read", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 29 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 282, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "maxPerRow": 3, + "repeatDirection": "h", + "reverseYBuckets": false, + "targets": [ { - "format": "dateTimeAsIso", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, + "exemplar": true, + "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Read duration - Next", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The time of sorter iterator read", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 29 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 281, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "maxPerRow": 3, + "repeatDirection": "h", + "reverseYBuckets": false, + "targets": [ { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "exemplar": true, + "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"release\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Read duration - Release", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null }, { "aliasColors": {}, @@ -12055,28 +12372,25 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The details of changefeed lag including resolved ts and checkpoint ts in different stages.", "fieldConfig": { - "defaults": { - "links": [] - }, + "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, - "w": 24, + "w": 8, "x": 0, - "y": 22 + "y": 35 }, "hiddenSeries": false, - "id": 699, + "id": 280, "legend": { - "alignAsTable": false, - "avg": false, + "alignAsTable": true, + "avg": true, "current": true, - "max": false, + "max": true, "min": false, "rightSide": false, "show": true, @@ -12090,7 +12404,6 @@ "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, @@ -12103,35 +12416,20 @@ "targets": [ { "exemplar": true, - "expr": "max(ticdc_scheduler_slow_table_stage_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)", + "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[1m])) by (instance)", "format": "time_series", + "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}", + "legendFormat": "{{instance}}-sorter", "refId": "A" - }, - { - "exemplar": true, - "expr": "max(tikv_cdc_min_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000)", - "hide": false, - "interval": "", - "legendFormat": "tikv", - "refId": "B" - }, - { - "exemplar": true, - "expr": "max(ticdc_scheduler_slow_table_stage_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", stage=\"sink\"}) by (namespace,changefeed)", - "hide": false, - "interval": "", - "legendFormat": "sink-checkpoint", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Slow table lag details", + "title": "Read OPS - First", "tooltip": { "shared": true, "sort": 0, @@ -12147,7 +12445,7 @@ }, "yaxes": [ { - "format": "s", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -12155,9 +12453,9 @@ "show": true }, { - "format": "short", + "format": "percentunit", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": null, "show": true @@ -12174,7 +12472,6 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Lag distribution of the slowest table in different stages.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -12183,15 +12480,15 @@ "fillGradient": 0, "gridPos": { "h": 6, - "w": 24, - "x": 0, - "y": 28 + "w": 8, + "x": 8, + "y": 35 }, "hiddenSeries": false, - "id": 703, + "id": 283, "legend": { - "alignAsTable": false, - "avg": false, + "alignAsTable": true, + "avg": true, "current": true, "max": true, "min": false, @@ -12202,6 +12499,7 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { "alertThreshold": true @@ -12218,26 +12516,20 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_scheduler_slow_table_stage_resolved_ts_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[100s])) by (le,namespace,changefeed,stage))", + "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[1m])) by (instance)", + "format": "time_series", "hide": false, "interval": "", - "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}-p99", + "intervalFactor": 1, + "legendFormat": "{{instance}}-sorter", "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_scheduler_slow_table_stage_checkpoint_ts_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", stage=\"sink\"}[100s])) by (le,namespace,changefeed,stage))", - "hide": false, - "interval": "", - "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}-checkpoint-p99", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Slow table lag duration percentile", + "title": "Read OPS - Next", "tooltip": { "shared": true, "sort": 0, @@ -12253,17 +12545,17 @@ }, "yaxes": [ { - "format": "s", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "none", + "format": "percentunit", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": null, "show": true @@ -12273,22 +12565,7 @@ "align": false, "alignLevel": null } - } - ], - "title": "Slow Table", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 269, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -12304,22 +12581,25 @@ "gridPos": { "h": 6, "w": 8, - "x": 0, - "y": 11 + "x": 16, + "y": 35 }, "hiddenSeries": false, - "id": 271, + "id": 287, "legend": { - "avg": false, - "current": false, - "max": false, + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, "min": false, + "rightSide": false, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { "alertThreshold": true @@ -12335,10 +12615,13 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_sorter_on_disk_data_size_gauge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", + "exemplar": true, + "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"release\"}[1m])) by (instance)", "format": "time_series", + "hide": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}", + "legendFormat": "{{instance}}-sorter", "refId": "A" } ], @@ -12346,7 +12629,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "On disk data size", + "title": "Read OPS - Release", "tooltip": { "shared": true, "sort": 0, @@ -12362,7 +12645,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -12370,9 +12653,9 @@ "show": true }, { - "format": "short", + "format": "percentunit", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": null, "show": true @@ -12383,34 +12666,139 @@ "alignLevel": null } }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The time of sorter compact", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 41 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 285, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "maxPerRow": 3, + "repeatDirection": "h", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sorter_db_compact_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Compact duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + } + ], + "title": "DB", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 447, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The number of established Eventfeed RPC between TiCDC and TiKV", "fieldConfig": { - "defaults": {}, + "defaults": { + "links": [] + }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 11 + "h": 7, + "w": 12, + "x": 0, + "y": 12 }, "hiddenSeries": false, - "id": 273, + "id": 15, "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, + "hideEmpty": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -12419,6 +12807,7 @@ "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, @@ -12426,7 +12815,7 @@ "renderer": "flot", "seriesOverrides": [ { - "alias": "/cache-hit.*/", + "alias": "/.*-rpc/", "yaxis": 2 } ], @@ -12435,25 +12824,41 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_sorter_in_memory_data_size_gauge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", + "expr": "sum(ticdc_kvclient_event_feed_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", "refId": "A" }, { - "expr": "sum(ticdc_db_block_cache_access_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"hit\"}) by (instance) / sum(ticdc_db_block_cache_access_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", + "expr": "sum(grpc_client_started_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance) - sum(grpc_client_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "cache-hit-{{instance}}", + "legendFormat": "{{instance}}-rpc", "refId": "B" + }, + { + "expr": "sum(grpc_client_started_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}-rpc-started", + "refId": "C" + }, + { + "expr": "sum(grpc_client_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}-rpc-handled", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "In-memory data size", + "title": "Eventfeed count", "tooltip": { "shared": true, "sort": 0, @@ -12469,19 +12874,19 @@ }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ], @@ -12496,26 +12901,32 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The count of files of each levels", + "description": "The number of KV client received events from TiKV per seconds", "fieldConfig": { - "defaults": {}, + "defaults": { + "links": [] + }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 11 + "h": 7, + "w": 12, + "x": 12, + "y": 12 }, "hiddenSeries": false, - "id": 272, + "id": 29, "legend": { + "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, + "rightSide": false, "show": true, "total": false, "values": true @@ -12539,10 +12950,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_db_level_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, level)", + "expr": "sum(rate(ticdc_kvclient_pull_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{instance}}-{{level}}", + "legendFormat": "{{instance}}-{{type}}", "refId": "A" } ], @@ -12550,7 +12961,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Level files", + "title": "KV client receive events/s", "tooltip": { "shared": true, "sort": 0, @@ -12588,88 +12999,126 @@ } }, { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The time of sorter write", + "description": "The number of errors that interrupt Eventfeed RPC", "fieldConfig": { - "defaults": {}, + "defaults": { + "links": [] + }, "overrides": [] }, + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 6, - "w": 8, + "h": 7, + "w": 12, "x": 0, - "y": 17 + "y": 19 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 274, + "hiddenSeries": false, + "id": 28, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, + "lines": true, + "linewidth": 1, "links": [], - "maxPerRow": 3, - "repeatDirection": "h", - "reverseYBuckets": false, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", + "expr": "sum(increase(ticdc_kvclient_event_feed_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{type}}", "refId": "A" + }, + { + "expr": "-sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*leader\"}[1m]))", + "format": "time_series", + "hide": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "transfer-leader", + "refId": "B" + }, + { + "expr": "-sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*(peer|region)\"}[1m]))", + "format": "time_series", + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "move-region", + "refId": "C" } ], - "title": "Write duration", + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Eventfeed error/m", "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true + "shared": true, + "sort": 0, + "value_type": "individual" }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, "show": true, - "splitFactor": null + "values": [] }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -12677,27 +13126,32 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The number of KV client dispatched event per second", "fieldConfig": { - "defaults": {}, + "defaults": { + "links": [] + }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 17 + "h": 7, + "w": 12, + "x": 12, + "y": 19 }, "hiddenSeries": false, - "id": 288, + "id": 31, "legend": { "alignAsTable": true, - "avg": true, + "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -12709,6 +13163,7 @@ "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, @@ -12716,7 +13171,7 @@ "renderer": "flot", "seriesOverrides": [ { - "alias": "/.*amplification/", + "alias": "/.*batch-resolved/", "yaxis": 2 } ], @@ -12725,41 +13180,25 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_write_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "expr": "sum(rate(ticdc_kvclient_send_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[1m])) by (instance, namespace,changefeed, type)", "format": "time_series", - "hide": false, - "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}-sorter", + "legendFormat": "{{instance}}-{{namespace}}-{{changefeed}}-{{type}}", "refId": "A" }, { - "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_sorter_db_write_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance,namespace, changefeed, table)", "format": "time_series", - "hide": false, - "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}-amplification", + "legendFormat": "{{instance}}-{{namespace}}-{{changefeed}}-batch-resolved", "refId": "B" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}-disk", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Write speed", + "title": "KV client dispatch events/s", "tooltip": { "shared": true, "sort": 0, @@ -12775,17 +13214,17 @@ }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "percentunit", + "format": "short", "label": null, - "logBase": 10, + "logBase": 1, "max": null, "min": null, "show": true @@ -12802,7 +13241,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "CPU usage of LevelDB sorter", + "description": "Active stream count of each gRPC connection", "fieldConfig": { "defaults": { "links": [] @@ -12812,21 +13251,22 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 17 + "h": 7, + "w": 12, + "x": 0, + "y": 26 }, "hiddenSeries": false, - "id": 286, + "id": 188, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, + "hideZero": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -12844,21 +13284,16 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*sorter-[0-9]+/", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_actor_worker_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", name=~\"sorter.*\"}[1m])) by (name, instance)", + "expr": "sum(ticdc_kvclient_grpc_stream_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (store)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{instance}}-{{name}}", + "legendFormat": "{{store}}", "refId": "A" } ], @@ -12866,7 +13301,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "CPU usage", + "title": "KV client gRPC stream count", "tooltip": { "shared": true, "sort": 0, @@ -12882,7 +13317,7 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -12890,10 +13325,10 @@ "show": true }, { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, - "max": "1.2", + "max": null, "min": null, "show": true } @@ -12904,88 +13339,113 @@ } }, { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The time of sorter write", + "decimals": null, + "description": "Percentiles of Eventfeed message size", "fieldConfig": { - "defaults": {}, + "defaults": { + "links": [] + }, "overrides": [] }, + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 23 + "h": 7, + "w": 12, + "x": 12, + "y": 26 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 277, + "hiddenSeries": false, + "id": 17, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": true, + "max": false, "min": false, "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, + "lines": true, + "linewidth": 1, "links": [], - "maxPerRow": 3, - "repeatDirection": "h", - "reverseYBuckets": false, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_write_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", + "expr": "histogram_quantile(0.999, sum(rate(ticdc_kvclient_event_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}-p999", "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_kvclient_event_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}-p95", + "refId": "B" } ], - "title": "Write bytes", + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Event size percentile", "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true + "shared": true, + "sort": 0, + "value_type": "individual" }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "bytes", - "logBase": 1, - "max": null, - "min": null, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, "show": true, - "splitFactor": null + "values": [] }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -12993,27 +13453,32 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "description": "The number of regions that are being scanned", "fieldConfig": { - "defaults": {}, + "defaults": { + "links": [] + }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 23 + "h": 7, + "w": 12, + "x": 0, + "y": 33 }, "hiddenSeries": false, - "id": 279, + "id": 177, "legend": { "alignAsTable": true, - "avg": true, + "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -13025,6 +13490,7 @@ "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, @@ -13037,12 +13503,11 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_write_bytes_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "expr": "sum(ticdc_kvclient_region_token{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance,namespace, changefeed, store)", "format": "time_series", - "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}-sorter", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{store}}", "refId": "A" } ], @@ -13050,7 +13515,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Write OPS", + "title": "KV client scanning regions", "tooltip": { "shared": true, "sort": 0, @@ -13066,17 +13531,17 @@ }, "yaxes": [ { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "percentunit", + "format": "short", "label": null, - "logBase": 10, + "logBase": 1, "max": null, "min": null, "show": true @@ -13093,31 +13558,35 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The count and duration of write delay", + "decimals": 1, + "description": "The count of regoin cache operation initiated by TiCDC", "fieldConfig": { - "defaults": { - "links": [] - }, + "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 23 + "h": 7, + "w": 12, + "x": 12, + "y": 33 }, "hiddenSeries": false, - "id": 275, + "id": 459, "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -13128,47 +13597,31 @@ "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*count.*/", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(rate(ticdc_db_write_delay_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}-duration", - "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_db_write_delay_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "expr": "sum(rate(ticdc_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)", "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}-count", - "refId": "B" + "intervalFactor": 2, + "legendFormat": "{{type}}", + "metric": "tikv_grpc_msg_duration_seconds_bucket", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Write delay", + "title": "RegionCache operation/s", "tooltip": { "shared": true, "sort": 0, @@ -13184,7 +13637,7 @@ }, "yaxes": [ { - "format": "dtdurations", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -13220,21 +13673,21 @@ }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "The time of sorter iterator read", + "description": "The size of batch resolved regions count", "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { - "h": 6, - "w": 8, + "h": 7, + "w": 12, "x": 0, - "y": 29 + "y": 40 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 278, + "id": 97, "legend": { "alignAsTable": true, "avg": false, @@ -13249,13 +13702,11 @@ "values": true }, "links": [], - "maxPerRow": 3, - "repeatDirection": "h", "reverseYBuckets": false, "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[1m])) by (le)", + "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -13264,7 +13715,7 @@ "refId": "A" } ], - "title": "Read duration - First", + "title": "KV client batch resolved region count", "tooltip": { "show": true, "showHistogram": true @@ -13278,7 +13729,7 @@ "xBucketSize": null, "yAxis": { "decimals": 1, - "format": "s", + "format": "none", "logBase": 1, "max": null, "min": null, @@ -13290,200 +13741,37 @@ "yBucketSize": null }, { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "The time of sorter iterator read", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 29 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 282, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "maxPerRow": 3, - "repeatDirection": "h", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Read duration - Next", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "The time of sorter iterator read", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 29 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 281, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "maxPerRow": 3, - "repeatDirection": "h", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"release\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Read duration - Release", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": {}, - "overrides": [] + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The number of regions that have not connected to TiKV", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 35 + "h": 7, + "w": 12, + "x": 12, + "y": 40 }, "hiddenSeries": false, - "id": 280, + "id": 251, "legend": { "alignAsTable": true, - "avg": true, + "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -13495,6 +13783,7 @@ "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, @@ -13507,12 +13796,11 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[1m])) by (instance)", + "expr": "sum(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance, namespace,changefeed, store)", "format": "time_series", - "hide": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}-sorter", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{store}}", "refId": "A" } ], @@ -13520,7 +13808,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Read OPS - First", + "title": "KV client cached regions", "tooltip": { "shared": true, "sort": 0, @@ -13536,17 +13824,17 @@ }, "yaxes": [ { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "percentunit", + "format": "short", "label": null, - "logBase": 10, + "logBase": 1, "max": null, "min": null, "show": true @@ -13563,6 +13851,8 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "description": "The count of resolve lock RPC initiated by TiCDC", "fieldConfig": { "defaults": {}, "overrides": [] @@ -13570,21 +13860,26 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 35 + "h": 7, + "w": 12, + "x": 0, + "y": 47 }, "hiddenSeries": false, - "id": 283, + "id": 453, "legend": { "alignAsTable": true, - "avg": true, + "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -13597,7 +13892,7 @@ }, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -13606,21 +13901,20 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[1m])) by (instance)", + "expr": "sum(delta(ticdc_tikvclient_lock_resolver_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)", "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}-sorter", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "{{type}}", + "metric": "tikv_grpc_msg_duration_seconds_bucket", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Read OPS - Next", + "title": "ResolveLock count/m", "tooltip": { "shared": true, "sort": 0, @@ -13636,17 +13930,17 @@ }, "yaxes": [ { - "format": "ops", + "format": "opm", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "percentunit", + "format": "short", "label": null, - "logBase": 10, + "logBase": 1, "max": null, "min": null, "show": true @@ -13663,6 +13957,8 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "description": "", "fieldConfig": { "defaults": {}, "overrides": [] @@ -13670,21 +13966,26 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 35 + "h": 7, + "w": 12, + "x": 12, + "y": 47 }, "hiddenSeries": false, - "id": 287, + "id": 460, "legend": { "alignAsTable": true, - "avg": true, + "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -13697,7 +13998,7 @@ }, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -13707,20 +14008,21 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"release\"}[1m])) by (instance)", + "expr": "histogram_quantile(0.99999, sum(rate(ticdc_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, type))", "format": "time_series", - "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}-sorter", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "{{type}}", + "metric": "tikv_grpc_msg_duration_seconds_bucket", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Read OPS - Release", + "title": "Backoff percentiles", "tooltip": { "shared": true, "sort": 0, @@ -13736,20 +14038,20 @@ }, "yaxes": [ { - "format": "ops", + "format": "ms", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "percentunit", + "format": "short", "label": null, - "logBase": 10, + "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -13757,112 +14059,13 @@ "alignLevel": null } }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "The time of sorter compact", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 41 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 285, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "maxPerRow": 3, - "repeatDirection": "h", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_sorter_db_compact_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Compact duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - } - ], - "title": "DB", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 447, - "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of established Eventfeed RPC between TiCDC and TiKV", + "description": "The number of events that puller outputs to sorter \n per second", "fieldConfig": { "defaults": { "links": [] @@ -13875,18 +14078,17 @@ "h": 7, "w": 12, "x": 0, - "y": 12 + "y": 54 }, "hiddenSeries": false, - "id": 15, + "id": 5, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": false, - "max": false, + "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true @@ -13904,52 +14106,26 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*-rpc/", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_kvclient_event_feed_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance)", + "exemplar": true, + "expr": "sum (rate(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", changefeed!~\".*ddl_puller\"}[1m])) by (instance, type)", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}", + "legendFormat": "{{instance}}-{{type}}", "refId": "A" - }, - { - "expr": "sum(grpc_client_started_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance) - sum(grpc_client_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-rpc", - "refId": "B" - }, - { - "expr": "sum(grpc_client_started_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}-rpc-started", - "refId": "C" - }, - { - "expr": "sum(grpc_client_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}-rpc-handled", - "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Eventfeed count", + "title": "Puller output events/s", "tooltip": { "shared": true, "sort": 0, @@ -13965,7 +14141,7 @@ }, "yaxes": [ { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -13992,10 +14168,10 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of KV client received events from TiKV per seconds", + "description": "Estimate the remaining time for a changefeed initialization (on a specific capture)", "fieldConfig": { "defaults": { - "links": [] + "unit": "s" }, "overrides": [] }, @@ -14005,10 +14181,10 @@ "h": 7, "w": 12, "x": 12, - "y": 12 + "y": 54 }, "hiddenSeries": false, - "id": 29, + "id": 252, "legend": { "alignAsTable": true, "avg": false, @@ -14017,7 +14193,7 @@ "hideZero": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "total": false, "values": true @@ -14041,10 +14217,12 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_kvclient_pull_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", + "exemplar": true, + "expr": "abs(sum(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"} / deriv(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (instance, namespace,changefeed, store))", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{store}}", "refId": "A" } ], @@ -14052,7 +14230,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV client receive events/s", + "title": "Estimate remaining time for initialization", "tooltip": { "shared": true, "sort": 0, @@ -14068,9 +14246,9 @@ }, "yaxes": [ { - "format": "short", - "label": null, - "logBase": 1, + "format": "s", + "label": "", + "logBase": 2, "max": null, "min": null, "show": true @@ -14090,46 +14268,207 @@ } }, { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of errors that interrupt Eventfeed RPC", + "description": "Duration of sorting DDL events", "fieldConfig": { - "defaults": { - "links": [] - }, + "defaults": {}, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 19 + "y": 61 }, - "hiddenSeries": false, - "id": 28, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 99, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", - "options": { + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "DDL sorter sort duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "Duration of merging sorted events", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 61 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 105, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "expr": "sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "DDL sorter merge duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Percentiles of sorting events duration", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 68 + }, + "hiddenSeries": false, + "id": 53, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { "alertThreshold": true }, "paceLength": 10, @@ -14144,37 +14483,30 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(ticdc_kvclient_event_feed_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)", + "exemplar": true, + "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "format": "time_series", - "hide": false, - "interval": "1m", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{type}}", + "legendFormat": "{{instance}}-p999", "refId": "A" }, { - "expr": "-sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*leader\"}[1m]))", + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "format": "time_series", "hide": false, - "interval": "1m", + "interval": "", "intervalFactor": 1, - "legendFormat": "transfer-leader", + "legendFormat": "{{instance}}-p95", "refId": "B" - }, - { - "expr": "-sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*(peer|region)\"}[1m]))", - "format": "time_series", - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "move-region", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Eventfeed error/m", + "title": "DDL sorter sort duration percentile", "tooltip": { "shared": true, "sort": 0, @@ -14190,9 +14522,9 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -14217,7 +14549,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of KV client dispatched event per second", + "description": "Percentiles of merging sorted events duration", "fieldConfig": { "defaults": { "links": [] @@ -14230,16 +14562,15 @@ "h": 7, "w": 12, "x": 12, - "y": 19 + "y": 68 }, "hiddenSeries": false, - "id": 31, + "id": 106, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": true, + "hideEmpty": false, "max": true, "min": false, "rightSide": true, @@ -14260,28 +14591,28 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*batch-resolved/", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_kvclient_send_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[1m])) by (instance, namespace,changefeed, type)", + "exemplar": true, + "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}-{{namespace}}-{{changefeed}}-{{type}}", + "legendFormat": "{{instance}}-p999", "refId": "A" }, { - "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance,namespace, changefeed, table)", + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "format": "time_series", + "hide": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}-{{namespace}}-{{changefeed}}-batch-resolved", + "legendFormat": "{{instance}}-p95", "refId": "B" } ], @@ -14289,7 +14620,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV client dispatch events/s", + "title": "DDL sorter merge duration percentile", "tooltip": { "shared": true, "sort": 0, @@ -14305,9 +14636,9 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -14332,11 +14663,9 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Active stream count of each gRPC connection", + "description": "batch read events from the sorter, record each batch size ", "fieldConfig": { - "defaults": { - "links": [] - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -14345,31 +14674,27 @@ "h": 7, "w": 12, "x": 0, - "y": 26 + "y": 75 }, "hiddenSeries": false, - "id": 188, + "id": 705, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": true, "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, @@ -14381,10 +14706,11 @@ "steppedLine": false, "targets": [ { - "expr": "sum(ticdc_kvclient_grpc_stream_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (store)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{store}}", + "exemplar": true, + "expr": "histogram_quantile(0.999, sum(rate(ticdc_processor_sorter_batch_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,instance,changefeed))", + "hide": false, + "interval": "", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-p999", "refId": "A" } ], @@ -14392,7 +14718,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV client gRPC stream count", + "title": "batch read sorter size percentile", "tooltip": { "shared": true, "sort": 0, @@ -14408,7 +14734,7 @@ }, "yaxes": [ { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -14416,12 +14742,12 @@ "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -14435,12 +14761,9 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": null, - "description": "Percentiles of Eventfeed message size", + "description": "batch read events from the sorter, record each batch read duration ", "fieldConfig": { - "defaults": { - "links": [] - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -14449,29 +14772,27 @@ "h": 7, "w": 12, "x": 12, - "y": 26 + "y": 75 }, "hiddenSeries": false, - "id": 17, + "id": 706, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": false, + "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, @@ -14483,25 +14804,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_kvclient_event_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}-p999", + "exemplar": true, + "expr": "histogram_quantile(0.999, sum(rate(ticdc_processor_sorter_batch_read_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance,namespace,changefeed))", + "hide": false, + "interval": "", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-p999", "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, sum(rate(ticdc_kvclient_event_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}-p95", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Event size percentile", + "title": "batch read sorter duration percentile", "tooltip": { "shared": true, "sort": 0, @@ -14517,7 +14832,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -14525,12 +14840,12 @@ "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -14544,11 +14859,9 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of regions that are being scanned", + "description": "", "fieldConfig": { - "defaults": { - "links": [] - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -14557,31 +14870,27 @@ "h": 7, "w": 12, "x": 0, - "y": 33 + "y": 82 }, "hiddenSeries": false, - "id": 177, + "id": 707, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": true, "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, @@ -14594,11 +14903,10 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_kvclient_region_token{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance,namespace, changefeed, store)", - "format": "time_series", + "expr": "ticdc_mounter_group_input_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}", + "hide": false, "interval": "", - "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{store}}", + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{index}}", "refId": "A" } ], @@ -14606,7 +14914,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV client scanning regions", + "title": "mounter group channel size", "tooltip": { "shared": true, "sort": 0, @@ -14622,7 +14930,7 @@ }, "yaxes": [ { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -14630,27 +14938,40 @@ "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { "align": false, "alignLevel": null } - }, + } + ], + "title": "KVClient", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 266, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The count of regoin cache operation initiated by TiCDC", "fieldConfig": { "defaults": {}, "overrides": [] @@ -14660,11 +14981,11 @@ "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 33 + "x": 0, + "y": 13 }, "hiddenSeries": false, - "id": 459, + "id": 289, "legend": { "alignAsTable": true, "avg": false, @@ -14675,9 +14996,6 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, @@ -14690,7 +15008,7 @@ }, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -14699,20 +15017,19 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(ticdc_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)", + "expr": "etcd_debugging_mvcc_db_total_size_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"pd\"}", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_grpc_msg_duration_seconds_bucket", - "refId": "A", - "step": 10 + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "RegionCache operation/s", + "title": "Etcd MVCC DB total size", "tooltip": { "shared": true, "sort": 0, @@ -14728,7 +15045,7 @@ }, "yaxes": [ { - "format": "ops", + "format": "decbytes", "label": null, "logBase": 1, "max": null, @@ -14741,7 +15058,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -14749,95 +15066,12 @@ "alignLevel": null } }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "The size of batch resolved regions count", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 40 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 97, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "KV client batch resolved region count", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "none", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of regions that have not connected to TiKV", "fieldConfig": { "defaults": { "links": [] @@ -14850,16 +15084,15 @@ "h": 7, "w": 12, "x": 12, - "y": 40 + "y": 13 }, "hiddenSeries": false, - "id": 251, + "id": 114, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, - "hideZero": true, "max": true, "min": false, "rightSide": true, @@ -14886,20 +15119,32 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance, namespace,changefeed, store)", + "expr": "histogram_quantile(0.999, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance))", "format": "time_series", - "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{store}}", + "legendFormat": "p999-{{instance}}", "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p99-{{instance}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p95-{{instance}}", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "KV client cached regions", + "title": "Etcd health check duration", "tooltip": { "shared": true, "sort": 0, @@ -14915,7 +15160,7 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -14937,53 +15182,120 @@ } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": 1, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The count of resolve lock RPC initiated by TiCDC", + "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 47 + "y": 20 }, - "hiddenSeries": false, - "id": 453, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 267, + "legend": { + "show": true + }, + "pluginVersion": "6.1.6", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "EtcdWorker tick reactor duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 20 + }, + "hiddenSeries": false, + "id": 264, "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -14992,20 +15304,27 @@ "steppedLine": false, "targets": [ { - "expr": "sum(delta(ticdc_tikvclient_lock_resolver_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_grpc_msg_duration_seconds_bucket", - "refId": "A", - "step": 10 + "exemplar": true, + "expr": "histogram_quantile(0.999, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}-999", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}-99", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "ResolveLock count/m", + "title": "EtcdWorker tick reactor duration", "tooltip": { "shared": true, "sort": 0, @@ -15021,7 +15340,7 @@ }, "yaxes": [ { - "format": "opm", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -15029,7 +15348,7 @@ "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -15042,16 +15361,93 @@ "alignLevel": null } }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "max": null, + "min": 1, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 27 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 256, + "legend": { + "show": true + }, + "pluginVersion": "6.1.6", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "EtcdWorker exec etcd txn duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, "description": "", "fieldConfig": { - "defaults": {}, + "defaults": { + "unit": "s" + }, "overrides": [] }, "fill": 1, @@ -15060,36 +15456,28 @@ "h": 7, "w": 12, "x": 12, - "y": 47 + "y": 27 }, "hiddenSeries": false, - "id": 460, + "id": 258, "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -15099,21 +15487,30 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(0.99999, sum(rate(ticdc_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, type))", + "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "format": "time_series", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_grpc_msg_duration_seconds_bucket", - "refId": "A", - "step": 10 + "intervalFactor": 1, + "legendFormat": "{{instance}}-p95", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p99", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Backoff percentiles", + "title": "EtcdWorker exec etcd txn duration percentile", "tooltip": { "shared": true, "sort": 0, @@ -15129,9 +15526,9 @@ }, "yaxes": [ { - "format": "ms", + "format": "s", "label": null, - "logBase": 1, + "logBase": 2, "max": null, "min": null, "show": true @@ -15142,7 +15539,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -15151,47 +15548,114 @@ } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "The number of events that puller outputs to sorter \n per second", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolatePurples", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 34 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 254, + "legend": { + "show": true + }, + "pluginVersion": "6.1.6", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "EtcdWorker txn size", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": null, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "decbytes", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 54 + "x": 12, + "y": 34 }, "hiddenSeries": false, - "id": 5, + "id": 260, "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": false, "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, @@ -15204,19 +15668,26 @@ "targets": [ { "exemplar": true, - "expr": "sum (rate(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", changefeed!~\".*ddl_puller\"}[1m])) by (instance, type)", - "format": "time_series", + "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}", + "legendFormat": "{{instance}}-p95", + "queryType": "randomWalk", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}-p99", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Puller output events/s", + "title": "EtcdWorker txn size percentile", "tooltip": { "shared": true, "sort": 0, @@ -15232,7 +15703,7 @@ }, "yaxes": [ { - "format": "none", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -15259,23 +15730,24 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Estimate the remaining time for a changefeed initialization (on a specific capture)", + "description": "The time consumed of writing WAL into the persistent storage in .99", + "editable": true, + "error": false, "fieldConfig": { - "defaults": { - "unit": "s" - }, + "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, + "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 54 + "x": 0, + "y": 41 }, "hiddenSeries": false, - "id": 252, + "id": 291, "legend": { "alignAsTable": true, "avg": false, @@ -15286,6 +15758,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -15299,7 +15772,7 @@ "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -15308,24 +15781,25 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "abs(sum(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"} / deriv(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (instance, namespace,changefeed, store))", + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, le))", "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{store}}", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "", + "refId": "A", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Estimate remaining time for initialization", + "title": "Etcd 99% WAL fsync duration", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -15338,8 +15812,8 @@ "yaxes": [ { "format": "s", - "label": "", - "logBase": 2, + "label": null, + "logBase": 1, "max": null, "min": null, "show": true @@ -15359,35 +15833,29 @@ } }, { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Duration of sorting DDL events", + "description": "The time consumed of handling etcd transactions in .99", + "editable": true, + "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, + "fill": 1, + "fillGradient": 0, + "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 61 + "x": 12, + "y": 41 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 99, + "hiddenSeries": false, + "id": 290, "legend": { "alignAsTable": true, "avg": false, @@ -15396,141 +15864,104 @@ "min": false, "rightSide": true, "show": true, - "sort": "current", - "sortDesc": true, + "sideWidth": 300, "total": false, "values": true }, + "lines": true, + "linewidth": 1, "links": [], - "reverseYBuckets": false, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", + "expr": "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, result, le))", + "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" + "legendFormat": "{{instance}} {{result}}", + "refId": "A", + "step": 4 } ], - "title": "DDL sorter sort duration", + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Etcd 99% Handle transactions duration", "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, "show": true, - "splitFactor": null + "values": [] }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "Duration of merging sorted events", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 61 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 105, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "reverseYBuckets": false, - "targets": [ + "yaxes": [ { - "expr": "sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } ], - "title": "DDL sorter merge duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Etcd", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 58, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Percentiles of sorting events duration", + "decimals": 1, + "description": "CPU usage of TiKV CDC component", "fieldConfig": { - "defaults": { - "links": [] - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -15539,19 +15970,21 @@ "h": 7, "w": 12, "x": 0, - "y": 68 + "y": 166 }, "hiddenSeries": false, - "id": 53, + "id": 60, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": false, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -15565,39 +15998,57 @@ "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/.*workers/", + "transform": "negative-Y", + "yaxis": 2 + }, + { + "alias": "/.*tso/", + "fill": 0, + "pointradius": 1, + "points": true + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdc_.*|cdc\"}[1m])) by (instance)", "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}-p999", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "{{instance}}-endpoint", + "refId": "A", + "step": 4 }, { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdcwkr.*\"}[1m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}-workers", + "refId": "B", + "step": 4 + }, + { + "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"tso\"}[1m])) by (instance)", "format": "time_series", "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}-p95", - "refId": "B" + "intervalFactor": 2, + "legendFormat": "{{instance}}-tso", + "refId": "C", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "DDL sorter sort duration percentile", + "title": "CDC CPU", "tooltip": { "shared": true, "sort": 0, @@ -15613,15 +16064,15 @@ }, "yaxes": [ { - "format": "s", + "format": "percentunit", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true }, { - "format": "short", + "format": "percent", "label": null, "logBase": 1, "max": null, @@ -15636,40 +16087,46 @@ }, { "aliasColors": {}, - "bars": false, + "bars": true, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Percentiles of merging sorted events duration", + "decimals": 1, + "description": "Outbound network traffic of TiKV CDC component", + "editable": true, + "error": false, "fieldConfig": { - "defaults": { - "links": [] - }, + "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 68 + "y": 166 }, "hiddenSeries": false, - "id": 106, + "id": 74, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": false, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, "total": false, "values": true }, - "lines": true, + "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null", @@ -15679,40 +16136,41 @@ "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/.*resolved_ts/", + "bars": false, + "fill": 1, + "lines": true, + "linewidth": 2, + "transform": "negative-Y", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}-p999", - "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "expr": "sum(rate(tikv_cdc_grpc_message_sent_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[30s])) by (instance, type)", "format": "time_series", "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}-p95", - "refId": "B" + "intervalFactor": 2, + "legendFormat": "{{instance}}-{{type}}", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "DDL sorter merge duration percentile", + "title": "CDC network traffic", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -15727,15 +16185,15 @@ }, "yaxes": [ { - "format": "s", + "format": "Bps", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true }, { - "format": "short", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -15745,7 +16203,7 @@ ], "yaxis": { "align": false, - "alignLevel": null + "alignLevel": 0 } }, { @@ -15754,7 +16212,8 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "batch read events from the sorter, record each batch size ", + "decimals": 1, + "description": "The count of different kinds of gRPC message", "fieldConfig": { "defaults": {}, "overrides": [] @@ -15765,30 +16224,36 @@ "h": 7, "w": 12, "x": 0, - "y": 75 + "y": 173 }, "hiddenSeries": false, - "id": 705, + "id": 147, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -15797,19 +16262,20 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(ticdc_processor_sorter_batch_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,instance,changefeed))", - "hide": false, - "interval": "", - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-p999", - "refId": "A" + "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type!=\"kv_gc\"}[1m])) by (type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "metric": "tikv_grpc_msg_duration_seconds_bucket", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "batch read sorter size percentile", + "title": "gRPC message count", "tooltip": { "shared": true, "sort": 0, @@ -15825,7 +16291,7 @@ }, "yaxes": [ { - "format": "none", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -15833,12 +16299,12 @@ "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -15852,63 +16318,139 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "batch read events from the sorter, record each batch read duration ", + "decimals": 1, + "description": "The memory usage per TiKV instance", + "editable": true, + "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 75 + "y": 173 }, "hiddenSeries": false, - "id": 706, + "id": 194, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, - "rightSide": false, + "rightSide": true, "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/.*-cap-.*/", + "yaxis": 2 + }, + { + "alias": "/.*tikv.*/", + "pointradius": 1, + "points": true, + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(ticdc_processor_sorter_batch_read_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance,namespace,changefeed))", + "expr": "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"tikv.*\"}) by (instance)", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "tikv-{{instance}}", + "refId": "A", + "step": 10 + }, + { + "expr": "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"cdc.*\"}) by (instance)", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "cdc-{{instance}}", + "refId": "B", + "step": 10 + }, + { + "expr": "(avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"tikv.*\"}) by (instance)) - (avg(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", db=\"kv\"}) by(instance))", + "format": "time_series", "hide": false, - "interval": "", - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-p999", - "refId": "A" + "intervalFactor": 2, + "legendFormat": "tikv-{{instance}}", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(tikv_cdc_sink_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "sink-{{instance}}", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "old-value-{{instance}}", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(tikv_cdc_sink_memory_capacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "sink-cap-{{instance}}", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(tikv_cdc_old_value_cache_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "old-value-cap-{{instance}}", + "refId": "G", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "batch read sorter duration percentile", + "title": "CDC memory", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -15923,20 +16465,20 @@ }, "yaxes": [ { - "format": "s", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "none", + "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -15950,63 +16492,106 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", + "decimals": 1, + "description": "The min resolved ts of each TiKV", + "editable": true, + "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { "h": 7, - "w": 12, + "w": 7, "x": 0, - "y": 82 + "y": 180 }, "hiddenSeries": false, - "id": 707, + "id": 152, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": true, + "max": false, "min": false, "rightSide": false, "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 1, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/.*-ts/", + "lines": true, + "linewidth": 3, + "points": false, + "yaxis": 2 + }, + { + "alias": "/.*-lag/", + "bars": true, + "fill": 1 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "ticdc_mounter_group_input_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}", + "expr": "scalar(max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}))/1000 - avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0", + "format": "time_series", "hide": false, - "interval": "", - "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{index}}", - "refId": "A" + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}-min-resolved-lag", + "refId": "A", + "step": 10 + }, + { + "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "current-ts", + "refId": "B", + "step": 10 + }, + { + "expr": "avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}-min-resolved-ts", + "refId": "C", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "mounter group channel size", + "title": "Min resolved ts", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -16021,11 +16606,11 @@ }, "yaxes": [ { - "format": "none", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -16034,73 +16619,65 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { "align": false, "alignLevel": null } - } - ], - "title": "KVClient", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 266, - "panels": [ + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "description": "The ID of the min resolved region of each TiKV", + "editable": true, + "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { "h": 7, - "w": 12, - "x": 0, - "y": 13 + "w": 5, + "x": 7, + "y": 180 }, "hiddenSeries": false, - "id": 289, + "id": 153, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": true, "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": null, "total": false, "values": true }, - "lines": true, + "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, - "points": false, + "pointradius": 1, + "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -16108,20 +16685,24 @@ "steppedLine": false, "targets": [ { - "expr": "etcd_debugging_mvcc_db_total_size_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"pd\"}", + "exemplar": true, + "expr": "avg(tikv_cdc_min_resolved_ts_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "format": "time_series", "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}-min-resolved-region", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Etcd MVCC DB total size", + "title": "Min resolved Region", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -16136,15 +16717,15 @@ }, "yaxes": [ { - "format": "decbytes", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -16163,10 +16744,10 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "description": "", "fieldConfig": { - "defaults": { - "links": [] - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -16175,19 +16756,23 @@ "h": 7, "w": 12, "x": 12, - "y": 13 + "y": 180 }, "hiddenSeries": false, - "id": 114, + "id": 70, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, + "hideEmpty": false, + "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": null, "total": false, "values": true }, @@ -16201,7 +16786,7 @@ "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -16210,32 +16795,18 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance))", + "expr": "histogram_quantile(0.99999, sum(rate(tikv_cdc_resolved_ts_gap_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "p999-{{instance}}", + "legendFormat": "{{instance}}-p9999", "refId": "A" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "p99-{{instance}}", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.95, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "p95-{{instance}}", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Etcd health check duration", + "title": "Resolved ts lag duration percentile", "tooltip": { "shared": true, "sort": 0, @@ -16253,7 +16824,7 @@ { "format": "s", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": null, "show": true @@ -16278,17 +16849,16 @@ "cardRound": 0 }, "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", + "cardColor": "#FF9830", + "colorScale": "linear", "colorScheme": "interpolateSpectral", "exponent": 0.5, - "max": null, - "min": 1, + "min": 0, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "description": "", + "description": "The time consumed to CDC incremental scan", "fieldConfig": { "defaults": {}, "overrides": [] @@ -16297,33 +16867,41 @@ "h": 7, "w": 12, "x": 0, - "y": 20 + "y": 187 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 267, + "id": 68, "legend": { - "show": true + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true }, - "pluginVersion": "6.1.6", + "links": [], + "maxPerRow": 3, + "repeat": null, + "repeatDirection": "h", "reverseYBuckets": false, "targets": [ { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", + "expr": "sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", "format": "heatmap", "instant": false, - "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", - "queryType": "randomWalk", "refId": "A" } ], - "timeFrom": null, - "timeShift": null, - "title": "EtcdWorker tick reactor duration", + "title": "Initial scan duration", "tooltip": { "show": true, "showHistogram": true @@ -16354,6 +16932,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "description": "", "fieldConfig": { "defaults": {}, @@ -16363,30 +16942,39 @@ "fillGradient": 0, "gridPos": { "h": 7, - "w": 12, + "w": 6, "x": 12, - "y": 20 + "y": 187 }, "hiddenSeries": false, - "id": 264, + "id": 72, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "hideEmpty": false, + "hideZero": true, + "max": true, "min": false, + "rightSide": false, "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -16395,27 +16983,18 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", - "hide": false, - "interval": "", - "legendFormat": "{{instance}}-999", + "expr": "histogram_quantile(0.9999, sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}-p9999", "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", - "hide": false, - "interval": "", - "legendFormat": "{{instance}}-99", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "EtcdWorker tick reactor duration", + "title": "Initial scan duration percentile", "tooltip": { "shared": true, "sort": 0, @@ -16439,12 +17018,12 @@ "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -16452,148 +17031,81 @@ "alignLevel": null } }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "max": null, - "min": 1, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 27 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 256, - "legend": { - "show": true - }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "EtcdWorker exec etcd txn duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", + "decimals": 1, + "description": "The number of incremental scan task in different status.", "fieldConfig": { - "defaults": { - "unit": "s" - }, + "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 27 + "w": 6, + "x": 18, + "y": 187 }, "hiddenSeries": false, - "id": 258, + "id": 140, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "hideEmpty": false, + "hideZero": true, + "max": true, "min": false, + "rightSide": false, "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/.*ongoing/", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "expr": "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"ongoing\"}) by (type, instance)", "format": "time_series", - "interval": "", + "hide": false, "intervalFactor": 1, - "legendFormat": "{{instance}}-p95", - "queryType": "randomWalk", + "legendFormat": "{{instance}}-{{type}}", "refId": "A" }, { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "expr": "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"total\"}) by (instance) - sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=~\"abort|finish\"}) by (instance)", "format": "time_series", "hide": false, - "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}-p99", + "legendFormat": "{{instance}}-pending", "refId": "B" } ], @@ -16601,7 +17113,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "EtcdWorker exec etcd txn duration percentile", + "title": "Initial scan tasks status", "tooltip": { "shared": true, "sort": 0, @@ -16617,15 +17129,15 @@ }, "yaxes": [ { - "format": "s", + "format": "none", "label": null, - "logBase": 2, + "logBase": 1, "max": null, "min": null, "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -16638,118 +17150,56 @@ "alignLevel": null } }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolatePurples", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 34 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 254, - "legend": { - "show": true - }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "EtcdWorker txn size", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": null, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "decbytes", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "", + "decimals": 1, + "description": "The memory usage per TiKV instance", + "editable": true, + "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 34 + "x": 0, + "y": 194 }, "hiddenSeries": false, - "id": 260, + "id": 78, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, + "rightSide": true, "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": null, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -16758,28 +17208,31 @@ "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", - "interval": "", - "legendFormat": "{{instance}}-p95", - "queryType": "randomWalk", - "refId": "A" + "expr": "avg(tikv_cdc_captured_region_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "tikv-{{instance}}-total", + "refId": "A", + "step": 10 }, { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "expr": "sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance, status)", + "format": "time_series", "hide": false, - "interval": "", - "legendFormat": "{{instance}}-p99", - "refId": "B" + "intervalFactor": 2, + "legendFormat": "tikv-{{instance}}-{{status}}", + "refId": "B", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "EtcdWorker txn size percentile", + "title": "Captured region count", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -16794,11 +17247,11 @@ }, "yaxes": [ { - "format": "bytes", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -16821,24 +17274,25 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The time consumed of writing WAL into the persistent storage in .99", + "decimals": 1, + "description": "The speed of TiKV CDC incremental scan", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, - "w": 12, - "x": 0, - "y": 41 + "w": 6, + "x": 12, + "y": 194 }, "hiddenSeries": false, - "id": 291, + "id": 76, "legend": { "alignAsTable": true, "avg": false, @@ -16847,9 +17301,11 @@ "hideZero": true, "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, - "sideWidth": 300, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -16872,25 +17328,25 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, le))", + "expr": "sum(rate(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}[30s])) by (instance)", "format": "time_series", + "hide": false, "intervalFactor": 2, - "legendFormat": "{{instance}}", - "metric": "", + "legendFormat": "tikv-{{instance}}", "refId": "A", - "step": 4 + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Etcd 99% WAL fsync duration", + "title": "CDC scan speed", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -16902,11 +17358,11 @@ }, "yaxes": [ { - "format": "s", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -16925,41 +17381,46 @@ }, { "aliasColors": {}, - "bars": false, + "bars": true, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The time consumed of handling etcd transactions in .99", + "decimals": 1, + "description": "The total bytes of TiKV CDC incremental scan", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 41 + "w": 6, + "x": 18, + "y": 194 }, "hiddenSeries": false, - "id": 290, + "id": 139, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, - "sideWidth": 300, + "sideWidth": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, - "lines": true, + "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null", @@ -16978,24 +17439,25 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, result, le))", + "expr": "sum(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}) by (instance)", "format": "time_series", + "hide": false, "intervalFactor": 2, - "legendFormat": "{{instance}} {{result}}", + "legendFormat": "tikv-{{instance}}", "refId": "A", - "step": 4 + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Etcd 99% Handle transactions duration", + "title": "CDC total scan bytes", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -17007,7 +17469,7 @@ }, "yaxes": [ { - "format": "s", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -17027,22 +17489,7 @@ "align": false, "alignLevel": null } - } - ], - "title": "Etcd", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 58, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -17050,27 +17497,29 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "CPU usage of TiKV CDC component", + "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 4, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 166 + "y": 201 }, "hiddenSeries": false, - "id": 60, + "id": 143, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": true, - "min": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": true, "rightSide": true, "show": true, "sideWidth": null, @@ -17089,20 +17538,15 @@ "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, + "pointradius": 1, + "points": true, "renderer": "flot", "seriesOverrides": [ { - "alias": "/.*workers/", - "transform": "negative-Y", - "yaxis": 2 - }, - { - "alias": "/.*tso/", + "alias": "/(access|miss).*/", "fill": 0, - "pointradius": 1, - "points": true + "points": false, + "yaxis": 2 } ], "spaceLength": 10, @@ -17110,36 +17554,43 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdc_.*|cdc\"}[1m])) by (instance)", + "expr": "(sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance) - sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)) / sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-endpoint", - "refId": "A", - "step": 4 + "hide": false, + "intervalFactor": 1, + "legendFormat": "hit-rate-{{instance}}", + "refId": "A" }, { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdcwkr.*\"}[1m])) by (instance)", + "expr": "-sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-workers", - "refId": "B", - "step": 4 + "hide": true, + "intervalFactor": 1, + "legendFormat": "access-{{instance}}", + "refId": "B" }, { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"tso\"}[1m])) by (instance)", + "expr": "-sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", "format": "time_series", "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}-tso", - "refId": "C", - "step": 4 + "intervalFactor": 1, + "legendFormat": "miss-{{instance}}", + "refId": "C" + }, + { + "expr": "-sum(rate(tikv_cdc_old_value_cache_miss_none{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "miss-none-{{instance}}", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "CDC CPU", + "title": "Old value cache hit", "tooltip": { "shared": true, "sort": 0, @@ -17158,12 +17609,12 @@ "format": "percentunit", "label": null, "logBase": 1, - "max": null, - "min": null, + "max": "1", + "min": "0", "show": true }, { - "format": "percent", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -17178,46 +17629,43 @@ }, { "aliasColors": {}, - "bars": true, + "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "Outbound network traffic of TiKV CDC component", - "editable": true, - "error": false, + "description": "The total number of cache entries in the old value cache.", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 166 + "y": 201 }, "hiddenSeries": false, - "id": 74, + "id": 145, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": false, - "hideZero": false, + "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, - "sort": "max", + "sort": "current", "sortDesc": true, "total": false, "values": true }, - "lines": false, + "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", @@ -17227,17 +17675,12 @@ "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 5, + "pointradius": 1, "points": false, "renderer": "flot", "seriesOverrides": [ { - "alias": "/.*resolved_ts/", - "bars": false, - "fill": 1, - "lines": true, - "linewidth": 2, - "transform": "negative-Y", + "alias": "/.*len/", "yaxis": 2 } ], @@ -17246,22 +17689,36 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_cdc_grpc_message_sent_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[30s])) by (instance, type)", + "expr": "sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", "format": "time_series", "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{type}}", - "refId": "A", - "step": 10 + "intervalFactor": 1, + "legendFormat": "{{instance}}-len", + "refId": "A" + }, + { + "expr": "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance) / sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}-avg entry bytes", + "refId": "B" + }, + { + "expr": "sum(tikv_cdc_old_value_cache_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}-quota", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "CDC network traffic", + "title": "Old value cache size", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -17276,25 +17733,26 @@ }, "yaxes": [ { - "format": "Bps", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "bytes", + "decimals": null, + "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ], "yaxis": { "align": false, - "alignLevel": 0 + "alignLevel": null } }, { @@ -17304,21 +17762,24 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The count of different kinds of gRPC message", + "description": "", + "editable": true, + "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 173 + "y": 208 }, "hiddenSeries": false, - "id": 147, + "id": 141, "legend": { "alignAsTable": true, "avg": false, @@ -17342,6 +17803,7 @@ "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, @@ -17353,11 +17815,11 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type!=\"kv_gc\"}[1m])) by (type)", + "expr": "sum(rate(tikv_cdc_old_value_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance, cf, tag)", "format": "time_series", + "hide": false, "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_grpc_msg_duration_seconds_bucket", + "legendFormat": "{{instance}}-{{cf}}-{{tag}}", "refId": "A", "step": 10 } @@ -17366,8 +17828,9 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "gRPC message count", + "title": "Old value seek operation", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -17386,7 +17849,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -17404,30 +17867,35 @@ } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#FF9830", + "colorScale": "linear", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The memory usage per TiKV instance", - "editable": true, - "error": false, + "description": "The time consumed to get an old value (both from cache and from disk)", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 0, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, - "w": 12, + "w": 6, "x": 12, - "y": 173 + "y": 208 }, - "hiddenSeries": false, - "id": 194, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 146, "legend": { "alignAsTable": true, "avg": false, @@ -17436,146 +17904,49 @@ "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "max", + "sort": "current", "sortDesc": true, "total": false, "values": true }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*-cap-.*/", - "yaxis": 2 - }, - { - "alias": "/.*tikv.*/", - "pointradius": 1, - "points": true, - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "maxPerRow": 3, + "repeatDirection": "h", + "reverseYBuckets": false, "targets": [ { - "expr": "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"tikv.*\"}) by (instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "tikv-{{instance}}", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"cdc.*\"}) by (instance)", - "format": "time_series", - "hide": true, + "expr": "sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", + "format": "heatmap", + "instant": false, "intervalFactor": 2, - "legendFormat": "cdc-{{instance}}", - "refId": "B", - "step": 10 - }, - { - "expr": "(avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"tikv.*\"}) by (instance)) - (avg(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", db=\"kv\"}) by(instance))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "tikv-{{instance}}", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(tikv_cdc_sink_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "sink-{{instance}}", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "old-value-{{instance}}", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(tikv_cdc_sink_memory_capacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "sink-cap-{{instance}}", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(tikv_cdc_old_value_cache_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "old-value-cap-{{instance}}", - "refId": "G", - "step": 10 + "legendFormat": "{{le}}", + "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CDC memory", + "title": "Old value seek duration", "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" + "show": true, + "showHistogram": true }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null }, { "aliasColors": {}, @@ -17584,35 +17955,34 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "description": "The min resolved ts of each TiKV", - "editable": true, - "error": false, + "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, - "w": 7, - "x": 0, - "y": 180 + "w": 6, + "x": 18, + "y": 208 }, "hiddenSeries": false, - "id": 152, + "id": 142, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": false, + "hideEmpty": false, + "hideZero": true, + "max": true, "min": false, "rightSide": false, "show": true, "sideWidth": null, - "sort": null, - "sortDesc": null, + "sort": "current", + "sortDesc": true, "total": false, "values": true }, @@ -17626,63 +17996,42 @@ "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 1, + "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*-ts/", - "lines": true, - "linewidth": 3, - "points": false, - "yaxis": 2 - }, - { - "alias": "/.*-lag/", - "bars": true, - "fill": 1 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "scalar(max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}))/1000 - avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0", + "expr": "histogram_quantile(0.99, sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag))", "format": "time_series", - "hide": false, - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}-min-resolved-lag", - "refId": "A", - "step": 10 + "intervalFactor": 1, + "legendFormat": "{{instance}}-99%-{{tag}}", + "refId": "A" }, { - "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", + "expr": "histogram_quantile(0.95, sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag))", "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "current-ts", - "refId": "B", - "step": 10 + "intervalFactor": 1, + "legendFormat": "{{instance}}-95%-{{tag}}", + "refId": "B" }, { - "expr": "avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", + "expr": "sum(rate(tikv_cdc_old_value_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag) / sum(rate(tikv_cdc_old_value_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag)", "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}-min-resolved-ts", - "refId": "C", - "step": 10 + "intervalFactor": 1, + "legendFormat": "{{instance}}-avg-{{tag}}", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Min resolved ts", + "title": "Old value seek duration", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -17701,74 +18050,77 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { "align": false, "alignLevel": null } - }, + } + ], + "title": "TiKV", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 294, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The ID of the min resolved region of each TiKV", - "editable": true, - "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, - "w": 5, - "x": 7, - "y": 180 + "w": 4, + "x": 0, + "y": 15 }, "hiddenSeries": false, - "id": 153, + "id": 321, "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": false, "show": true, - "sideWidth": null, - "sort": null, - "sortDesc": null, "total": false, - "values": true + "values": false }, - "lines": false, + "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 1, - "points": true, + "pointradius": 2, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, @@ -17777,23 +18129,19 @@ "targets": [ { "exemplar": true, - "expr": "avg(tikv_cdc_min_resolved_ts_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", - "format": "time_series", - "hide": false, + "expr": "sum without (from) (rate(ticdc_message_server_message_count{instance=~\"$ticdc_instance\"}[30s]))", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}-min-resolved-region", - "refId": "A", - "step": 10 + "legendFormat": "{{instance}}", + "queryType": "randomWalk", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Min resolved Region", + "title": "Message Receive Rate", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -17808,20 +18156,20 @@ }, "yaxes": [ { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -17835,8 +18183,6 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "", "fieldConfig": { "defaults": {}, "overrides": [] @@ -17845,39 +18191,30 @@ "fillGradient": 0, "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 180 + "w": 4, + "x": 4, + "y": 15 }, "hiddenSeries": false, - "id": 70, + "id": 323, "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": true, "show": true, - "sideWidth": null, - "sort": null, - "sortDesc": null, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -17886,10 +18223,11 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99999, sum(rate(tikv_cdc_resolved_ts_gap_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-p9999", + "exemplar": true, + "expr": "sum without (to) (rate(ticdc_message_client_message_count{instance=~\"$ticdc_instance\"}[30s]))", + "interval": "", + "legendFormat": "{{instance}}", + "queryType": "randomWalk", "refId": "A" } ], @@ -17897,7 +18235,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Resolved ts lag duration percentile", + "title": "Message Send Rate", "tooltip": { "shared": true, "sort": 0, @@ -17913,9 +18251,9 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, - "logBase": 10, + "logBase": 1, "max": null, "min": null, "show": true @@ -17934,138 +18272,46 @@ "alignLevel": null } }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "The time consumed to CDC incremental scan", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 187 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 68, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "maxPerRow": 3, - "repeat": null, - "repeatDirection": "h", - "reverseYBuckets": false, - "targets": [ - { - "expr": "sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Initial scan duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, - "w": 6, - "x": 12, - "y": 187 + "w": 4, + "x": 8, + "y": 15 }, "hiddenSeries": false, - "id": 72, + "id": 352, "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "hideEmpty": false, + "current": false, + "hideEmpty": true, "hideZero": true, - "max": true, + "max": false, "min": false, - "rightSide": false, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -18074,23 +18320,41 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}-p9999", + "exemplar": true, + "expr": "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))", + "interval": "", + "legendFormat": "p50", + "queryType": "randomWalk", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))", + "hide": false, + "interval": "", + "legendFormat": "p90", + "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))", + "hide": false, + "interval": "", + "legendFormat": "p99", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Initial scan duration percentile", + "title": "Message Batch Size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, + "transformations": [], "type": "graph", "xaxis": { "buckets": null, @@ -18101,7 +18365,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -18114,7 +18378,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -18128,75 +18392,69 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The number of incremental scan task in different status.", "fieldConfig": { - "defaults": {}, + "defaults": { + "unit": "short" + }, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, - "x": 18, - "y": 187 + "x": 12, + "y": 15 }, "hiddenSeries": false, - "id": 140, + "id": 354, "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": false, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*ongoing/", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"ongoing\"}) by (type, instance)", - "format": "time_series", + "exemplar": true, + "expr": "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))", "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}", + "interval": "", + "legendFormat": "p50", + "refId": "C" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))", + "interval": "", + "legendFormat": "p95", + "queryType": "randomWalk", "refId": "A" }, { - "expr": "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"total\"}) by (instance) - sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=~\"abort|finish\"}) by (instance)", - "format": "time_series", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))", "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}-pending", + "interval": "", + "legendFormat": "p99", "refId": "B" } ], @@ -18204,7 +18462,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Initial scan tasks status", + "title": "Receive message batch bytes percentiles", "tooltip": { "shared": true, "sort": 0, @@ -18220,7 +18478,7 @@ }, "yaxes": [ { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -18228,7 +18486,7 @@ "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -18247,50 +18505,38 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The memory usage per TiKV instance", - "editable": true, - "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, - "w": 12, - "x": 0, - "y": 194 + "w": 6, + "x": 18, + "y": 15 }, "hiddenSeries": false, - "id": 78, + "id": 356, "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": true, "show": true, - "sideWidth": null, - "sort": null, - "sortDesc": null, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -18299,31 +18545,36 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_cdc_captured_region_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", - "format": "time_series", + "exemplar": true, + "expr": "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))", + "interval": "", + "legendFormat": "p50", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))", "hide": false, - "intervalFactor": 2, - "legendFormat": "tikv-{{instance}}-total", - "refId": "A", - "step": 10 + "interval": "", + "legendFormat": "p95", + "refId": "B" }, { - "expr": "sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance, status)", - "format": "time_series", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))", "hide": false, - "intervalFactor": 2, - "legendFormat": "tikv-{{instance}}-{{status}}", - "refId": "B", - "step": 10 + "interval": "", + "legendFormat": "p99", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Captured region count", + "title": "Receive Message Bytes Percentile", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -18338,11 +18589,11 @@ }, "yaxes": [ { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -18359,58 +18610,150 @@ "alignLevel": null } }, + { + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "filterable": false + }, + "decimals": 2, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "unit", + "value": "time: YYYY-MM-DD HH:mm:ss" + }, + { + "id": "custom.align", + "value": null + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 20, + "x": 0, + "y": 22 + }, + "id": 350, + "options": { + "showHeader": true + }, + "pluginVersion": "7.5.11", + "targets": [ + { + "exemplar": true, + "expr": "ticdc_message_server_cur_stream_count", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Stream Count Between Nodes", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "from" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "instance": false, + "job": true + }, + "indexByName": {}, + "renameByName": { + "instance": "dest\\src" + } + } + }, + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The speed of TiKV CDC incremental scan", - "editable": true, - "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, - "grid": {}, "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 194 + "h": 8, + "w": 4, + "x": 20, + "y": 22 }, "hiddenSeries": false, - "id": 76, + "id": 358, "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": false, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -18419,22 +18762,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}[30s])) by (instance)", - "format": "time_series", + "exemplar": true, + "expr": "sum(ticdc_message_server_cur_stream_count)", + "interval": "", + "legendFormat": "actual", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "2 * count(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}) - 1", "hide": false, - "intervalFactor": 2, - "legendFormat": "tikv-{{instance}}", - "refId": "A", - "step": 10 + "interval": "", + "legendFormat": "expected", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "CDC scan speed", + "title": "Total Stream Count", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -18449,11 +18798,11 @@ }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -18469,117 +18818,317 @@ "align": false, "alignLevel": null } - }, + } + ], + "title": "Peer Messages", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 10021, + "panels": [ { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The total bytes of TiKV CDC incremental scan", - "editable": true, - "error": false, - "fieldConfig": { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The latency distributions of fsync called by redo writer", + "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 0, - "fillGradient": 0, - "grid": {}, "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 194 + "h": 8, + "w": 12, + "x": 0, + "y": 1 }, - "hiddenSeries": false, - "id": 139, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 10023, "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": false, + "show": true + }, + "pluginVersion": "6.1.6", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "max(rate(ticdc_redo_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", + "format": "heatmap", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Redo fsync duration", + "tooltip": { "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true + "showHistogram": true }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true }, - "paceLength": 10, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The latency distributions of flushall called by redo writer", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 10025, + "legend": { + "show": true + }, + "pluginVersion": "6.1.6", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "max(rate(ticdc_redo_flushall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", + "format": "heatmap", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Redo flushall duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The latency distributions of writeLog called by redoManager", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 10027, + "legend": { + "show": true + }, + "pluginVersion": "6.1.6", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "max(rate(ticdc_redo_write_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", + "format": "heatmap", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Redo write log duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": 0, + "cardRound": 0 + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_TEST-CLUSTER}", + "description": "The latency distributions of flushLog called by redoManager", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 10029, + "legend": { + "show": true + }, + "pluginVersion": "6.1.6", + "reverseYBuckets": false, "targets": [ { - "expr": "sum(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}) by (instance)", - "format": "time_series", - "hide": false, + "exemplar": true, + "expr": "max(rate(ticdc_redo_flush_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", + "format": "heatmap", + "interval": "", "intervalFactor": 2, - "legendFormat": "tikv-{{instance}}", - "refId": "A", - "step": 10 + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" } ], - "thresholds": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "CDC total scan bytes", + "title": "Redo flush log duration", "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" + "show": true, + "showHistogram": true }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, + "tooltipDecimals": 1, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 1, + "format": "s", + "logBase": 1, + "max": null, + "min": null, "show": true, - "values": [] + "splitFactor": null }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null }, { "aliasColors": {}, @@ -18587,101 +19136,74 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "", + "description": "The total count of rows that are processed by redo writer", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 4, + "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 8, "w": 12, "x": 0, - "y": 201 + "y": 17 }, "hiddenSeries": false, - "id": 143, + "id": 10031, "legend": { "alignAsTable": true, - "avg": false, + "avg": true, "current": true, - "hideEmpty": false, - "hideZero": true, - "max": false, - "min": true, + "max": true, + "min": false, "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 1, - "points": true, + "pointradius": 2, + "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/(access|miss).*/", - "fill": 0, - "points": false, - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "(sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance) - sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)) / sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", + "exemplar": true, + "expr": "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance)", "format": "time_series", - "hide": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "hit-rate-{{instance}}", + "legendFormat": "{{instance}}", + "queryType": "randomWalk", "refId": "A" }, { - "expr": "-sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", - "format": "time_series", - "hide": true, - "intervalFactor": 1, - "legendFormat": "access-{{instance}}", - "refId": "B" - }, - { - "expr": "-sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "miss-{{instance}}", - "refId": "C" - }, - { - "expr": "-sum(rate(tikv_cdc_old_value_cache_miss_none{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", + "exemplar": true, + "expr": "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (changefeed)", "format": "time_series", "hide": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "miss-none-{{instance}}", - "refId": "D" + "legendFormat": "total", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Old value cache hit", + "title": "Redo Write rows/s ", "tooltip": { "shared": true, "sort": 0, @@ -18697,20 +19219,20 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, - "max": "1", - "min": "0", + "max": null, + "min": null, "show": true }, { - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -18724,8 +19246,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "The total number of cache entries in the old value cache.", + "description": "Total number of bytes redo log written", "fieldConfig": { "defaults": {}, "overrides": [] @@ -18733,82 +19254,52 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 8, "w": 12, "x": 12, - "y": 201 + "y": 17 }, "hiddenSeries": false, - "id": 145, + "id": 10033, "legend": { - "alignAsTable": true, "avg": false, - "current": true, - "hideEmpty": false, - "hideZero": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": true, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 1, + "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*len/", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}-len", + "exemplar": true, + "expr": "sum(rate(ticdc_redo_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}", + "queryType": "randomWalk", "refId": "A" - }, - { - "expr": "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance) / sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}-avg entry bytes", - "refId": "B" - }, - { - "expr": "sum(tikv_cdc_old_value_cache_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}-quota", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Old value cache size", + "title": "Redo Write bytes/s ", "tooltip": { "shared": true, "sort": 0, @@ -18828,16 +19319,15 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "decimals": null, - "format": "none", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ], @@ -18852,52 +19342,41 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "", - "editable": true, - "error": false, + "description": "Redo bgUpdateLog worker busy ratio", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, - "grid": {}, "gridPos": { - "h": 7, + "h": 8, "w": 12, "x": 0, - "y": 208 + "y": 25 }, "hiddenSeries": false, - "id": 141, + "id": 10035, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, + "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, - "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.11", - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -18906,22 +19385,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_cdc_old_value_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance, cf, tag)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{instance}}-{{cf}}-{{tag}}", - "refId": "A", - "step": 10 + "exemplar": true, + "expr": "sum(rate(ticdc_redo_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])/10) by (changefeed,instance)", + "interval": "", + "legendFormat": "{{changefeed}}-{{instance}}", + "queryType": "randomWalk", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Old value seek operation", + "title": "Worker Busy Ratio", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -18936,11 +19413,11 @@ }, "yaxes": [ { - "format": "ops", + "format": "percent", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -18949,180 +19426,180 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { "align": false, "alignLevel": null } - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "The time consumed to get an old value (both from cache and from disk)", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 208 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 146, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "links": [], - "maxPerRow": 3, - "repeatDirection": "h", - "reverseYBuckets": false, - "targets": [ - { - "expr": "sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", - "format": "heatmap", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Old value seek duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, + } + ], + "title": "Redo", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 155, + "panels": [ { "aliasColors": {}, - "bars": false, + "bars": true, + "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 1, - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, + "description": "TiCDC process rss memory usage. TiCDC heap memory size in use ", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, "gridPos": { "h": 7, - "w": 6, - "x": 18, - "y": 208 + "w": 12, + "x": 0, + "y": 6 }, - "hiddenSeries": false, - "id": 142, + "id": 157, "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, - "current": true, - "hideEmpty": false, + "current": false, + "hideEmpty": true, "hideZero": true, - "max": true, + "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, - "sort": "current", - "sortDesc": true, "total": false, - "values": true + "values": false }, - "lines": true, + "lines": false, "linewidth": 1, "links": [], + "maxPerRow": 3, "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "paceLength": 10, "percentage": false, - "pluginVersion": "7.5.11", + "pluginVersion": "6.1.6", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "repeat": null, + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "172.16.5.33:47912", + "value": "172.16.5.33:47912" + }, + "runtime_instance": { + "selected": false, + "text": "10.2.6.163:8300", + "value": "10.2.6.163:8300" + } + }, + "seriesOverrides": [ + { + "alias": "alloc-from-os", + "fill": 3, + "lines": true, + "stack": false + }, + { + "alias": "gc-threshold", + "bars": false, + "color": "#C4162A", + "lines": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "gc", + "bars": false, + "color": "#C4162A", + "hideTooltip": true, + "legend": false, + "pointradius": 3, + "points": true, + "stack": false + } + ], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag))", + "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "{{instance}}-99%-{{tag}}", + "legendFormat": "alloc-from-os", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag))", + "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "{{instance}}-95%-{{tag}}", - "refId": "B" + "legendFormat": "estimate-inuse", + "refId": "H" }, { - "expr": "sum(rate(tikv_cdc_old_value_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag) / sum(rate(tikv_cdc_old_value_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag)", + "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", "format": "time_series", + "hide": false, "intervalFactor": 1, - "legendFormat": "{{instance}}-avg-{{tag}}", + "legendFormat": "estimate-garbage", "refId": "C" + }, + { + "expr": "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "reserved-by-go", + "refId": "B" + }, + { + "expr": "go_memstats_stack_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mspan_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mcache_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_buck_hash_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_gc_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_other_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "used-by-go", + "refId": "D" + }, + { + "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "gc-threshold", + "refId": "E" + }, + { + "expr": "(clamp_max(idelta(go_memstats_last_gc_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[1m]), 1) * go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}) > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "gc", + "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Old value seek duration", + "title": "Memory Usage", "tooltip": { + "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" @@ -19137,16 +19614,16 @@ }, "yaxes": [ { - "format": "s", - "label": null, + "format": "bytes", + "label": "", "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, "min": null, @@ -19157,73 +19634,75 @@ "align": false, "alignLevel": null } - } - ], - "title": "TiKV", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 294, - "panels": [ + }, { "aliasColors": {}, "bars": false, + "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, + "description": "Count of live objects.", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, "gridPos": { "h": 7, - "w": 4, - "x": 0, - "y": 15 + "w": 12, + "x": 12, + "y": 6 }, - "hiddenSeries": false, - "id": 321, + "id": 158, "legend": { + "alignAsTable": false, "avg": false, "current": false, + "hideEmpty": true, + "hideZero": true, "max": false, "min": false, + "rightSide": false, "show": true, + "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, + "links": [], + "maxPerRow": 3, "nullPointMode": "null", - "options": { - "alertThreshold": true - }, "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, + "pluginVersion": "6.1.6", + "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "scopedVars": { + "instance": { + "selected": false, + "text": "172.16.5.33:47912", + "value": "172.16.5.33:47912" + }, + "runtime_instance": { + "selected": false, + "text": "10.2.6.163:8300", + "value": "10.2.6.163:8300" + } + }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum without (from) (rate(ticdc_message_server_message_count{instance=~\"$ticdc_instance\"}[30s]))", - "interval": "", - "legendFormat": "{{instance}}", - "queryType": "randomWalk", + "expr": "go_memstats_heap_objects{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "objects", "refId": "A" } ], @@ -19231,8 +19710,9 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Message Receive Rate", + "title": "Estimated Live Objects", "tooltip": { + "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" @@ -19248,19 +19728,19 @@ "yaxes": [ { "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -19271,65 +19751,97 @@ { "aliasColors": {}, "bars": false, + "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, + "description": "TiCDC process Go garbage collection STW pause duration", + "editable": true, + "error": false, "fill": 1, - "fillGradient": 0, + "grid": {}, "gridPos": { "h": 7, - "w": 4, - "x": 4, - "y": 15 + "w": 12, + "x": 0, + "y": 13 }, - "hiddenSeries": false, - "id": 323, + "id": 160, "legend": { + "alignAsTable": false, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, + "links": [], + "nullPointMode": "null as zero", "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, + "pluginVersion": "6.1.6", + "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "scopedVars": { + "instance": { + "selected": false, + "text": "172.16.5.33:47912", + "value": "172.16.5.33:47912" + }, + "runtime_instance": { + "selected": false, + "text": "10.2.6.163:8300", + "value": "10.2.6.163:8300" + } + }, + "seriesOverrides": [ + {} + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum without (to) (rate(ticdc_message_client_message_count{instance=~\"$ticdc_instance\"}[30s]))", - "interval": "", - "legendFormat": "{{instance}}", - "queryType": "randomWalk", - "refId": "A" + "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"0\"}", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A", + "step": 40 + }, + { + "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile!~\"0|1\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{quantile}}", + "refId": "B" + }, + { + "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"1\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Message Send Rate", + "title": "GC STW Duration (last 256 GC cycles)", "tooltip": { + "msResolution": false, "shared": true, - "sort": 0, + "sort": 1, "value_type": "individual" }, "type": "graph", @@ -19342,11 +19854,11 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -19355,7 +19867,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -19369,83 +19881,104 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, + "description": "The throughput of Go's memory allocator.", + "fill": 1, "gridPos": { "h": 7, - "w": 4, - "x": 8, - "y": 15 + "w": 12, + "x": 12, + "y": 13 }, - "hiddenSeries": false, - "id": 352, + "id": 161, "legend": { + "alignAsTable": false, "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, + "current": true, + "max": true, "min": false, + "rightSide": false, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", - "options": { - "alertThreshold": true - }, "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "scopedVars": { + "instance": { + "selected": false, + "text": "172.16.5.33:47912", + "value": "172.16.5.33:47912" + }, + "runtime_instance": { + "selected": false, + "text": "10.2.6.163:8300", + "value": "10.2.6.163:8300" + } + }, + "seriesOverrides": [ + { + "alias": "sweep", + "transform": "negative-Y" + }, + { + "alias": "alloc-ops", + "yaxis": 2 + }, + { + "alias": "swepp-ops", + "transform": "negative-Y", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))", - "interval": "", - "legendFormat": "p50", - "queryType": "randomWalk", + "expr": "irate(go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "alloc", "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "p90", + }, + { + "expr": "irate((go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"})[30s:])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sweep", "refId": "B" }, { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "p99", + "expr": "irate(go_memstats_mallocs_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "alloc-ops", "refId": "C" + }, + { + "expr": "irate(go_memstats_frees_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "swepp-ops", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Message Batch Size", + "title": "Allocator Throughput", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, - "transformations": [], "type": "graph", "xaxis": { "buckets": null, @@ -19456,7 +19989,7 @@ }, "yaxes": [ { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -19464,7 +19997,7 @@ "show": true }, { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -19473,10 +20006,26 @@ } ], "yaxis": { - "align": false, + "align": true, "alignLevel": null } - }, + } + ], + "repeat": "runtime_instance", + "title": "Runtime $runtime_instance", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 10000, + "panels": [ { "aliasColors": {}, "bars": false, @@ -19484,21 +20033,19 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "fieldConfig": { - "defaults": { - "unit": "short" - }, + "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 15 + "h": 9, + "w": 12, + "x": 0, + "y": 1 }, "hiddenSeries": false, - "id": 354, + "id": 10001, "legend": { "avg": false, "current": false, @@ -19510,7 +20057,7 @@ }, "lines": true, "linewidth": 1, - "nullPointMode": "null", + "nullPointMode": "null as zero", "options": { "alertThreshold": true }, @@ -19521,39 +20068,33 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "p50", - "refId": "C" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))", + "expr": "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2", + "format": "time_series", "interval": "", - "legendFormat": "p95", + "legendFormat": "", "queryType": "randomWalk", "refId": "A" }, { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))", - "hide": false, + "expr": "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}", + "format": "time_series", "interval": "", - "legendFormat": "p99", - "refId": "B" + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Receive message batch bytes percentiles", + "title": "Pulsar published DDL schema count", "tooltip": { "shared": true, "sort": 0, @@ -19600,16 +20141,16 @@ "defaults": {}, "overrides": [] }, - "fill": 0, + "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 15 + "h": 9, + "w": 12, + "x": 12, + "y": 1 }, "hiddenSeries": false, - "id": 356, + "id": 10002, "legend": { "avg": false, "current": false, @@ -19621,7 +20162,7 @@ }, "lines": true, "linewidth": 1, - "nullPointMode": "null", + "nullPointMode": "null as zero", "options": { "alertThreshold": true }, @@ -19632,39 +20173,33 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))", + "expr": "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2", + "format": "time_series", "interval": "", - "legendFormat": "p50", + "legendFormat": "", "queryType": "randomWalk", "refId": "A" }, { "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "p95", - "refId": "B" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))", - "hide": false, + "expr": "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}", + "format": "time_series", "interval": "", - "legendFormat": "p99", - "refId": "C" + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Receive Message Bytes Percentile", + "title": "Pulsar published DDL schema success", "tooltip": { "shared": true, "sort": 0, @@ -19701,112 +20236,6 @@ "alignLevel": null } }, - { - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "left", - "filterable": false - }, - "decimals": 2, - "displayName": "", - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "displayName", - "value": "Time" - }, - { - "id": "unit", - "value": "time: YYYY-MM-DD HH:mm:ss" - }, - { - "id": "custom.align", - "value": null - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 20, - "x": 0, - "y": 22 - }, - "id": 350, - "options": { - "showHeader": true - }, - "pluginVersion": "7.5.11", - "targets": [ - { - "exemplar": true, - "expr": "ticdc_message_server_cur_stream_count", - "format": "time_series", - "instant": true, - "interval": "", - "legendFormat": "", - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "Stream Count Between Nodes", - "transformations": [ - { - "id": "labelsToFields", - "options": { - "valueLabel": "from" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "instance": false, - "job": true - }, - "indexByName": {}, - "renameByName": { - "instance": "dest\\src" - } - } - }, - { - "id": "merge", - "options": { - "reducers": [] - } - } - ], - "type": "table" - }, { "aliasColors": {}, "bars": false, @@ -19820,13 +20249,13 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 8, - "w": 4, - "x": 20, - "y": 22 + "h": 9, + "w": 12, + "x": 0, + "y": 10 }, "hiddenSeries": false, - "id": 358, + "id": 10003, "legend": { "avg": false, "current": false, @@ -19838,7 +20267,7 @@ }, "lines": true, "linewidth": 1, - "nullPointMode": "null", + "nullPointMode": "null as zero", "options": { "alertThreshold": true }, @@ -19849,31 +20278,33 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "sum(ticdc_message_server_cur_stream_count)", + "expr": "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2\r", + "format": "time_series", "interval": "", - "legendFormat": "actual", + "legendFormat": "", "queryType": "randomWalk", "refId": "A" }, { "exemplar": true, - "expr": "2 * count(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}) - 1", - "hide": false, + "expr": "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}", + "format": "time_series", "interval": "", - "legendFormat": "expected", - "refId": "B" + "legendFormat": "", + "queryType": "randomWalk", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Total Stream Count", + "title": "Pulsar published DDL schema fail", "tooltip": { "shared": true, "sort": 0, @@ -19909,317 +20340,198 @@ "align": false, "alignLevel": null } - } - ], - "title": "Peer Messages", - "type": "row" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 187, - "panels": [ - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "The latency distributions of fsync called by redo writer", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 172, - "legend": { - "show": true - }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "max(rate(ticdc_redo_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Redo fsync duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null }, { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The latency distributions of flushall called by redo writer", "fieldConfig": { "defaults": {}, "overrides": [] }, + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 8, + "h": 9, "w": 12, "x": 12, - "y": 8 + "y": 10 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 180, + "hiddenSeries": false, + "id": 10004, "legend": { - "show": true - }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "max(rate(ticdc_redo_flushall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Redo flushall duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, + "avg": false, + "current": false, + "max": false, + "min": false, "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_TEST-CLUSTER}", - "description": "The latency distributions of writeLog called by redoManager", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 + "total": false, + "values": false }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 175, - "legend": { - "show": true + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "max(rate(ticdc_redo_write_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", + "expr": "increase(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",\r\nschema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2", + "format": "time_series", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", + "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], + "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Redo write log duration", + "title": "Pulsar published DML schema count", "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true + "shared": true, + "sort": 0, + "value_type": "individual" }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, "show": true, - "splitFactor": null + "values": [] }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The latency distributions of flushLog called by redoManager", "fieldConfig": { "defaults": {}, "overrides": [] }, + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 8, + "h": 9, "w": 12, - "x": 12, - "y": 23 + "x": 0, + "y": 19 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 178, + "hiddenSeries": false, + "id": 10005, "legend": { - "show": true + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "pluginVersion": "6.1.6", - "reverseYBuckets": false, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.11", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "max(rate(ticdc_redo_flush_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", + "expr": "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2", + "format": "time_series", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", + "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], + "thresholds": [], "timeFrom": null, + "timeRegions": [], "timeShift": null, - "title": "Redo flush log duration", + "title": "Pulsar published DML schema success", "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true + "shared": true, + "sort": 0, + "value_type": "individual" }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, "show": true, - "splitFactor": null + "values": [] }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -20227,7 +20539,6 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The total count of rows that are processed by redo writer", "fieldConfig": { "defaults": {}, "overrides": [] @@ -20235,66 +20546,53 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 8, + "h": 9, "w": 12, - "x": 0, - "y": 31 + "x": 12, + "y": 19 }, "hiddenSeries": false, - "id": 170, + "id": 10006, "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, + "avg": false, + "current": false, + "max": false, "min": false, - "rightSide": true, "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "nullPointMode": "null", + "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, - "pluginVersion": "6.1.6", + "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "expr": "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2", "format": "time_series", "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", + "legendFormat": "", "queryType": "randomWalk", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (changefeed)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "total", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Redo Write rows/s ", + "title": "Pulsar published DML schema fail", "tooltip": { "shared": true, "sort": 0, @@ -20323,7 +20621,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -20337,7 +20635,6 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Total number of bytes redo log written", "fieldConfig": { "defaults": {}, "overrides": [] @@ -20345,25 +20642,26 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 8, + "h": 9, "w": 12, - "x": 12, - "y": 31 + "x": 0, + "y": 28 }, "hiddenSeries": false, - "id": 173, + "id": 10007, "legend": { + "alignAsTable": true, "avg": false, - "current": false, + "current": true, "max": false, "min": false, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, - "nullPointMode": "null", + "nullPointMode": "null as zero", "options": { "alertThreshold": true }, @@ -20374,14 +20672,15 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_redo_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "expr": "sum(pulsar_client_bytes_published{changefeed=~\"$changefeed\"}) by (changefeed, instance)", + "format": "time_series", "interval": "", - "legendFormat": "{{instance}}", + "legendFormat": "{{changefeed}}-{{instance}}", "queryType": "randomWalk", "refId": "A" } @@ -20390,7 +20689,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Redo Write bytes/s ", + "title": "Pulsar client bytes published ", "tooltip": { "shared": true, "sort": 0, @@ -20433,7 +20732,6 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Redo bgUpdateLog worker busy ratio", "fieldConfig": { "defaults": {}, "overrides": [] @@ -20441,27 +20739,26 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 8, + "h": 9, "w": 12, - "x": 0, - "y": 106 + "x": 12, + "y": 28 }, "hiddenSeries": false, - "id": 723, + "id": 10008, "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, "max": false, "min": false, - "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, - "nullPointMode": "null", + "nullPointMode": "null as zero", "options": { "alertThreshold": true }, @@ -20472,13 +20769,16 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_redo_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])/10) by (changefeed,instance)", + "expr": "sum(pulsar_client_connections_opened{changefeed=~\"$changefeed\"}) by (changefeed, instance)", + "format": "time_series", + "instant": false, "interval": "", + "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{instance}}", "queryType": "randomWalk", "refId": "A" @@ -20488,7 +20788,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Worker Busy Ratio", + "title": "Pulsar client connections opened", "tooltip": { "shared": true, "sort": 0, @@ -20504,7 +20804,7 @@ }, "yaxes": [ { - "format": "percent", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -20517,180 +20817,78 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "title": "Redo", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 155, - "panels": [ + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, - "bars": true, - "cacheTimeout": null, + "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "TiCDC process rss memory usage. TiCDC heap memory size in use ", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 7, + "h": 9, "w": 12, "x": 0, - "y": 6 + "y": 37 }, - "id": 157, + "hiddenSeries": false, + "id": 10009, "legend": { "alignAsTable": false, "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, + "current": true, "max": false, "min": false, - "rightSide": false, "show": true, - "sideWidth": null, "total": false, - "values": false + "values": true }, - "lines": false, + "lines": true, "linewidth": 1, - "links": [], - "maxPerRow": 3, - "nullPointMode": "null", + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, "percentage": false, - "pluginVersion": "6.1.6", - "pointradius": 5, + "pluginVersion": "7.5.11", + "pointradius": 2, "points": false, "renderer": "flot", - "repeat": null, - "repeatDirection": "h", - "scopedVars": { - "instance": { - "selected": false, - "text": "172.16.5.33:47912", - "value": "172.16.5.33:47912" - }, - "runtime_instance": { - "selected": false, - "text": "10.2.6.77:8300", - "value": "10.2.6.77:8300" - } - }, - "seriesOverrides": [ - { - "alias": "alloc-from-os", - "fill": 3, - "lines": true, - "stack": false - }, - { - "alias": "gc-threshold", - "bars": false, - "color": "#C4162A", - "lines": true, - "linewidth": 2, - "stack": false - }, - { - "alias": "gc", - "bars": false, - "color": "#C4162A", - "hideTooltip": true, - "legend": false, - "pointradius": 3, - "points": true, - "stack": false - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", + "exemplar": true, + "expr": "sum(pulsar_client_rpc_count{changefeed=~\"$changefeed\"}) by (changefeed, instance)", "format": "time_series", - "hide": false, + "instant": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "alloc-from-os", + "legendFormat": "{{changefeed}}-{{instance}}", + "queryType": "randomWalk", "refId": "A" - }, - { - "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "estimate-inuse", - "refId": "H" - }, - { - "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "estimate-garbage", - "refId": "C" - }, - { - "expr": "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "reserved-by-go", - "refId": "B" - }, - { - "expr": "go_memstats_stack_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mspan_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mcache_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_buck_hash_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_gc_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_other_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "used-by-go", - "refId": "D" - }, - { - "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "gc-threshold", - "refId": "E" - }, - { - "expr": "(clamp_max(idelta(go_memstats_last_gc_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[1m]), 1) * go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}) > 0", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "gc", - "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Usage", + "title": "Pulsar client rpc count", "tooltip": { - "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" @@ -20705,20 +20903,20 @@ }, "yaxes": [ { - "format": "bytes", - "label": "", + "format": "none", + "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { "format": "short", - "label": "", + "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -20726,74 +20924,153 @@ "alignLevel": null } }, + { + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 10010, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.11", + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.999, sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=~\"$changefeed\"}[1m])) by (le,changefeed,instance))\r", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{changefeed}}-{{instance}}-P999", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[1m])) by (changefeed,instance) / \r\nsum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[1m])) by (changefeed,instance)", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Pulsar client producer latency ", + "type": "timeseries" + }, { "aliasColors": {}, "bars": false, - "cacheTimeout": null, "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "Count of live objects.", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 7, + "h": 9, "w": 12, - "x": 12, - "y": 6 + "x": 0, + "y": 46 }, - "id": 158, + "hiddenSeries": false, + "id": 10011, "legend": { - "alignAsTable": false, "avg": false, "current": false, - "hideEmpty": true, - "hideZero": true, "max": false, "min": false, - "rightSide": false, "show": true, - "sideWidth": null, "total": false, "values": false }, "lines": true, "linewidth": 1, - "links": [], - "maxPerRow": 3, - "nullPointMode": "null", + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, "percentage": false, - "pluginVersion": "6.1.6", - "pointradius": 5, + "pluginVersion": "7.5.11", + "pointradius": 2, "points": false, "renderer": "flot", - "repeat": null, - "repeatDirection": "h", - "scopedVars": { - "instance": { - "selected": false, - "text": "172.16.5.33:47912", - "value": "172.16.5.33:47912" - }, - "runtime_instance": { - "selected": false, - "text": "10.2.6.77:8300", - "value": "10.2.6.77:8300" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_objects{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", + "exemplar": true, + "expr": "histogram_quantile(0.999, sum(rate(pulsar_client_producer_rpc_latency_seconds_bucket{changefeed=~\"$changefeed\"}[1m])) by (le,changefeed,instance))\r", "format": "time_series", - "hide": false, + "instant": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "objects", + "legendFormat": "{{changefeed}}-{{instance}}", + "queryType": "randomWalk", "refId": "A" } ], @@ -20801,9 +21078,8 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Estimated Live Objects", + "title": "Pulsar client producer rpc latency", "tooltip": { - "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" @@ -20818,20 +21094,20 @@ }, "yaxes": [ { - "format": "short", - "label": "", + "format": "s", + "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { "format": "short", - "label": "", + "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -20842,97 +21118,71 @@ { "aliasColors": {}, "bars": false, - "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "TiCDC process Go garbage collection STW pause duration", - "editable": true, - "error": false, + "description": "", + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, "fill": 1, - "grid": {}, + "fillGradient": 0, "gridPos": { - "h": 7, + "h": 9, "w": 12, - "x": 0, - "y": 13 + "x": 12, + "y": 46 }, - "id": 160, + "hiddenSeries": false, + "id": 10012, "legend": { - "alignAsTable": false, "avg": false, - "current": true, + "current": false, "max": false, "min": false, - "rightSide": false, "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, "percentage": false, - "pluginVersion": "6.1.6", - "pointradius": 5, + "pluginVersion": "7.5.11", + "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "instance": { - "selected": false, - "text": "172.16.5.33:47912", - "value": "172.16.5.33:47912" - }, - "runtime_instance": { - "selected": false, - "text": "10.2.6.77:8300", - "value": "10.2.6.77:8300" - } - }, - "seriesOverrides": [ - {} - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"0\"}", - "format": "time_series", - "hide": false, - "instant": false, - "intervalFactor": 1, - "legendFormat": "min", - "refId": "A", - "step": 40 - }, - { - "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile!~\"0|1\"}", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{quantile}}", - "refId": "B" - }, - { - "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"1\"}", + "exemplar": true, + "expr": "sum(pulsar_client_producer_pending_messages{changefeed=~\"$changefeed\"}) by (changefeed, instance)\r", "format": "time_series", "instant": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "max", - "refId": "C" + "legendFormat": "{{changefeed}}-{{instance}}", + "queryType": "randomWalk", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "GC STW Duration (last 256 GC cycles)", + "title": "Pulsar client producer pending messages", "tooltip": { - "msResolution": false, "shared": true, - "sort": 1, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -20945,11 +21195,11 @@ }, "yaxes": [ { - "format": "s", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -20958,7 +21208,7 @@ "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -20972,99 +21222,65 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "The throughput of Go's memory allocator.", + "description": "", + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 7, + "h": 9, "w": 12, "x": 12, - "y": 13 + "y": 46 }, - "id": 161, + "hiddenSeries": false, + "id": 10013, "legend": { - "alignAsTable": false, "avg": false, - "current": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": false, "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, "percentage": false, - "pointradius": 5, + "pluginVersion": "7.5.11", + "pointradius": 2, "points": false, "renderer": "flot", - "scopedVars": { - "instance": { - "selected": false, - "text": "172.16.5.33:47912", - "value": "172.16.5.33:47912" - }, - "runtime_instance": { - "selected": false, - "text": "10.2.6.77:8300", - "value": "10.2.6.77:8300" - } - }, - "seriesOverrides": [ - { - "alias": "sweep", - "transform": "negative-Y" - }, - { - "alias": "alloc-ops", - "yaxis": 2 - }, - { - "alias": "swepp-ops", - "transform": "negative-Y", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "irate(go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])", + "exemplar": true, + "expr": "sum(published_message_type_resolved_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\"}) by (changefeed)", "format": "time_series", + "instant": false, + "interval": "", "intervalFactor": 1, - "legendFormat": "alloc", + "legendFormat": "{{changefeed}}-{{topic}}", + "queryType": "randomWalk", "refId": "A" - }, - { - "expr": "irate((go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"})[30s:])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "sweep", - "refId": "B" - }, - { - "expr": "irate(go_memstats_mallocs_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "alloc-ops", - "refId": "C" - }, - { - "expr": "irate(go_memstats_frees_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "swepp-ops", - "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Allocator Throughput", + "title": "Pulsar client producer pending messages", "tooltip": { "shared": true, "sort": 0, @@ -21080,7 +21296,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "none", "label": null, "logBase": 1, "max": null, @@ -21088,7 +21304,7 @@ "show": true }, { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -21097,13 +21313,12 @@ } ], "yaxis": { - "align": true, + "align": false, "alignLevel": null } } ], - "repeat": "runtime_instance", - "title": "Runtime $runtime_instance", + "title": "Pulsar Sink", "type": "row" } ],