From c0a70b1f93319e1d295822d92f216be07ad4a421 Mon Sep 17 00:00:00 2001 From: Jan Horstmann Date: Sat, 7 Dec 2024 10:17:29 +0100 Subject: [PATCH] Use mixins for node-exporter and prometheus from PR As reviewing/merging the pull requests to update the dashboards for the node-exporter and prometheus mixins takes a long time ([1]), we use the mixins from the pull requests directly. This may be reverted once the pull requests are merged. [1] https://github.com/prometheus/node_exporter/pull/3147 https://github.com/prometheus/prometheus/pull/15334 Signed-off-by: Jan Horstmann --- .src/jsonnetfile.json | 4 +- .src/jsonnetfile.lock.json | 28 +- .../infrastructure/node-cluster-rsrc-use.json | 1316 ++++------- .../infrastructure/node-rsrc-use.json | 1327 ++++------- grafana/dashboards/infrastructure/nodes.json | 1467 +++++------- .../prometheus-remote-write.json | 2026 ++++++----------- .../dashboards/infrastructure/prometheus.json | 1644 ++++++------- prometheus/prometheus.rules | 4 +- 8 files changed, 2955 insertions(+), 4861 deletions(-) diff --git a/.src/jsonnetfile.json b/.src/jsonnetfile.json index 148f54c..0704c53 100644 --- a/.src/jsonnetfile.json +++ b/.src/jsonnetfile.json @@ -26,7 +26,7 @@ "subdir": "docs/node-mixin" } }, - "version": "master" + "version": "e67804b52a715ac6e64c25a72c0beb2e244c510b" }, { "source": { @@ -35,7 +35,7 @@ "subdir": "documentation/prometheus-mixin" } }, - "version": "main" + "version": "f5483284c642801310db66ddbeb13f6f1b78e43a" } ], "legacyImports": true diff --git a/.src/jsonnetfile.lock.json b/.src/jsonnetfile.lock.json index ceea336..1dea92c 100644 --- a/.src/jsonnetfile.lock.json +++ b/.src/jsonnetfile.lock.json @@ -21,16 +21,6 @@ "version": "a1d61cce1da59c71409b99b5c7568511fec661ea", "sum": "342u++/7rViR/zj2jeJOjshzglkZ1SY+hFNuyCBFMdc=" }, - { - "source": { - "git": { - "remote": "https://github.com/grafana/grafonnet-lib.git", - "subdir": "grafonnet-7.0" - } - }, - "version": "a1d61cce1da59c71409b99b5c7568511fec661ea", - "sum": "gCtR9s/4D5fxU9aKXg0Bru+/njZhA0YjLjPiASc61FM=" - }, { "source": { "git": { @@ -51,16 +41,6 @@ "version": "82a19822e54a0a12a51e24dbd48fcde717dc0864", "sum": "41w7p/rwrNsITqNHMXtGSJAfAyKmnflg6rFhKBduUxM=" }, - { - "source": { - "git": { - "remote": "https://github.com/grafana/jsonnet-libs.git", - "subdir": "grafana-builder" - } - }, - "version": "767befa8fb46a07be516dec2777d7d89909a529d", - "sum": "yxqWcq/N3E/a/XreeU6EuE6X7kYPnG0AspAQFKOjASo=" - }, { "source": { "git": { @@ -98,8 +78,8 @@ "subdir": "docs/node-mixin" } }, - "version": "cf8c6891cc610e54f70383addd4bb6079f0add35", - "sum": "cQCW+1N0Xae5yXecCWDK2oAlN0luBS/5GrwBYSlaFms=" + "version": "e67804b52a715ac6e64c25a72c0beb2e244c510b", + "sum": "PTQYhjoJACrffMXkKOKIhv7BmTzjI82Klt8G3/YStZs=" }, { "source": { @@ -108,8 +88,8 @@ "subdir": "documentation/prometheus-mixin" } }, - "version": "af2a1cb10c89de496cd4309ac532624b34112d74", - "sum": "OYT5u3S8DbamuJV/v3gbWSteOvFzMeNwMj+u4Apk7jM=" + "version": "f5483284c642801310db66ddbeb13f6f1b78e43a", + "sum": "ucMaBRtj28sNZjxYRv2TtKj9Ae50M+JpnLI5Hmi8Imc=" } ], "legacyImports": false diff --git a/grafana/dashboards/infrastructure/node-cluster-rsrc-use.json b/grafana/dashboards/infrastructure/node-cluster-rsrc-use.json index b40d090..d836c0f 100644 --- a/grafana/dashboards/infrastructure/node-cluster-rsrc-use.json +++ b/grafana/dashboards/infrastructure/node-cluster-rsrc-use.json @@ -1,889 +1,571 @@ { - "__inputs": [ ], - "__requires": [ ], - "annotations": { - "list": [ ] - }, - "editable": false, - "gnetId": null, "graphTooltip": 1, - "hideControls": false, - "id": null, - "links": [ ], - "refresh": "30s", - "rows": [ + "panels": [ { - "collapse": false, "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 2, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "((\n instance:node_cpu_utilisation:rate5m{job=\"node\", cluster=\"$cluster\"}\n *\n instance:node_num_cpu:sum{job=\"node\", cluster=\"$cluster\"}\n) != 0 )\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node\", cluster=\"$cluster\"}))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ instance }}", - "refId": "A" + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [ ], + "title": "CPU", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Utilisation", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "unit": "percentunit" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "showLegend": false }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 3, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "(\n instance:node_load1_per_cpu:ratio{job=\"node\", cluster=\"$cluster\"}\n / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node\", cluster=\"$cluster\"}))\n) != 0\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" + "expr": "((\n instance:node_cpu_utilisation:rate5m{job=\"node\", cluster=\"$cluster\"}\n *\n instance:node_num_cpu:sum{job=\"node\", cluster=\"$cluster\"}\n) != 0 )\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node\", cluster=\"$cluster\"}))\n", + "legendFormat": "{{ instance }}" + } + ], + "title": "CPU Utilisation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Saturation (Load1 per CPU)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "unit": "percentunit" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "legend": { + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "(\n instance:node_load1_per_cpu:ratio{job=\"node\", cluster=\"$cluster\"}\n / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node\", cluster=\"$cluster\"}))\n) != 0\n", + "legendFormat": "{{ instance }}" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU", - "titleSize": "h6", - "type": "row" + "title": "CPU Saturation (Load1 per CPU)", + "type": "timeseries" }, { - "collapse": false, "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 4, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "(\n instance:node_memory_utilisation:ratio{job=\"node\", cluster=\"$cluster\"}\n / scalar(count(instance:node_memory_utilisation:ratio{job=\"node\", cluster=\"$cluster\"}))\n) != 0\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 4, + "panels": [ ], + "title": "Memory", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Utilisation", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "unit": "percentunit" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 5, + "options": { + "legend": { + "showLegend": false }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 5, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node\", cluster=\"$cluster\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" + "expr": "(\n instance:node_memory_utilisation:ratio{job=\"node\", cluster=\"$cluster\"}\n / scalar(count(instance:node_memory_utilisation:ratio{job=\"node\", cluster=\"$cluster\"}))\n) != 0\n", + "legendFormat": "{{ instance }}" + } + ], + "title": "Memory Utilisation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Saturation (Major Page Faults)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "unit": "rds" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 6, + "options": { + "legend": { + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "rds", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "rds", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node\", cluster=\"$cluster\"}", + "legendFormat": "{{ instance }}" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory", - "titleSize": "h6", - "type": "row" + "title": "Memory Saturation (Major Page Faults)", + "type": "timeseries" }, { - "collapse": false, "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 6, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - { - "alias": "/Receive/", - "stack": "A" - }, - { - "alias": "/Transmit/", - "stack": "B", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node\", cluster=\"$cluster\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} Receive", - "refId": "A" - }, - { - "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node\", cluster=\"$cluster\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} Transmit", - "refId": "B" + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 7, + "panels": [ ], + "title": "Network", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Network Utilisation (Bytes Receive/Transmit)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Transmit/" }, - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 8, + "options": { + "legend": { + "showLegend": false }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 7, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - { - "alias": "/ Receive/", - "stack": "A" - }, - { - "alias": "/ Transmit/", - "stack": "B", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node\", cluster=\"$cluster\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} Receive", - "refId": "A" - }, - { - "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node\", cluster=\"$cluster\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} Transmit", - "refId": "B" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Network Saturation (Drops Receive/Transmit)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node\", cluster=\"$cluster\"} != 0", + "legendFormat": "{{ instance }} Receive" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node\", cluster=\"$cluster\"} != 0", + "legendFormat": "{{ instance }} Transmit" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6", - "type": "row" + "title": "Network Utilisation (Bytes Receive/Transmit)", + "type": "timeseries" }, { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 8, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "(\n instance_device:node_disk_io_time_seconds:rate5m{job=\"node\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node\", cluster=\"$cluster\"}))\n) != 0\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} {{device}}", - "refId": "A" + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Disk IO Utilisation", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Transmit/" }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 9, + "options": { + "legend": { + "showLegend": false }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 9, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "(\n instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node\", cluster=\"$cluster\"}))\n) != 0\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} {{device}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Disk IO Saturation", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node\", cluster=\"$cluster\"} != 0", + "legendFormat": "{{ instance }} Receive" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node\", cluster=\"$cluster\"} != 0", + "legendFormat": "{{ instance }} Transmit" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, + "title": "Network Saturation (Drops Receive/Transmit)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 10, + "panels": [ ], "title": "Disk IO", - "titleSize": "h6", "type": "row" }, { - "collapse": false, - "collapsed": false, - "panels": [ + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 11, + "options": { + "legend": { + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 10, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum without (device) (\n max without (fstype, mountpoint) ((\n node_filesystem_size_bytes{job=\"node\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n -\n node_filesystem_avail_bytes{job=\"node\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n ) != 0)\n)\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"})))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" + "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node\", cluster=\"$cluster\"}\n/ scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node\", cluster=\"$cluster\"}))\n", + "legendFormat": "{{ instance }} {{device}}" + } + ], + "title": "Disk IO Utilisation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Disk Space Utilisation", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "unit": "percentunit" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 12, + "options": { + "legend": { + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node\", cluster=\"$cluster\"}\n/ scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node\", cluster=\"$cluster\"}))\n", + "legendFormat": "{{ instance }} {{device}}" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, + "title": "Disk IO Saturation", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 13, + "panels": [ ], "title": "Disk Space", - "titleSize": "h6", "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 14, + "options": { + "legend": { + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum without (device) (\n max without (fstype, mountpoint) ((\n node_filesystem_size_bytes{job=\"node\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n -\n node_filesystem_avail_bytes{job=\"node\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n ) != 0)\n)\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"})))\n", + "legendFormat": "{{ instance }}" + } + ], + "title": "Disk Space Utilisation", + "type": "timeseries" } ], - "schemaVersion": 14, - "style": "dark", + "refresh": "30s", + "schemaVersion": 39, "tags": [ "node-exporter-mixin" ], "templating": { "list": [ { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data Source", "name": "datasource", - "options": [ ], "query": "prometheus", - "refresh": 1, - "regex": "", "type": "datasource" }, { - "allValue": null, - "current": { - "text": "", - "value": "" + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "datasource": "$datasource", "hide": 2, "includeAll": false, - "label": null, - "multi": false, "name": "cluster", - "options": [ ], "query": "label_values(node_time_seconds, cluster)", "refresh": 2, - "regex": "", "sort": 1, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refresh": 2, + "sort": 1 } ] }, @@ -891,33 +573,7 @@ "from": "now-1h", "to": "now" }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, "timezone": "utc", "title": "Node Exporter / USE Method / Cluster", - "uid": "3e97d1d02672cdd0861f4c97c64f89b2", - "version": 0 + "uid": "3e97d1d02672cdd0861f4c97c64f89b2" } diff --git a/grafana/dashboards/infrastructure/node-rsrc-use.json b/grafana/dashboards/infrastructure/node-rsrc-use.json index 1f8c1c8..5044932 100644 --- a/grafana/dashboards/infrastructure/node-rsrc-use.json +++ b/grafana/dashboards/infrastructure/node-rsrc-use.json @@ -1,909 +1,574 @@ { - "__inputs": [ ], - "__requires": [ ], - "annotations": { - "list": [ ] - }, - "editable": false, - "gnetId": null, "graphTooltip": 1, - "hideControls": false, - "id": null, - "links": [ ], - "refresh": "30s", - "rows": [ + "panels": [ { - "collapse": false, "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 2, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "instance:node_cpu_utilisation:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Utilisation", - "refId": "A" + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [ ], + "title": "CPU", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Utilisation", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "unit": "percentunit" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "showLegend": false }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 3, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "instance:node_load1_per_cpu:ratio{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Saturation", - "refId": "A" + "expr": "instance:node_cpu_utilisation:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "legendFormat": "Utilisation" + } + ], + "title": "CPU Utilisation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Saturation (Load1 per CPU)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "unit": "percentunit" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "legend": { + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "instance:node_load1_per_cpu:ratio{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "legendFormat": "Saturation" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU", - "titleSize": "h6", - "type": "row" + "title": "CPU Saturation (Load1 per CPU)", + "type": "timeseries" }, { - "collapse": false, "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 4, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "instance:node_memory_utilisation:ratio{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Utilisation", - "refId": "A" + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 4, + "panels": [ ], + "title": "Memory", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Utilisation", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "unit": "percentunit" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 5, + "options": { + "legend": { + "showLegend": false }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 5, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Major page Faults", - "refId": "A" + "expr": "instance:node_memory_utilisation:ratio{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "legendFormat": "Utilisation" + } + ], + "title": "Memory Utilisation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Saturation (Major Page Faults)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "unit": "rds" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 6, + "options": { + "legend": { + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "rds", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "rds", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "legendFormat": "Major page Faults" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory", - "titleSize": "h6", - "type": "row" + "title": "Memory Saturation (Major Page Faults)", + "type": "timeseries" }, { - "collapse": false, "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 6, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - { - "alias": "/Receive/", - "stack": "A" - }, - { - "alias": "/Transmit/", - "stack": "B", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Receive", - "refId": "A" - }, - { - "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Transmit", - "refId": "B" + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 7, + "panels": [ ], + "title": "Network", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Network Utilisation (Bytes Receive/Transmit)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Transmit/" }, - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 8, + "options": { + "legend": { + "showLegend": false }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 7, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - { - "alias": "/ Receive/", - "stack": "A" - }, - { - "alias": "/ Transmit/", - "stack": "B", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Receive", - "refId": "A" - }, - { - "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Transmit", - "refId": "B" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Network Saturation (Drops Receive/Transmit)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "legendFormat": "Receive" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "legendFormat": "Transmit" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6", - "type": "row" + "title": "Network Utilisation (Bytes Receive/Transmit)", + "type": "timeseries" }, { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 8, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{device}}", - "refId": "A" + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Disk IO Utilisation", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Transmit/" }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 9, + "options": { + "legend": { + "showLegend": false }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 9, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{device}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Disk IO Saturation", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "legendFormat": "Receive" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "legendFormat": "Transmit" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, + "title": "Network Saturation (Drops Receive/Transmit)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 10, + "panels": [ ], "title": "Disk IO", - "titleSize": "h6", "type": "row" }, { - "collapse": false, - "collapsed": false, - "panels": [ + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 11, + "options": { + "legend": { + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "fillGradient": 0, - "gridPos": { }, - "id": 10, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "sideWidth": null, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sort_desc(1 -\n (\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n /\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n ) != 0\n)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{device}}", - "refId": "A" + "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "legendFormat": "{{device}}" + } + ], + "title": "Disk IO Utilisation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Disk Space Utilisation", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "unit": "percentunit" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 12, + "options": { + "legend": { + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"} != 0", + "legendFormat": "{{device}}" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, + "title": "Disk IO Saturation", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 13, + "panels": [ ], "title": "Disk Space", - "titleSize": "h6", "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 14, + "options": { + "legend": { + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sort_desc(1 -\n (\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n /\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n ) != 0\n)\n", + "legendFormat": "{{device}}" + } + ], + "title": "Disk Space Utilisation", + "type": "timeseries" } ], - "schemaVersion": 14, - "style": "dark", + "refresh": "30s", + "schemaVersion": 39, "tags": [ "node-exporter-mixin" ], "templating": { "list": [ { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data Source", "name": "datasource", - "options": [ ], "query": "prometheus", - "refresh": 1, - "regex": "", "type": "datasource" }, { - "allValue": null, - "current": { - "text": "", - "value": "" + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "datasource": "$datasource", "hide": 2, "includeAll": false, - "label": null, - "multi": false, "name": "cluster", - "options": [ ], "query": "label_values(node_time_seconds, cluster)", "refresh": 2, - "regex": "", "sort": 1, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { - "allValue": null, - "current": { }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "name": "instance", - "options": [ ], "query": "label_values(node_exporter_build_info{job=\"node\", cluster=\"$cluster\"}, instance)", "refresh": 2, - "regex": "", "sort": 1, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" } ] }, @@ -911,33 +576,7 @@ "from": "now-1h", "to": "now" }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, "timezone": "utc", "title": "Node Exporter / USE Method / Node", - "uid": "fac67cfbe174d3ef53eb473d73d9212f", - "version": 0 + "uid": "fac67cfbe174d3ef53eb473d73d9212f" } diff --git a/grafana/dashboards/infrastructure/nodes.json b/grafana/dashboards/infrastructure/nodes.json index f19045f..ad79310 100644 --- a/grafana/dashboards/infrastructure/nodes.json +++ b/grafana/dashboards/infrastructure/nodes.json @@ -1,928 +1,705 @@ { - "__inputs": [ ], - "__requires": [ ], - "annotations": { - "list": [ ] - }, - "editable": false, - "gnetId": null, "graphTooltip": 1, - "hideControls": false, - "id": null, - "links": [ ], - "refresh": "30s", - "rows": [ + "panels": [ { - "collapse": false, "collapsed": false, - "panels": [ + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [ ], + "title": "CPU", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + }, + "max": 1, + "min": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 2, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node\", mode=~\"idle|iowait|steal\", instance=\"$instance\", cluster=\"$cluster\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node\", mode=\"idle\", instance=\"$instance\", cluster=\"$cluster\"})\n)\n", - "format": "time_series", - "intervalFactor": 5, - "legendFormat": "{{cpu}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "expr": "(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node\", mode=~\"idle|iowait|steal\", instance=\"$instance\", cluster=\"$cluster\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node\", mode=\"idle\", instance=\"$instance\", cluster=\"$cluster\"})\n)\n", + "intervalFactor": 5, + "legendFormat": "{{cpu}}" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "showPoints": "never" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "min": 0, + "unit": "short" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": 1, - "min": 0, - "show": true - } - ] + "expr": "node_load1{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", + "legendFormat": "1m load average" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 0, - "fillGradient": 0, - "gridPos": { }, - "id": 3, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node_load1{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "1m load average", - "refId": "A" - }, - { - "expr": "node_load5{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "5m load average", - "refId": "B" - }, - { - "expr": "node_load15{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "15m load average", - "refId": "C" - }, - { - "expr": "count(node_cpu_seconds_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", mode=\"idle\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "logical cores", - "refId": "D" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Load Average", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "expr": "node_load5{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", + "legendFormat": "5m load average" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "expr": "node_load15{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", + "legendFormat": "15m load average" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] + "expr": "count(node_cpu_seconds_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", mode=\"idle\"})", + "legendFormat": "logical cores" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU", - "titleSize": "h6", - "type": "row" + "title": "Load Average", + "type": "timeseries" }, { - "collapse": false, "collapsed": false, - "panels": [ + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 4, + "title": "Memory", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + }, + "min": 0, + "unit": "bytes" + } + }, + "gridPos": { + "h": 7, + "w": 18, + "x": 0, + "y": 9 + }, + "id": 5, + "options": { + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 4, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 9, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "(\n node_memory_MemTotal_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}\n-\n node_memory_MemFree_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}\n-\n node_memory_Buffers_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}\n-\n node_memory_Cached_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}\n)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory used", - "refId": "A" - }, - { - "expr": "node_memory_Buffers_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory buffers", - "refId": "B" - }, - { - "expr": "node_memory_Cached_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory cached", - "refId": "C" - }, - { - "expr": "node_memory_MemFree_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory free", - "refId": "D" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "expr": "(\n node_memory_MemTotal_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}\n-\n node_memory_MemFree_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}\n-\n node_memory_Buffers_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}\n-\n node_memory_Cached_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}\n)\n", + "legendFormat": "memory used" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "expr": "node_memory_Buffers_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", + "legendFormat": "memory buffers" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] + "expr": "node_memory_Cached_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", + "legendFormat": "memory cached" }, { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)" - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 80 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "node_memory_MemFree_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}", + "legendFormat": "memory free" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "thresholds": { + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" }, - "unit": "percent" - } + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] }, - "gridPos": { }, - "id": 5, - "span": 3, - "targets": [ - { - "expr": "100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}) /\n avg(node_memory_MemTotal_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"})\n* 100\n)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "" - } - ], - "title": "Memory Usage", - "transparent": false, - "type": "gauge" + "unit": "percent" + } + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 9 + }, + "id": 6, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"}) /\n avg(node_memory_MemTotal_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\"})\n* 100\n)\n" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory", - "titleSize": "h6", - "type": "row" + "title": "Memory Usage", + "type": "gauge" }, { - "collapse": false, "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 0, - "fillGradient": 0, - "gridPos": { }, - "id": 6, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 7, + "panels": [ ], + "title": "Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "showPoints": "never" }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - { - "alias": "/ read| written/", - "yaxis": 1 - }, - { - "alias": "/ io time/", - "yaxis": 2 - } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(node_disk_read_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} read", - "refId": "A" + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/ read| written/" }, - { - "expr": "rate(node_disk_written_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} written", - "refId": "B" + "properties": [ + { + "id": "unit", + "value": "Bps" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/ io time/" }, - { - "expr": "rate(node_disk_io_time_seconds_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"\"}[$__rate_interval])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}} io time", - "refId": "C" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Disk I/O", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "properties": [ + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 8, + "options": { + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "expr": "rate(node_disk_read_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} read" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "rate(node_disk_written_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} written" }, { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "yellow", - "value": 0.80000000000000004 - }, - { - "color": "red", - "value": 0.90000000000000002 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Mounted on" - }, - "properties": [ - { - "id": "custom.width", - "value": 260 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Size" - }, - "properties": [ - { - "id": "custom.width", - "value": 93 - } - ] - }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(node_disk_io_time_seconds_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} io time" + } + ], + "title": "Disk I/O", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "steps": [ { - "matcher": { - "id": "byName", - "options": "Used" - }, - "properties": [ - { - "id": "custom.width", - "value": 72 - } - ] + "color": "green" }, { - "matcher": { - "id": "byName", - "options": "Available" - }, - "properties": [ - { - "id": "custom.width", - "value": 88 - } - ] + "color": "yellow", + "value": 0.80000000000000004 }, { - "matcher": { - "id": "byName", - "options": "Used, %" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "custom.displayMode", - "value": "gradient-gauge" - }, - { - "id": "max", - "value": 1 - }, - { - "id": "min", - "value": 0 - } - ] + "color": "red", + "value": 0.90000000000000002 } ] }, - "gridPos": { }, - "id": 7, - "span": 6, - "targets": [ - { - "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", fstype!=\"\", mountpoint!=\"\"})\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "" + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Mounted on" }, - { - "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", fstype!=\"\", mountpoint!=\"\"})\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "" - } - ], - "title": "Disk Space Usage", - "transformations": [ - { - "id": "groupBy", - "options": { - "fields": { - "Value #A": { - "aggregations": [ - "lastNotNull" - ], - "operation": "aggregate" - }, - "Value #B": { - "aggregations": [ - "lastNotNull" - ], - "operation": "aggregate" - }, - "mountpoint": { - "aggregations": [ ], - "operation": "groupby" - } - } + "properties": [ + { + "id": "custom.width", + "value": 260 } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" }, - { - "id": "merge", - "options": { } - }, - { - "id": "calculateField", - "options": { - "alias": "Used", - "binary": { - "left": "Value #A (lastNotNull)", - "operator": "-", - "reducer": "sum", - "right": "Value #B (lastNotNull)" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } + "properties": [ + { + "id": "custom.width", + "value": 93 } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" }, - { - "id": "calculateField", - "options": { - "alias": "Used, %", - "binary": { - "left": "Used", - "operator": "/", - "reducer": "sum", - "right": "Value #A (lastNotNull)" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } + "properties": [ + { + "id": "custom.width", + "value": 72 } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" }, - { - "id": "organize", - "options": { - "excludeByName": { }, - "indexByName": { }, - "renameByName": { - "Value #A (lastNotNull)": "Size", - "Value #B (lastNotNull)": "Available", - "mountpoint": "Mounted on" - } + "properties": [ + { + "id": "custom.width", + "value": 88 } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used, %" }, - { - "id": "sortBy", - "options": { - "fields": { }, - "sort": [ - { - "field": "Mounted on" - } - ] + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "gauge" + } + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 } - } - ], - "transparent": false, - "type": "table" + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 9, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "max by (mountpoint) (node_filesystem_size_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "legendFormat": "" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "max by (mountpoint) (node_filesystem_avail_bytes{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", fstype!=\"\", mountpoint!=\"\"})\n", + "format": "table", + "instant": true, + "legendFormat": "" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Disk", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ + "title": "Disk Space Usage", + "transformations": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Network received (bits/s)", - "fill": 0, - "fillGradient": 0, - "gridPos": { }, - "id": 8, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(node_network_receive_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"lo\"}[$__rate_interval]) * 8", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}}", - "refId": "A" + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "mountpoint": { + "aggregations": [ ], + "operation": "groupby" + } } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Network Received", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + } + }, + { + "id": "merge" + }, + { + "id": "calculateField", + "options": { + "alias": "Used", + "binary": { + "left": "Value #A (lastNotNull)", + "operator": "-", + "reducer": "sum", + "right": "Value #B (lastNotNull)" }, - { - "format": "bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "mode": "binary", + "reduce": { + "reducer": "sum" } - ] + } }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Network transmitted (bits/s)", - "fill": 0, - "fillGradient": 0, - "gridPos": { }, - "id": 9, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(node_network_transmit_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"lo\"}[$__rate_interval]) * 8", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Network Transmitted", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "id": "calculateField", + "options": { + "alias": "Used, %", + "binary": { + "left": "Used", + "operator": "/", + "reducer": "sum", + "right": "Value #A (lastNotNull)" }, - { - "format": "bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { }, + "indexByName": { }, + "renameByName": { + "Value #A (lastNotNull)": "Size", + "Value #B (lastNotNull)": "Available", + "mountpoint": "Mounted on" } - ] + } + }, + { + "id": "sortBy", + "options": { + "fields": { }, + "sort": [ + { + "field": "Mounted on" + } + ] + } } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 10, + "panels": [ ], "title": "Network", - "titleSize": "h6", "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Network received (bits/s)", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "showPoints": "never" + }, + "min": 0, + "unit": "bps" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 11, + "options": { + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(node_network_receive_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"lo\"}[$__rate_interval]) * 8", + "intervalFactor": 1, + "legendFormat": "{{device}}" + } + ], + "title": "Network Received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Network transmitted (bits/s)", + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0 + }, + "min": 0, + "unit": "bps" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 12, + "options": { + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(node_network_transmit_bytes_total{job=\"node\", instance=\"$instance\", cluster=\"$cluster\", device!=\"lo\"}[$__rate_interval]) * 8", + "intervalFactor": 1, + "legendFormat": "{{device}}" + } + ], + "title": "Network Transmitted", + "type": "timeseries" } ], - "schemaVersion": 14, - "style": "dark", + "refresh": "30s", + "schemaVersion": 39, "tags": [ "node-exporter-mixin" ], "templating": { "list": [ { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data Source", "name": "datasource", - "options": [ ], "query": "prometheus", - "refresh": 1, - "regex": "", "type": "datasource" }, { - "allValue": null, - "current": { }, - "datasource": "$datasource", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "hide": 2, - "includeAll": false, "label": "Cluster", - "multi": false, "name": "cluster", - "options": [ ], "query": "label_values(node_uname_info{job=\"node\", sysname!=\"Darwin\"}, cluster)", "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { - "allValue": null, - "current": { }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "label": "Instance", - "multi": false, "name": "instance", - "options": [ ], "query": "label_values(node_uname_info{job=\"node\", cluster=\"$cluster\", sysname!=\"Darwin\"}, instance)", "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" } ] }, @@ -930,33 +707,7 @@ "from": "now-1h", "to": "now" }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, "timezone": "utc", "title": "Node Exporter / Nodes", - "uid": "7d57716318ee0dddbac5a7f451fb7753", - "version": 0 + "uid": "7d57716318ee0dddbac5a7f451fb7753" } diff --git a/grafana/dashboards/infrastructure/prometheus-remote-write.json b/grafana/dashboards/infrastructure/prometheus-remote-write.json index 304f88e..36a0c2d 100644 --- a/grafana/dashboards/infrastructure/prometheus-remote-write.json +++ b/grafana/dashboards/infrastructure/prometheus-remote-write.json @@ -1,1443 +1,777 @@ { - "__inputs": [ ], - "__requires": [ ], - "annotations": { - "list": [ ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [ ], - "refresh": "60s", - "rows": [ + "panels": [ { - "collapse": false, "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 2, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"} != 0)\n)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Highest Timestamp In vs. Highest Timestamp Sent", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [ ], + "title": "Timestamps", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "showPoints": "never" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, + "unit": "short" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 3, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "clamp_min(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m])\n, 0)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Rate[5m]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}\n-\n ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"} != 0)\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Timestamps", - "titleSize": "h6", - "type": "row" + "title": "Highest Timestamp In vs. Highest Timestamp Sent", + "type": "timeseries" }, { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 4, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]))\n- \n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]))\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Rate, in vs. succeeded or dropped [5m]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "showPoints": "never" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "unit": "short" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "clamp_min(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n-\n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m])\n, 0)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, + "title": "Rate[5m]", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 4, + "panels": [ ], "title": "Samples", - "titleSize": "h6", "type": "row" }, { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 5, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "minSpan": 6, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Current Shards", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 6, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Max Shards", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 7, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Min Shards", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "showPoints": "never" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, + "unit": "short" + } + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 5, + "options": { + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 8, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Desired Shards", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n-\n ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]))\n-\n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, + "title": "Rate, in vs. succeeded or dropped [5m]", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 6, + "panels": [ ], "title": "Shards", - "titleSize": "h6", "type": "row" }, { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 9, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Shard Capacity", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "showPoints": "never" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, + "unit": "short" + } + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 10, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}" + } + ], + "title": "Current Shards", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "showPoints": "never" }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"} or prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Pending Samples", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "unit": "short" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 8, + "options": { + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}" + } + ], + "title": "Max Shards", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "showPoints": "never" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "unit": "short" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 9, + "options": { + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}" + } + ], + "title": "Min Shards", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "showPoints": "never" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "unit": "short" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 10, + "options": { + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, + "title": "Desired Shards", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 11, + "panels": [ ], "title": "Shard Details", - "titleSize": "h6", "type": "row" }, { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 11, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "prometheus_tsdb_wal_segment_current{cluster=~\"$cluster\", instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "TSDB Current Segment", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "showPoints": "never" }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, + "unit": "short" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 12, + "options": { + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 12, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}} {{consumer}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Remote Write Current Segment", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}" + } + ], + "title": "Shard Capacity", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "showPoints": "never" }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "unit": "short" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 13, + "options": { + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"} or prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, + "title": "Pending Samples", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 14, + "panels": [ ], "title": "Segments", - "titleSize": "h6", "type": "row" }, { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 13, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Dropped Samples", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 14, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Failed Samples", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "showPoints": "never" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 15, + "options": { + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 15, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Retried Samples", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "prometheus_tsdb_wal_segment_current{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}}" + } + ], + "title": "TSDB Current Segment", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "showPoints": "never" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 16, + "options": { + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 16, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Enqueue Retries", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}} {{consumer}}" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, + "title": "Remote Write Current Segment", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 17, + "panels": [ ], "title": "Misc. Rates", - "titleSize": "h6", "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "showPoints": "never" + } + } + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 41 + }, + "id": 18, + "options": { + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}" + } + ], + "title": "Dropped Samples", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "showPoints": "never" + } + } + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 41 + }, + "id": 19, + "options": { + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}" + } + ], + "title": "Failed Samples", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "showPoints": "never" + } + } + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 41 + }, + "id": 20, + "options": { + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}" + } + ], + "title": "Retried Samples", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 0, + "showPoints": "never" + } + } + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 41 + }, + "id": 21, + "options": { + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\", url=~\"$url\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{cluster}}:{{instance}} {{remote_name}}:{{url}}" + } + ], + "title": "Enqueue Retries", + "type": "timeseries" } ], - "schemaVersion": 14, - "style": "dark", + "schemaVersion": 39, "tags": [ "prometheus-mixin" ], "templating": { "list": [ { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, "hide": 0, - "label": null, "name": "datasource", - "options": [ ], "query": "prometheus", - "refresh": 1, - "regex": "", "type": "datasource" }, { - "allValue": null, "current": { - "text": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "value": { - "selected": true, - "text": "All", - "value": "$__all" - } + "selected": false, + "text": "All", + "value": "All" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "datasource": "$datasource", "hide": 0, "includeAll": true, - "label": null, - "multi": false, "name": "cluster", - "options": [ ], "query": "label_values(prometheus_build_info, cluster)", "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { - "allValue": null, - "current": { - "text": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "value": { - "selected": true, - "text": "All", - "value": "$__all" - } + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "datasource": "$datasource", - "hide": 0, "includeAll": true, - "label": null, - "multi": false, "name": "instance", - "options": [ ], "query": "label_values(prometheus_build_info{cluster=~\"$cluster\"}, instance)", "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { - "allValue": null, - "current": { }, - "datasource": "$datasource", - "hide": 0, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "includeAll": true, - "label": null, - "multi": false, "name": "url", - "options": [ ], "query": "label_values(prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}, url)", "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" } ] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" + "60s" ] }, - "timezone": "browser", - "title": "Prometheus / Remote Write", - "version": 0 + "timezone": "utc", + "title": "Prometheus / Remote Write" } diff --git a/grafana/dashboards/infrastructure/prometheus.json b/grafana/dashboards/infrastructure/prometheus.json index ab442f5..6fb43c1 100644 --- a/grafana/dashboards/infrastructure/prometheus.json +++ b/grafana/dashboards/infrastructure/prometheus.json @@ -1,959 +1,734 @@ { - "annotations": { - "list": [ ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ ], - "refresh": "60s", - "rows": [ + "panels": [ { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Count", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ ], - "type": "hidden", - "unit": "short" + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [ ], + "title": "Prometheus Stats", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "decimals": 2, + "displayName": "", + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" }, - { - "alias": "Uptime", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ ], - "type": "number", - "unit": "s" + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "custom.align", + "value": null + }, + { + "id": "custom.hidden", + "value": "true" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cluster" }, - { - "alias": "Cluster", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "cluster", - "thresholds": [ ], - "type": "number", - "unit": "short" + "properties": [ + { + "id": "custom.align", + "value": null + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "displayName", + "value": "Cluster" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" }, - { - "alias": "Instance", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "instance", - "thresholds": [ ], - "type": "number", - "unit": "short" + "properties": [ + { + "id": "custom.align", + "value": null + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "displayName", + "value": "Job" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" }, - { - "alias": "Job", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "job", - "thresholds": [ ], - "type": "number", - "unit": "short" + "properties": [ + { + "id": "displayName", + "value": "Instance" + }, + { + "id": "custom.align", + "value": null + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "version" }, - { - "alias": "Version", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "version", - "thresholds": [ ], - "type": "number", - "unit": "short" + "properties": [ + { + "id": "displayName", + "value": "Version" + }, + { + "id": "custom.align", + "value": null + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" }, - { - "alias": "", - "colorMode": null, - "colors": [ ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "count by (cluster, job, instance, version) (prometheus_build_info{cluster=~\"$cluster\", job=~\"$job\", instance=~\"$instance\"})", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "A" + "properties": [ + { + "id": "displayName", + "value": "Count" + }, + { + "id": "custom.align", + "value": null + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.hidden", + "value": "true" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" }, - { - "expr": "max by (cluster, job, instance) (time() - process_start_time_seconds{cluster=~\"$cluster\", job=~\"$job\", instance=~\"$instance\"})", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "B" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Prometheus Stats", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" + "properties": [ + { + "id": "displayName", + "value": "Uptime" + }, + { + "id": "custom.align", + "value": null + }, + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "expr": "count by (cluster, job, instance, version) (prometheus_build_info{cluster=~\"$cluster\", job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "legendFormat": "" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "expr": "max by (cluster, job, instance) (time() - process_start_time_seconds{cluster=~\"$cluster\", job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "legendFormat": "" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, "title": "Prometheus Stats", - "titleSize": "h6" + "type": "table" }, { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[5m])) by (cluster, job, scrape_job, instance) * 1e3", - "format": "time_series", - "legendFormat": "{{cluster}}:{{job}}:{{instance}}:{{scrape_job}}", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Target Sync", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "panels": [ ], + "title": "Discovery", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "showPoints": "never" }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, + "min": 0, + "unit": "ms" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 4, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (cluster, job, instance) (prometheus_sd_discovered_targets{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"})", - "format": "time_series", - "legendFormat": "{{cluster}}:{{job}}:{{instance}}", - "legendLink": null + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[5m])) by (cluster, job, scrape_job, instance) * 1e3", + "format": "time_series", + "legendFormat": "{{cluster}}:{{job}}:{{instance}}:{{scrape_job}}" + } + ], + "title": "Target Sync", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "lineWidth": 0, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Targets", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "min": 0, + "unit": "short" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 5, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "expr": "sum by (cluster, job, instance) (prometheus_sd_discovered_targets{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "legendFormat": "{{cluster}}:{{job}}:{{instance}}" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Discovery", - "titleSize": "h6" + "title": "Targets", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 6, + "panels": [ ], + "title": "Retrieval", + "type": "row" }, { - "collapse": false, - "height": "250px", - "panels": [ + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "showPoints": "never" + }, + "min": 0, + "unit": "ms" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 7, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(prometheus_target_interval_length_seconds_sum{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", - "format": "time_series", - "legendFormat": "{{cluster}}:{{job}}:{{instance}} {{interval}} configured", - "legendLink": null + "expr": "rate(prometheus_target_interval_length_seconds_sum{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", + "format": "time_series", + "legendFormat": "{{cluster}}:{{job}}:{{instance}} {{interval}} configured" + } + ], + "title": "Average Scrape Interval Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "lineWidth": 0, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Average Scrape Interval Duration", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "min": 0, + "unit": "ms" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 8, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "expr": "sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))", + "format": "time_series", + "legendFormat": "exceeded body size limit: {{cluster}} {{job}} {{instance}}" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))", - "format": "time_series", - "legendFormat": "exceeded body size limit: {{cluster}} {{job}} {{instance}}", - "legendLink": null - }, - { - "expr": "sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_sample_limit_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))", - "format": "time_series", - "legendFormat": "exceeded sample limit: {{cluster}} {{job}} {{instance}}", - "legendLink": null - }, - { - "expr": "sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))", - "format": "time_series", - "legendFormat": "duplicate timestamp: {{cluster}} {{job}} {{instance}}", - "legendLink": null - }, - { - "expr": "sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_bounds_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))", - "format": "time_series", - "legendFormat": "out of bounds: {{cluster}} {{job}} {{instance}}", - "legendLink": null - }, - { - "expr": "sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_order_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))", - "format": "time_series", - "legendFormat": "out of order: {{cluster}} {{job}} {{instance}}", - "legendLink": null - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Scrape failures", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" + "expr": "sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_sample_limit_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))", + "format": "time_series", + "legendFormat": "exceeded sample limit: {{cluster}} {{job}} {{instance}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "expr": "sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))", + "format": "time_series", + "legendFormat": "duplicate timestamp: {{cluster}} {{job}} {{instance}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "expr": "sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_bounds_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))", + "format": "time_series", + "legendFormat": "out of bounds: {{cluster}} {{job}} {{instance}}" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "rate(prometheus_tsdb_head_samples_appended_total{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}[5m])", - "format": "time_series", - "legendFormat": "{{cluster}} {{job}} {{instance}}", - "legendLink": null + "expr": "sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_order_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))", + "format": "time_series", + "legendFormat": "out of order: {{cluster}} {{job}} {{instance}}" + } + ], + "title": "Scrape failures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "lineWidth": 0, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Appended Samples", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "min": 0, + "unit": "short" + } + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 9, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "expr": "rate(prometheus_tsdb_head_samples_appended_total{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "legendFormat": "{{cluster}} {{job}} {{instance}}" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Retrieval", - "titleSize": "h6" + "title": "Appended Samples", + "type": "timeseries" }, { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "prometheus_tsdb_head_series{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", - "format": "time_series", - "legendFormat": "{{cluster}} {{job}} {{instance}} head series", - "legendLink": null + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 10, + "panels": [ ], + "title": "Storage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "lineWidth": 0, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Head Series", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, + "min": 0, + "unit": "short" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 11, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "prometheus_tsdb_head_chunks{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", - "format": "time_series", - "legendFormat": "{{cluster}} {{job}} {{instance}} head chunks", - "legendLink": null + "expr": "prometheus_tsdb_head_series{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "legendFormat": "{{cluster}} {{job}} {{instance}} head series" + } + ], + "title": "Head Series", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "lineWidth": 0, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Head Chunks", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "min": 0, + "unit": "short" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 12, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "expr": "prometheus_tsdb_head_chunks{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "legendFormat": "{{cluster}} {{job}} {{instance}} head chunks" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Storage", - "titleSize": "h6" + "title": "Head Chunks", + "type": "timeseries" }, { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "rate(prometheus_engine_query_duration_seconds_count{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", - "format": "time_series", - "legendFormat": "{{cluster}} {{job}} {{instance}}", - "legendLink": null + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 13, + "panels": [ ], + "title": "Query", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "lineWidth": 0, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Query Rate", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, + "min": 0, + "unit": "short" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 14, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}) * 1e3", - "format": "time_series", - "legendFormat": "{{slice}}", - "legendLink": null + "expr": "rate(prometheus_engine_query_duration_seconds_count{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", + "format": "time_series", + "legendFormat": "{{cluster}} {{job}} {{instance}}" + } + ], + "title": "Query Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 100, + "lineWidth": 0, + "showPoints": "never", + "stacking": { + "mode": "normal" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Stage Duration", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "min": 0, + "unit": "ms" + } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 15, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}) * 1e3", + "format": "time_series", + "legendFormat": "{{slice}}" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Query", - "titleSize": "h6" + "title": "Stage Duration", + "type": "timeseries" } ], - "schemaVersion": 14, - "style": "dark", + "schemaVersion": 39, "tags": [ "prometheus-mixin" ], @@ -961,89 +736,70 @@ "list": [ { "current": { + "selected": false, "text": "default", "value": "default" }, "hide": 0, "label": "Data source", "name": "datasource", - "options": [ ], "query": "prometheus", - "refresh": 1, - "regex": "", "type": "datasource" }, { "allValue": ".+", "current": { - "selected": true, - "text": "All", - "value": "$__all" + "selected": false, + "text": [ + "All" + ], + "value": [ + "All" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "datasource": "$datasource", "hide": 0, "includeAll": true, "label": "cluster", "multi": true, "name": "cluster", - "options": [ ], "query": "label_values(prometheus_build_info{job=\"prometheus\"}, cluster)", - "refresh": 1, - "regex": "", + "refresh": 2, "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { "allValue": ".+", - "current": { - "selected": true, - "text": "All", - "value": "$__all" + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "datasource": "$datasource", - "hide": 0, "includeAll": true, "label": "job", "multi": true, "name": "job", - "options": [ ], "query": "label_values(prometheus_build_info{cluster=~\"$cluster\"}, job)", - "refresh": 1, - "regex": "", + "refresh": 2, "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" }, { "allValue": ".+", - "current": { - "selected": true, - "text": "All", - "value": "$__all" + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "datasource": "$datasource", - "hide": 0, "includeAll": true, "label": "instance", "multi": true, "name": "instance", - "options": [ ], "query": "label_values(prometheus_build_info{cluster=~\"$cluster\", job=~\"$job\"}, instance)", - "refresh": 1, - "regex": "", + "refresh": 2, "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false + "type": "query" } ] }, @@ -1053,31 +809,9 @@ }, "timepicker": { "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" + "60s" ] }, "timezone": "utc", - "title": "Prometheus / Overview", - "uid": "", - "version": 0 + "title": "Prometheus / Overview" } diff --git a/prometheus/prometheus.rules b/prometheus/prometheus.rules index f732349..33f65ab 100644 --- a/prometheus/prometheus.rules +++ b/prometheus/prometheus.rules @@ -47,8 +47,8 @@ "severity": "warning" - "alert": "PrometheusErrorSendingAlertsToSomeAlertmanagers" "annotations": - "description": "{{ printf \"%.1f\" $value }}% of alerts sent by Prometheus {{$labels.instance}} to Alertmanager {{$labels.alertmanager}} were affected by errors." - "summary": "More than 1% of alerts sent by Prometheus to a specific Alertmanager were affected by errors." + "description": "{{ printf \"%.1f\" $value }}% errors while sending alerts from Prometheus {{$labels.instance}} to Alertmanager {{$labels.alertmanager}}." + "summary": "Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager." "expr": | ( rate(prometheus_notifications_errors_total{job="prometheus"}[5m])