From f9f77d4364ffdb1dd73ea59ced2c9ef691bd2fcd Mon Sep 17 00:00:00 2001 From: NeoPlays <80448387+NeoPlays@users.noreply.github.com> Date: Mon, 10 Jun 2024 16:02:11 +0200 Subject: [PATCH] FIX IT --- .../dashboards/dash_charon_overview.json | 12841 ++++++++-------- .../tasks/2.1.4/updates-214.yaml | 2 +- .../tasks/2.1.5/updates-215.yaml | 2 +- 3 files changed, 6651 insertions(+), 6194 deletions(-) diff --git a/controls/roles/manage-service/files/grafana/CharonService/dashboards/dash_charon_overview.json b/controls/roles/manage-service/files/grafana/CharonService/dashboards/dash_charon_overview.json index fb4b528fd9..b3947e8ae1 100644 --- a/controls/roles/manage-service/files/grafana/CharonService/dashboards/dash_charon_overview.json +++ b/controls/roles/manage-service/files/grafana/CharonService/dashboards/dash_charon_overview.json @@ -1,6293 +1,6750 @@ { - "annotations": { - "list": [ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 2, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cluster-labels" + ], + "targetBlank": false, + "title": "", + "tooltip": "", + "type": "dashboards", + "url": "" + }, + { + "asDropdown": false, + "icon": "info", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Explore Charon Logs", + "tooltip": "", + "type": "link", + "url": "/explore?schemaVersion=1&panes=%7B\"xlh\"%3A%7B\"datasource\"%3A\"loki\"%2C\"queries\"%3A%5B%7B\"refId\"%3A\"A\"%2C\"expr\"%3A\"\"%2C\"queryType\"%3A\"range\"%2C\"datasource\"%3A%7B\"type\"%3A\"loki\"%2C\"uid\"%3A\"loki\"%7D%7D%5D%2C\"range\"%3A%7B\"from\"%3A\"now-1h\"%2C\"to\"%3A\"now\"%7D%7D%7D&orgId=1" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 168, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Status returned from the `/readyz` health status endpoint. Unhealthy reasons include beacon node problems, charon p2p problems, or validator client problems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" + "options": { + "0": { + "color": "text", + "index": 1, + "text": "Unknown" + }, + "1": { + "color": "green", + "index": 0, + "text": "OK" + }, + "2": { + "color": "red", + "index": 3, + "text": "BeaconNode Down" + }, + "3": { + "color": "orange", + "index": 4, + "text": "BeaconNode Syncing" + }, + "4": { + "color": "orange", + "index": 5, + "text": "Insufficient Peers" }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" + "5": { + "color": "orange", + "index": 6, + "text": "VC not connected" }, - "type": "dashboard" + "6": { + "color": "orange", + "index": 7, + "text": "VC missing validators" + }, + "7": { + "color": "orange", + "index": 8, + "text": "BeaconNode has zero peers" + }, + "8": { + "color": "orange", + "index": 9, + "text": "BeaconNode far behind" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "text", + "index": 2, + "text": "Dead" + } + }, + "type": "special" } - ] + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 170, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Health Status", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Successful cluster duty percentage by type over the whole time range", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 0.9 + }, + { + "color": "green", + "value": 0.99 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 14, + "x": 4, + "y": 1 + }, + "id": 176, + "options": { + "minVizHeight": 200, + "minVizWidth": 200, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_success_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) by (duty)\n) \n/ \n( \n sum(increase(core_tracker_expect_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) by (duty) \n) > 0", + "instant": true, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": false, + "refId": "A" + } + ], + "title": "Cluster Duty Success(%) ", + "type": "gauge" }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 10, - "links": [ + { + "datasource": "${DS_PROMETHEUS}", + "description": "Average beacon node score for the whole time window. The score is a combination of:\n - **errors**: the ratio or errors returned by the beacon node \n - **latency**: how fast or slow the beacon node is responding\n\n\nThe performance of the Beacon node is critical to the performance of the validators and therefore rewards. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 174, + "options": { + "minVizHeight": 200, + "minVizWidth": 200, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.4.2", + "targets": [ { - "asDropdown": false, - "icon": "external link", - "includeVars": false, - "keepTime": false, - "tags": [], - "targetBlank": false, - "title": "Explore Charon Logs", - "tooltip": "Navigate to Charon Logs Explorer", - "type": "link", - "url": "/explore?orgId=1&left=%7B%22datasource%22:%22loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bcluster_name%3D%5C%22$cluster_name%5C%22,%20cluster_hash%3D%5C%22$cluster_hash%5C%22,%20cluster_peer%3D%5C%22$cluster_peer%5C%22%7D%20%7C%20logfmt%20%7C%20line_format%20%60%7B%7B.pretty%7D%7D%60%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22loki%22%7D,%22editorMode%22:%22builder%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(clamp(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 4 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 5 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 6, 1, 1) OR on() vector(0))\n", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" }, { - "asDropdown": false, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "cluster-labels" - ], - "targetBlank": false, - "title": "New link", - "tooltip": "", - "type": "dashboards", - "url": "" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "(\n (\n0.5 * (1.0 - 10*(sum(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) / (sum(increase(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", endpoint !=\"submit_validator_registrations\"}[$__range]))))) OR on() vector(0.5)\n )\n +\n (\n 0.5 * (1.0 - clamp_max(histogram_quantile(0.99, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", endpoint !=\"submit_validator_registrations\"}[$__range])) by (le)),1))\n )\n)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + }, + { + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$A * $B", + "hide": false, + "refId": "C", + "type": "math" + } + ], + "title": "Beacon Node Score", + "transformations": [ + { + "id": "filterByRefId", + "options": { + "include": "C" + } } - ], - "liveNow": false, - "panels": [ + ], + "type": "gauge" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Information about the charon cluster.\n\n- **Network**: The blockchain network \n- **Operators/Nodes**: The number of operators or nodes or peers in the charon DV cluster.\n- **Threshold**: The minimum number of participating nodes (also quorum).\n- **Validators**: The number of validators added to the cluster, each validator is staking 32 ETH. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Network" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "1": { + "color": "blue", + "index": 0, + "text": "Mainnet" + }, + "2": { + "color": "purple", + "index": 1, + "text": "Goerli" + }, + "3": { + "color": "purple", + "index": 2, + "text": "Sepolia" + }, + "4": { + "color": "purple", + "index": 3, + "text": "Gnosis" + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 115, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 168, - "panels": [], - "title": "Overview", - "type": "row" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "( 1 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"mainnet\"} )\nor \n( 2 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"goerli\"} )\nor\n( 3 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"sepolia\"} )\nor\n( 4 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"gnosis\"} )", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": "Network", + "range": false, + "refId": "D" }, { - "datasource": "${DS_PROMETHEUS}", - "description": "Status returned from the `/readyz` health status endpoint. Unhealthy reasons include beacon node problems, charon p2p problems, or validator client problems.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "text", - "index": 1, - "text": "Unknown" - }, - "1": { - "color": "green", - "index": 0, - "text": "OK" - }, - "2": { - "color": "red", - "index": 3, - "text": "BeaconNode Down" - }, - "3": { - "color": "orange", - "index": 4, - "text": "BeaconNode Syncing" - }, - "4": { - "color": "orange", - "index": 5, - "text": "Insufficient Peers" - }, - "5": { - "color": "orange", - "index": 6, - "text": "VC not connected" - }, - "6": { - "color": "orange", - "index": 7, - "text": "VC missing validators" - }, - "7": { - "color": "orange", - "index": 8, - "text": "BeaconNode has zero peers" - }, - "8": { - "color": "orange", - "index": 9, - "text": "BeaconNode far behind" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "text", - "index": 2, - "text": "Dead" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 170, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Health Status", - "type": "stat" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "cluster_operators{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "instant": true, + "legendFormat": "Operators/Nodes", + "range": false, + "refId": "A" }, { - "datasource": "${DS_PROMETHEUS}", - "description": "Successful cluster duty percentage by type over the whole time range", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "links": [], - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "#EAB839", - "value": 0.9 - }, - { - "color": "green", - "value": 0.99 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 14, - "x": 4, - "y": 1 - }, - "id": 176, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": true - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_success_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) by (duty)\n) \n/ \n( \n sum(increase(core_tracker_expect_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) by (duty) \n) > 0", - "instant": true, - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": false, - "refId": "A" - } - ], - "title": "Cluster Duty Success(%) ", - "type": "gauge" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "cluster_threshold{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "legendFormat": "Threshold", + "range": false, + "refId": "B" }, { - "datasource": "${DS_PROMETHEUS}", - "description": "Average beacon node score for the whole time window. The score is a combination of:\n - **errors**: the ratio or errors returned by the beacon node \n - **latency**: how fast or slow the beacon node is responding\n\n\nThe performance of the Beacon node is critical to the performance of the validators and therefore rewards. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 0.5 - }, - { - "color": "green", - "value": 0.8 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 1 - }, - "id": 174, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(clamp(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 4 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 5 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 6, 1, 1) OR on() vector(0))\n*\non()\n(\n (\n0.5 * (1.0 - 10*(sum(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) / (sum(increase(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range]))))) OR on() vector(0.5)\n )\n +\n (\n 0.5 * (1.0 - clamp_max(histogram_quantile(0.99, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) by (le)),1))\n )\n)", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Beacon Node Score", - "type": "gauge" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "core_scheduler_validators_active{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "legendFormat": "Validators Active", + "range": false, + "refId": "E" }, { - "datasource": "${DS_PROMETHEUS}", - "description": "Information about the charon cluster.\n\n- **Network**: The blockchain network \n- **Operators/Nodes**: The number of operators or nodes or peers in the charon DV cluster.\n- **Threshold**: The minimum number of participating nodes (also quorum).\n- **Validators**: The number of validators added to the cluster, each validator is staking 32 ETH. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Network" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "1": { - "color": "blue", - "index": 0, - "text": "Mainnet" - }, - "2": { - "color": "purple", - "index": 1, - "text": "Goerli" - }, - "3": { - "color": "purple", - "index": 2, - "text": "Sepolia" - }, - "4": { - "color": "purple", - "index": 3, - "text": "Gnosis" - } - }, - "type": "value" - } - ] - } - ] - } + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "cluster_validators{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "legendFormat": "Validators Total", + "range": false, + "refId": "C" + } + ], + "title": "Cluster Info", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This panels shows any health checks that are failing. See for help troubleshooting. \n\nNote the different severities:\n- **info**: Only informational, might not be an actual problem.\n- **warning**: Performance is degraded, operator intervention only required if this persists for a long time.\n- **critical**: Significant problem, validators probably not operation at all. Immediate operator intervention required.\n \n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "noValue": "All checks are passing πŸŽ‰", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Severity" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "critical": { + "color": "red", + "index": 2, + "text": "critical" + }, + "info": { + "color": "blue", + "index": 0, + "text": "info" + }, + "warning": { + "color": "orange", + "index": 1, + "text": "warning" + } + }, + "type": "value" + } ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "custom.width", + "value": 75 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Check Name" }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 4 - }, - "id": 115, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "( 1 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"mainnet\"} )\nor \n( 2 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"goerli\"} )\nor\n( 3 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"sepolia\"} )\nor\n( 4 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"gnosis\"} )", - "format": "time_series", - "hide": false, - "instant": true, - "legendFormat": "Network", - "range": false, - "refId": "D" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "cluster_operators{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "instant": true, - "legendFormat": "Operators/Nodes", - "range": false, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "cluster_threshold{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "Threshold", - "range": false, - "refId": "B" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "core_scheduler_validators_active{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "Validators Active", - "range": false, - "refId": "E" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "cluster_validators{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "Validators Total", - "range": false, - "refId": "C" + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" } - ], - "title": "Cluster Info", - "transformations": [], - "type": "stat" + }, + { + "id": "color", + "value": { + "fixedColor": "super-light-yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 0, + "y": 8 + }, + "id": 233, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false }, + "showHeader": true + }, + "pluginVersion": "10.4.2", + "targets": [ { - "datasource": "${DS_PROMETHEUS}", - "description": "- **Peer**: The peer's name (inferred from their *charon-enr-private-key*)\n- **You**: ⭐️ is this local charon node\n- **Connected**: Whether you are currently connected to this peer.\n- **Direct**: Whether the connection is *direct* (πŸ‘) or *relay* (πŸ‘Ž) \n- **Latency**: The time messages take to travel to/from the peer.\n- **Version**: The version of the peer's charon node\n- **Supported**: Whether the peers charon version is supported/compatible or not.\n- **Uptime**\" The duration since peer was restarted.\n- **🚧Attest**: Number of attestation duties missed in the time window.\n- **🚧Propose**: Number of block proposal duties missed in the time window.\n- **🚧Other**: Number of other duties missed in the time window.\n\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "minWidth": 0, - "width": 90 - }, - "decimals": 0, - "mappings": [], - "noValue": "--", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Latency (ms)" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "from": 0, - "result": { - "color": "green", - "index": 0 - }, - "to": 150 - }, - "type": "range" - }, - { - "options": { - "from": 150, - "result": { - "color": "yellow", - "index": 1 - }, - "to": 300 - }, - "type": "range" - }, - { - "options": { - "from": 300, - "result": { - "color": "orange", - "index": 2 - }, - "to": 500 - }, - "type": "range" - }, - { - "options": { - "from": 500, - "result": { - "color": "red", - "index": 3 - }, - "to": 100000 - }, - "type": "range" - } - ] - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "lcd", - "type": "gauge", - "valueDisplayMode": "color" - } - }, - { - "id": "custom.width", - "value": 200 - }, - { - "id": "max", - "value": 700 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "You" - }, - "properties": [ - { - "id": "noValue", - "value": "⭐" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "auto" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "--" - }, - "1": { - "index": 1, - "text": "--" - } - }, - "type": "value" - } - ] - }, - { - "id": "custom.width", - "value": 60 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Connected" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": " ❌" - }, - "1": { - "index": 1, - "text": "βœ…" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ClockDiff" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "πŸ†—" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Direct" - }, - "properties": [ - { - "id": "custom.width", - "value": 55 - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "--" - } - }, - "type": "value" - }, - { - "options": { - "from": 1, - "result": { - "index": 1, - "text": "πŸ‘Ž" - }, - "to": 9 - }, - "type": "range" - }, - { - "options": { - "from": 10, - "result": { - "index": 2, - "text": "πŸ‘" - }, - "to": 1000 - }, - "type": "range" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Peer" - }, - "properties": [ - { - "id": "custom.width", - "value": 150 - } - ] + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(app_health_checks{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (name,severity) > 0", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Failing Health Checks", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true + }, + "indexByName": {}, + "renameByName": { + "Value": "", + "name": "Check Name", + "severity": "Severity" + } + } + } + ], + "type": "table" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "- **Peer**: The peer's name (inferred from their *charon-enr-private-key*)\n- **You**: ⭐️ is this local charon node\n- **Connected**: Whether you are currently connected to this peer.\n- **Direct**: Whether the connection is *direct* (πŸ‘) or *relay* (πŸ‘Ž) \n- **Latency**: The time messages take to travel to/from the peer.\n- **Version**: The version of the peer's charon node\n- **Supported**: Whether the peers charon version is supported/compatible or not.\n- **Uptime**\" The duration since peer was restarted.\n- **🚧Attest**: Number of attestation duties missed in the time window.\n- **🚧Propose**: Number of block proposal duties missed in the time window.\n- **🚧Other**: Number of other duties missed in the time window.\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "minWidth": 0, + "width": 90 + }, + "decimals": 0, + "mappings": [], + "noValue": "--", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Latency (ms)" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "from": 0, + "result": { + "color": "green", + "index": 0 + }, + "to": 150 + }, + "type": "range" + }, + { + "options": { + "from": 150, + "result": { + "color": "yellow", + "index": 1 + }, + "to": 300 + }, + "type": "range" + }, + { + "options": { + "from": 300, + "result": { + "color": "orange", + "index": 2 + }, + "to": 500 + }, + "type": "range" + }, + { + "options": { + "from": 500, + "result": { + "color": "red", + "index": 3 + }, + "to": 100000 + }, + "type": "range" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge", + "valueDisplayMode": "color" + } + }, + { + "id": "custom.width", + "value": 200 + }, + { + "id": "max", + "value": 700 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "You" + }, + "properties": [ + { + "id": "noValue", + "value": "⭐" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "--" + }, + "1": { + "index": 1, + "text": "--" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.width", + "value": 60 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Connected" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": " ❌" + }, + "1": { + "index": 1, + "text": "βœ…" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ClockDiff" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "πŸ†—" + } }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Direct" + }, + "properties": [ + { + "id": "custom.width", + "value": 55 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "--" + } + }, + "type": "value" + }, + { + "options": { + "from": 1, + "result": { + "index": 1, + "text": "πŸ‘Ž" + }, + "to": 9 + }, + "type": "range" + }, + { + "options": { + "from": 10, + "result": { + "index": 2, + "text": "πŸ‘" + }, + "to": 1000 + }, + "type": "range" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Peer" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Uptime" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Index" + }, + "properties": [ + { + "id": "custom.width", + "value": 30 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "🚧.*" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "thresholds" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "Uptime" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] + "color": "#585858", + "value": null }, { - "matcher": { - "id": "byName", - "options": "Index" - }, - "properties": [ - { - "id": "custom.width", - "value": 30 - } - ] + "color": "super-light-green", + "value": 0.00001 }, { - "matcher": { - "id": "byRegexp", - "options": "🚧.*" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "thresholds" - } - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "decimals", - "value": 1 - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#585858", - "value": null - }, - { - "color": "super-light-green", - "value": 0.00001 - }, - { - "color": "yellow", - "value": 0.01 - }, - { - "color": "red", - "value": 0.1 - } - ] - } - }, - { - "id": "noValue", - "value": "0%" - } - ] + "color": "yellow", + "value": 0.01 }, { - "matcher": { - "id": "byName", - "options": "Supported" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "❌" - }, - "1": { - "index": 1, - "text": "πŸ†—" - } - }, - "type": "value" - } - ] - } - ] + "color": "red", + "value": 0.1 } + ] + } + }, + { + "id": "noValue", + "value": "0%" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Supported" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "❌" + }, + "1": { + "index": 1, + "text": "πŸ†—" + } + }, + "type": "value" + } ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Builder API" }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 85, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "You" - } + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "❌" + }, + "1": { + "index": 1, + "text": "βœ…" + } + }, + "type": "value" + } ] + }, + { + "id": "custom.width", + "value": 99 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Version" }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "instant": true, - "range": false, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__rate_interval])) by (le,peer)) * 1000", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "B" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "F" + "properties": [ + { + "id": "custom.width", + "value": 104 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 19, + "x": 5, + "y": 8 + }, + "id": 85, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "You" + } + ] + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__rate_interval])) by (le,peer)) * 1000", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "B" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "F" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer)*10 + \nmax(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "K" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,version) ", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "L" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "time() - max(app_peerinfo_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} > 0) by (peer)", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "Q" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(app_peerinfo_index{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "R" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n) OR on() vector(0)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "C" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\".*proposer\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\".*proposer\"}[$__range])) by (peer) \n) OR on() vector(0)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "D" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n) OR on() vector(0)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "E" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(app_peerinfo_version_support{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer) ", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "G" + }, + { + "datasource": "${DS_PROMETHEUS}", + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "max(app_peerinfo_builder_api_enabled{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "H", + "useBackend": false + } + ], + "title": "Peer Connectivity and Missed Duties (🚧)", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "peer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 10": true, + "Time 11": true, + "Time 12": true, + "Time 13": true, + "Time 14": true, + "Time 15": true, + "Time 16": true, + "Time 17": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true, + "Time 8": true, + "Time 9": true, + "Value #F": false, + "Value #L": true, + "Value #M": true + }, + "includeByName": {}, + "indexByName": { + "Time 1": 9, + "Time 10": 22, + "Time 11": 23, + "Time 2": 11, + "Time 3": 12, + "Time 4": 13, + "Time 5": 14, + "Time 6": 15, + "Time 7": 16, + "Time 8": 17, + "Time 9": 21, + "Value #A": 3, + "Value #B": 5, + "Value #C": 18, + "Value #D": 19, + "Value #E": 20, + "Value #F": 2, + "Value #G": 7, + "Value #K": 4, + "Value #L": 10, + "Value #Q": 8, + "Value #R": 1, + "peer": 0, + "version": 6 + }, + "renameByName": { + "Value #A": "Connected", + "Value #B": "Latency (ms)", + "Value #C": "🚧Attest", + "Value #D": "🚧Propose", + "Value #E": "🚧Other", + "Value #F": "You", + "Value #G": "Supported", + "Value #H": "Builder API", + "Value #I": "Exit", + "Value #J": "ClockDiff", + "Value #K": "Direct", + "Value #L": "", + "Value #N": "SyncMsg", + "Value #O": "SyncContrib", + "Value #P": "PrepareContrib", + "Value #Q": "Uptime", + "Value #R": "Index", + "git_hash": "GitCommit", + "peer": "Peer", + "version": "Version" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 193, + "panels": [], + "title": "Overview Advanced", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 0, + "y": 16 + }, + "id": 83, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "none", + "sort": "none" + } + }, + "pluginVersion": "9.2.3", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (version)", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{version}}", + "range": true, + "refId": "A" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(app_git_commit{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (git_hash)", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{git_hash}}", + "range": true, + "refId": "B" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "rate(app_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) > 0", + "hide": false, + "instant": false, + "interval": "$interval", + "legendFormat": "Restarts", + "range": true, + "refId": "C" + } + ], + "title": "Versions and Git Hashes", + "type": "state-timeline" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Status returned from the `/readyz` health status endpoint. Unhealthy reasons include beacon node problems, charon p2p problems, or validator client problems.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#ffffff", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "color": "text", + "index": 1, + "text": "Unknown" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer)*10 + \nmax(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "K" + "1": { + "color": "green", + "index": 0, + "text": "OK" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,version) ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "L" + "2": { + "color": "red", + "index": 3, + "text": "BeaconNode Down" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "time() - max(app_peerinfo_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} > 0) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "Q" + "3": { + "color": "orange", + "index": 4, + "text": "BeaconNode Syncing" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(app_peerinfo_index{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "R" + "4": { + "color": "orange", + "index": 5, + "text": "Insufficient Peers" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n) OR on() vector(0)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "C" + "5": { + "color": "light-orange", + "index": 6, + "text": "VC not connected" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\".*proposer\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\".*proposer\"}[$__range])) by (peer) \n) OR on() vector(0)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "D" + "6": { + "color": "orange", + "index": 7, + "text": "VC missing validators" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n) OR on() vector(0)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "E" + "7": { + "color": "orange", + "index": 8, + "text": "BeaconNode has zero peers" }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(app_peerinfo_version_support{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer) ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "G" + "8": { + "index": 9, + "text": "BeaconNode far behind" } - ], - "title": "Peer Connectivity and Missed Duties (🚧)", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "peer" - } - }, - { - "id": "organize", + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "text", + "index": 2, + "text": "Dead" + } + }, + "type": "special" + } + ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#ccccdb", + "value": null + }, + { + "color": "green", + "value": 0.5 + }, + { + "color": "red", + "value": 1.5 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 7, + "y": 16 + }, + "id": 119, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.3", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "hide": false, + "instant": false, + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Historical Health Status", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Type of connections to each peer in the cluster during the time window:\n - 🟒 Green: Direct connection (this is the preferred)\n - 🟑 Yellow: Indirect relay connection (this is ok if only temporary)\n - πŸ”΄ Red: No connection to peer (this is a problem)\n\nSee https://docs.obol.tech/docs/charon/networking#external-p2p-network", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "green", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 9, + "x": 15, + "y": 16 + }, + "id": 87, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "(max(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer))\n+\n(1 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))\n+ \n(100 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer))", + "format": "time_series", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Peer Connections (🟒=direct, 🟑=relay, πŸ”΄=not connected)", + "type": "state-timeline" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [], + "max": 10, + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "P2P Reachability" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { "options": { - "excludeByName": { - "Time 1": true, - "Time 10": true, - "Time 11": true, - "Time 12": true, - "Time 13": true, - "Time 14": true, - "Time 15": true, - "Time 16": true, - "Time 17": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "Time 6": true, - "Time 7": true, - "Time 8": true, - "Time 9": true, - "Value #F": false, - "Value #L": true, - "Value #M": true - }, - "indexByName": { - "Time 1": 9, - "Time 10": 22, - "Time 11": 23, - "Time 2": 11, - "Time 3": 12, - "Time 4": 13, - "Time 5": 14, - "Time 6": 15, - "Time 7": 16, - "Time 8": 17, - "Time 9": 21, - "Value #A": 3, - "Value #B": 5, - "Value #C": 18, - "Value #D": 19, - "Value #E": 20, - "Value #F": 2, - "Value #G": 7, - "Value #K": 4, - "Value #L": 10, - "Value #Q": 8, - "Value #R": 1, - "peer": 0, - "version": 6 - }, - "renameByName": { - "Value #A": "Connected", - "Value #B": "Latency (ms)", - "Value #C": "🚧Attest", - "Value #D": "🚧Propose", - "Value #E": "🚧Other", - "Value #F": "You", - "Value #G": "Supported", - "Value #H": "Aggregate", - "Value #I": "Exit", - "Value #J": "ClockDiff", - "Value #K": "Direct", - "Value #L": "", - "Value #N": "SyncMsg", - "Value #O": "SyncContrib", - "Value #P": "PrepareContrib", - "Value #Q": "Uptime", - "Value #R": "Index", - "git_hash": "GitCommit", - "peer": "Peer", - "version": "Version" - } - } + "0": { + "color": "text", + "index": 0, + "text": "Unknown" + }, + "1": { + "color": "green", + "index": 1, + "text": "Public" + }, + "2": { + "color": "yellow", + "index": 2, + "text": "Private" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Uptime" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" } - ], - "type": "table" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 21 + }, + "id": 113, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 15, + "valueSize": 15 }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 193, - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 7, - "x": 0, - "y": 16 - }, - "id": 83, + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "time() - max(app_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "hide": false, + "instant": true, + "legendFormat": "Uptime", + "range": false, + "refId": "G" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "core_scheduler_current_slot{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Slot", + "range": false, + "refId": "A" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "core_scheduler_current_epoch{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "legendFormat": "Epoch", + "range": false, + "refId": "B" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "core_scheduler_validators_active{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "legendFormat": "Validators Active", + "range": false, + "refId": "C" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "hide": false, + "instant": true, + "legendFormat": "Charon Peers Connected", + "range": false, + "refId": "D" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(p2p_relay_connections{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) ", + "hide": false, + "instant": true, + "legendFormat": "Relays Connected", + "range": false, + "refId": "E" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "app_beacon_node_peers{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "legendFormat": "Beacon Node Peers", + "range": false, + "refId": "H" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "p2p_reachability_status{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "hide": false, + "instant": true, + "legendFormat": "P2P Reachability", + "range": false, + "refId": "F" + } + ], + "title": "Current Status", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Status and balance of each validator by public key with links to beaconcha.in.\n\nIf present, the \"❗️Peer Exits\" column indicate the number of peers that submitted partially signed exits for each validator.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-blue" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Balance" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "none", - "sort": "none" - } - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (version)", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{version}}", - "range": true, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(app_git_commit{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (git_hash)", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{git_hash}}", - "range": true, - "refId": "B" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "rate(app_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) > 0", - "hide": false, - "instant": false, - "interval": "$interval", - "legendFormat": "Restarts", - "range": true, - "refId": "C" - } - ], - "title": "Versions and Git Hashes", - "type": "state-timeline" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Status returned from the `/readyz` health status endpoint. Unhealthy reasons include beacon node problems, charon p2p problems, or validator client problems.", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "#ffffff", - "mode": "fixed" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "stepAfter", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "area" - } - }, - "decimals": 0, - "mappings": [ - { - "options": { - "0": { - "color": "text", - "index": 1, - "text": "Unknown" - }, - "1": { - "color": "green", - "index": 0, - "text": "OK" - }, - "2": { - "color": "red", - "index": 3, - "text": "BeaconNode Down" - }, - "3": { - "color": "orange", - "index": 4, - "text": "BeaconNode Syncing" - }, - "4": { - "color": "orange", - "index": 5, - "text": "Insufficient Peers" - }, - "5": { - "color": "light-orange", - "index": 6, - "text": "VC not connected" - }, - "6": { - "color": "orange", - "index": 7, - "text": "VC missing validators" - }, - "7": { - "color": "orange", - "index": 8, - "text": "BeaconNode has zero peers" - }, - "8": { - "index": 9, - "text": "BeaconNode far behind" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "text", - "index": 2, - "text": "Dead" - } - }, - "type": "special" - } - ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#ccccdb", - "value": null - }, - { - "color": "green", - "value": 0.5 - }, - { - "color": "red", - "value": 1.5 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 7, - "y": 16 - }, - "id": 119, + "0": { + "color": "transparent", + "index": 3 + } + }, + "type": "value" + }, + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "hide": false, - "instant": false, - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Historical Health Status", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Type of connections to each peer in the cluster during the time window:\n - 🟒 Green: Direct connection (this is the preferred)\n - 🟑 Yellow: Indirect relay connection (this is ok if only temporary)\n - πŸ”΄ Red: No connection to peer (this is a problem)\n\nSee https://docs.obol.tech/docs/charon/networking#external-p2p-network", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "#EAB839", - "value": 1 - }, - { - "color": "green", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 9, - "x": 15, - "y": 16 - }, - "id": 87, + "from": 32, + "result": { + "color": "green", + "index": 0 + }, + "to": 100 + }, + "type": "range" + }, + { "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "(max(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer))\n+\n(1 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))\n+ \n(100 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer))", - "format": "time_series", - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Peer Connections (🟒=direct, 🟑=relay, πŸ”΄=not connected)", - "type": "state-timeline" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "text", - "mode": "fixed" - }, - "mappings": [], - "max": 10, - "min": 0, - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "P2P Reachability" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "color": "text", - "index": 0, - "text": "Unknown" - }, - "1": { - "color": "green", - "index": 1, - "text": "Public" - }, - "2": { - "color": "yellow", - "index": 2, - "text": "Private" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Uptime" - }, - "properties": [ - { - "id": "unit", - "value": "s" - }, - { - "id": "decimals", - "value": 1 - }, - { - "id": "color", - "value": { - "fixedColor": "text", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 0, - "y": 21 - }, - "id": 113, + "from": 30, + "result": { + "color": "orange", + "index": 1 + }, + "to": 32 + }, + "type": "range" + }, + { "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "time() - max(app_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "hide": false, - "instant": true, - "legendFormat": "Uptime", - "range": false, - "refId": "G" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "core_scheduler_current_slot{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Slot", - "range": false, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "core_scheduler_current_epoch{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "Epoch", - "range": false, - "refId": "B" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "core_scheduler_validators_active{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "Validators Active", - "range": false, - "refId": "C" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "hide": false, - "instant": true, - "legendFormat": "Charon Peers Connected", - "range": false, - "refId": "D" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_relay_connections{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) ", - "hide": false, - "instant": true, - "legendFormat": "Relays Connected", - "range": false, - "refId": "E" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "app_beacon_node_peers{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "Beacon Node Peers", - "range": false, - "refId": "H" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "p2p_reachability_status{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "P2P Reachability", - "range": false, - "refId": "F" - } - ], - "title": "Current Status", - "transformations": [], - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Status and balance of each validator by public key with links to beaconcha.in.\n\nIf present, the \"❗️Peer Exits\" column indicate the number of peers that submitted partially signed exits for each validator.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "light-blue", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Balance" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "color": "transparent", - "index": 3 - } - }, - "type": "value" - }, - { - "options": { - "from": 32, - "result": { - "color": "green", - "index": 0 - }, - "to": 100 - }, - "type": "range" - }, - { - "options": { - "from": 30, - "result": { - "color": "orange", - "index": 1 - }, - "to": 32 - }, - "type": "range" - }, - { - "options": { - "from": 1, - "result": { - "color": "red", - "index": 2 - }, - "to": 30 - }, - "type": "range" - } - ] - }, - { - "id": "unit", - "value": "ETH" - }, - { - "id": "custom.width", - "value": 150 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": " Public Key" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "targetBlank": true, - "title": "beaconcha.in", - "url": "http://${__data.fields[\"cluster_network\"]}.beaconcha.in/validator/${__data.fields[\"pubkey_full\"]}" - } - ] - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "custom.width" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Status" - }, - "properties": [ - { - "id": "custom.width", - "value": 150 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Exits" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "__hide__" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 9, - "x": 4, - "y": 21 - }, - "id": 134, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(core_scheduler_validator_balance_gwei{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} / 1000000000) by (pubkey_full,pubkey, cluster_network)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "{{pubkey}}", - "range": false, - "refId": "B" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(core_scheduler_validator_status{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1) by (pubkey, status)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(core_parsigdb_exit_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (pubkey) ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "C" - } - ], - "title": "Validators", - "transformations": [ - { - "id": "joinByField", - "options": { - "byField": "pubkey", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Value #A": true, - "Value #B": false, - "cluster_network": false, - "job": true, - "pubkey_full": false - }, - "indexByName": { - "Time 1": 2, - "Time 2": 7, - "Time 3": 9, - "Value #A": 8, - "Value #B": 6, - "Value #C": 4, - "cluster_network": 3, - "pubkey": 0, - "pubkey_full": 1, - "status": 5 - }, - "renameByName": { - "Value #A": "", - "Value #B": "Balance", - "Value #C": "❗️Peer Exits", - "cluster_network": "__hide__", - "pubkey": " Public Key", - "pubkey_full": "__hide__", - "status": "Status" - } - } - } - ], - "type": "table" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Warning (w) and error (e) logs grouped by topic. \n\nOccasional warns and errors are normal, but a significant increase indicate problems that require operators to investigate by looking at the actual logs entries.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "area" - } - }, - "links": [ - { - "targetBlank": true, - "title": "Show in log explorer", - "url": "/explore?orgId=1&left=%7B%22datasource%22:%22loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcompose_service%3D%5C%22$job%5C%22%7D%20%7C%3D%20%60${__field.labels.msg}%60%20%7C%20logfmt%20%7C%20line_format%20%60%7B%7B.pretty%7D%7D%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D" - } - ], - "mappings": [], - "min": 0, - "noValue": "No errors or warnings", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 100 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 11, - "x": 13, - "y": 21 - }, - "id": 117, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "none", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(increase(app_log_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (topic) > 0 ", - "hide": false, - "interval": "$interval", - "legendFormat": "e:{{topic}}", - "range": true, - "refId": "B" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(increase(app_log_warn_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (topic) > 0 ", - "hide": false, - "interval": "$interval", - "legendFormat": "w:{{topic}}", - "range": true, - "refId": "A" - } - ], - "title": "Warning(w) and Error(e) logs (per $interval)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "- **Peer**: The peer's name (inferred from their *charon-enr-private-key*)\n- **You**: ⭐️ is this local charon node\n- **Connected**: Whether you are currently connected to this peer.\n- **Direct**: Whether the connection is *direct* (πŸ‘) or *relay* (πŸ‘Ž) \n- **Latency**: The time messages take to travel to/from the peer.\n- **Uptime**\" The duration since peer was restarted.\n- **ClockDiff**: Difference between local and peer's clock time. More than 2s is bad.\n- **🚧Attest**: Number of attestation duties missed in the time window.\n- **🚧Propose**: Number of block proposal duties missed in the time window.\n- **🚧Other**: Number of other duties missed in the time window.\n\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "minWidth": 0, - "width": 90 - }, - "decimals": 0, - "mappings": [], - "noValue": "--", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent" - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Latency" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "from": 0, - "result": { - "color": "green", - "index": 0 - }, - "to": 150 - }, - "type": "range" - }, - { - "options": { - "from": 150, - "result": { - "color": "yellow", - "index": 1 - }, - "to": 300 - }, - "type": "range" - }, - { - "options": { - "from": 300, - "result": { - "color": "orange", - "index": 2 - }, - "to": 500 - }, - "type": "range" - }, - { - "options": { - "from": 500, - "result": { - "color": "red", - "index": 3 - }, - "to": 100000 - }, - "type": "range" - } - ] - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "lcd", - "type": "gauge" - } - }, - { - "id": "custom.width", - "value": 200 - }, - { - "id": "max", - "value": 700 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "You" - }, - "properties": [ - { - "id": "noValue", - "value": "⭐" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "auto" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "--" - }, - "1": { - "index": 1, - "text": "--" - } - }, - "type": "value" - } - ] - }, - { - "id": "custom.width", - "value": 60 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Connected" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": " ❌" - }, - "1": { - "index": 1, - "text": "βœ…" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ClockDiff" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "πŸ†—" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Direct" - }, - "properties": [ - { - "id": "custom.width", - "value": 55 - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "--" - } - }, - "type": "value" - }, - { - "options": { - "from": 1, - "result": { - "index": 1, - "text": "πŸ‘Ž" - }, - "to": 9 - }, - "type": "range" - }, - { - "options": { - "from": 10, - "result": { - "index": 2, - "text": "πŸ‘" - }, - "to": 1000 - }, - "type": "range" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Peer" - }, - "properties": [ - { - "id": "custom.width", - "value": 150 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Uptime" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Index" - }, - "properties": [ - { - "id": "custom.width", - "value": 30 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "🚧.*" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "thresholds" - } - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "decimals", - "value": 1 - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#585858" - }, - { - "color": "super-light-green", - "value": 0.00001 - }, - { - "color": "yellow", - "value": 0.01 - }, - { - "color": "red", - "value": 0.1 - } - ] - } - }, - { - "id": "noValue", - "value": "0%" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 194, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "You" - } - ] - }, - "pluginVersion": "9.5.2", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "instant": true, - "range": false, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__rate_interval])) by (le,peer)) * 1000", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "B" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "F" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(round(app_peerinfo_clock_offset_seconds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},0.1)*1000) by (peer) ", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "J" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer)*10 + \nmax(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "K" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,version) ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "L" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(app_peerinfo_git_commit{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,git_hash) > 0", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "M" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "time() - max(app_peerinfo_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} > 0) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "Q" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "max(app_peerinfo_index{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "R" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n) > 0 ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "C" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"proposer\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"proposer\"}[$__range])) by (peer) \n) > 0 ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "D" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n) > 0 ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "E" - } - ], - "title": "Peer Connectivity and Missed Duties (🚧)", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "peer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time 1": true, - "Time 10": true, - "Time 11": true, - "Time 12": true, - "Time 13": true, - "Time 14": true, - "Time 15": true, - "Time 16": true, - "Time 17": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "Time 6": true, - "Time 7": true, - "Time 8": true, - "Time 9": true, - "Value #F": false, - "Value #L": true, - "Value #M": true - }, - "indexByName": { - "Time 1": 10, - "Time 10": 29, - "Time 11": 30, - "Time 12": 31, - "Time 13": 32, - "Time 14": 33, - "Time 15": 34, - "Time 16": 35, - "Time 17": 36, - "Time 2": 12, - "Time 3": 13, - "Time 4": 14, - "Time 5": 15, - "Time 6": 16, - "Time 7": 17, - "Time 8": 18, - "Time 9": 28, - "Value #A": 3, - "Value #B": 5, - "Value #C": 20, - "Value #D": 21, - "Value #E": 22, - "Value #F": 2, - "Value #G": 23, - "Value #H": 24, - "Value #J": 6, - "Value #K": 4, - "Value #L": 11, - "Value #M": 19, - "Value #N": 25, - "Value #O": 27, - "Value #P": 26, - "Value #Q": 9, - "Value #R": 1, - "git_hash": 8, - "peer": 0, - "version": 7 - }, - "renameByName": { - "Value #A": "Connected", - "Value #B": "Latency", - "Value #C": "🚧Attest", - "Value #D": "🚧Propose", - "Value #E": "🚧Other", - "Value #F": "You", - "Value #G": "PrepareAgg", - "Value #H": "Aggregate", - "Value #I": "Exit", - "Value #J": "ClockDiff", - "Value #K": "Direct", - "Value #L": "", - "Value #N": "SyncMsg", - "Value #O": "SyncContrib", - "Value #P": "PrepareContrib", - "Value #Q": "Uptime", - "Value #R": "Index", - "git_hash": "GitCommit", - "peer": "Peer", - "version": "Version" - } - } - } - ], - "type": "table" + "from": 1, + "result": { + "color": "red", + "index": 2 + }, + "to": 30 + }, + "type": "range" + } + ] + }, + { + "id": "unit", + "value": "ETH" + }, + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": " Public Key" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "beaconcha.in", + "url": "http://${__data.fields[\"cluster_network\"]}.beaconcha.in/validator/${__data.fields[\"pubkey_full\"]}" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" } - ], - "title": "Overview Advanced", - "type": "row" + }, + { + "id": "custom.width" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Exits" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__hide__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 4, + "y": 21 + }, + "id": 134, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": [ + "Value #B" + ], + "reducer": [ + "sum" + ], + "show": true }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Balance" + } + ] + }, + "pluginVersion": "10.4.2", + "targets": [ { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 178, - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "description": "Completed duties by type over time.", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "palette-classic" - }, - "custom": { - "fillOpacity": 100, - "lineWidth": 0 - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 92, - "options": { - "colWidth": 0.5, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "rowHeight": 0.43, - "showValue": "auto", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(core_bcast_broadcast_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) > 0", - "instant": false, - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": true, - "refId": "A" - } - ], - "title": "βœ… Completed duties by type ", - "type": "status-history" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Successful duty participation per peer (filtered by selected duty) over time.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 17 - }, - "id": 81, - "options": { - "colWidth": 0.9, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(core_tracker_participation_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\",duty=~\"$duty\"}[$interval])) by (peer) > 0 ", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "βœ… Successful Peer Participation: Duty=$duty", - "type": "status-history" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Failed duties by type over time. Use data link to show logs.", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "palette-classic" - }, - "custom": { - "fillOpacity": 100, - "lineWidth": 0 - }, - "links": [ - { - "targetBlank": true, - "title": "Show in log explorer", - "url": "/explore?orgId=1&left=%7B%22datasource%22:%22loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcompose_service%3D%5C%22$job%5C%22%7D%20%7C%3D%20%60Duty%20failed%60%20%7C%3D%20%60${__field.labels.duty}%60%20%7C%20logfmt%20%7C%20line_format%20%60%7B%7B.pretty%7D%7D%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D" - } - ], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 91, - "options": { - "colWidth": 0.5, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "rowHeight": 0.43, - "showValue": "auto", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) > 0", - "instant": false, - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": true, - "refId": "A" - } - ], - "title": "❌ Failed duties by type (per $interval)", - "type": "status-history" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Missed duty participation per peer (filtered by selected duty) over time.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 23 - }, - "id": 185, - "options": { - "colWidth": 0.9, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\",duty=~\"$duty\"}[$interval])) by (peer) > 0 ", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "🚧 Missed Peer Participation: Duty=$duty", - "type": "status-history" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Failed duties percentage by type over time", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 29 - }, - "id": 180, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty)\n) \n/ \n(\n sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) \n + \n sum(increase(core_bcast_broadcast_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) \n) > 0 ", - "instant": false, - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": true, - "refId": "A" - } - ], - "title": "❌ Failed duty percentage by type (per $interval)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Missed duty participation per duty (filtered by peer) over time.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 29 - }, - "id": 186, - "options": { - "colWidth": 0.9, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "repeat": "peer", - "repeatDirection": "v", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=\"$peer\"}[$interval])) by (duty) > 0 ", - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": true, - "refId": "A" - } - ], - "title": "🚧 Missed Peer Participation: Peer=$peer", - "type": "status-history" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of failed duties by reason per time window. Note this is only available from v0.16.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "noValue": "No failures πŸŽ‰", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 35 - }, - "id": 222, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(increase(core_tracker_failed_duty_reasons_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])) by (reason) > 0 ", - "hide": false, - "interval": "$interval", - "legendFormat": "{{.duty}}", - "range": true, - "refId": "A" - } - ], - "title": "🚨 Duty failure reason: $duty (per $interval)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Duties that was successfully broadcast to the Beacon Node, but that was never included on-chain. This could be due to broadcasting too late (too slow), but it could also just be to aspects of consensus layer out-of-our-control. See https://eth2book.info/bellatrix/part2/incentives/rewards/#remarks.", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 41 - }, - "id": 203, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(sum(increase(core_tracker_inclusion_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\"$duty\"}[$interval])) by (duty)) > 0 ", - "instant": false, - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": true, - "refId": "A" - } - ], - "title": "πŸ’₯ Broadcasted duties never included on-chain (per $interval)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "yellow", - "value": 2 - }, - { - "color": "orange", - "value": 5 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "slots" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 47 - }, - "id": 184, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "sum(core_tracker_inclusion_delay{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "interval": "$interval", - "legendFormat": "Instant", - "range": true, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(avg_over_time(core_tracker_inclusion_delay{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))", - "hide": false, - "interval": "$interval", - "legendFormat": "Avg", - "range": true, - "refId": "B" - } - ], - "title": "Attestation Inclusion Delay (slots)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "The delay after the start of the slot when the duty was successfully broadcasted. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "orange", - "value": 6 - }, - { - "color": "red", - "value": 15 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 218, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(histogram_quantile(0.90, rate(core_bcast_broadcast_delay_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval]))) ", - "hide": false, - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Broadcast delay: $duty", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "orange", - "value": 6 - }, - { - "color": "red", - "value": 15 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 59 - }, - "id": 208, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(histogram_quantile(0.90, increase(core_consensus_duration_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval]))) by (timer)", - "hide": false, - "interval": "$interval", - "legendFormat": "{{timer}}", - "range": true, - "refId": "A" - } - ], - "title": "Consensus duration: $duty", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "yellow", - "value": 2 - }, - { - "color": "orange", - "value": 4 - }, - { - "color": "red", - "value": 8 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 65 - }, - "id": 182, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(core_consensus_decided_rounds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}) by (timer)", - "hide": false, - "interval": "$interval", - "legendFormat": "{{timer}}", - "range": true, - "refId": "A" - } - ], - "title": "Consensus Rounds: $duty", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of consensus timeouts per time window.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "noValue": "No timeouts πŸŽ‰", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 71 - }, - "id": 189, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(increase(core_consensus_timeout_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])) by (timer) > 0", - "hide": false, - "interval": "$interval", - "legendFormat": "{{.duty}}", - "range": true, - "refId": "B" - } - ], - "title": "🚨 Consensus Timeouts (per $interval)", - "type": "timeseries" - } - ], - "title": "Duties", - "type": "row" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(core_scheduler_validator_balance_gwei{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} / 1000000000) by (pubkey_full,pubkey, cluster_network)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "{{pubkey}}", + "range": false, + "refId": "B" }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 123, - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 34 - }, - "id": 127, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(go_memstats_alloc_bytes{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "interval": "$interval", - "legendFormat": "Allocated Memory", - "range": true, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(go_memstats_heap_inuse_bytes{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "hide": true, - "legendFormat": "Inuse Heap", - "range": true, - "refId": "B" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 34 - }, - "id": 129, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.4", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(irate(process_cpu_seconds_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))", - "interval": "$interval", - "intervalFactor": 2, - "legendFormat": "CPU", - "metric": "go_gc_duration_seconds", - "range": true, - "refId": "A", - "step": 4 - } - ], - "title": "CPU", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "File Descriptors" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 41 - }, - "id": 125, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(go_goroutines{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "interval": "$interval", - "legendFormat": "Go Routines", - "range": true, - "refId": "A" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(process_open_fds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "hide": false, - "interval": "$interval", - "legendFormat": "File Descriptors", - "range": true, - "refId": "B" - } - ], - "title": "Go Routines and File Descriptors", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Total time spent on garbage collection per interval", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 41 - }, - "id": 130, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.4", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(increase(go_gc_duration_seconds_sum{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) ", - "interval": "$interval", - "intervalFactor": 2, - "legendFormat": "__auto", - "metric": "go_gc_duration_seconds", - "range": true, - "refId": "A", - "step": 4 - } - ], - "title": "Garbage Collection Duration (per $interval)", - "type": "timeseries" - } - ], - "title": "Process", - "type": "row" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(core_scheduler_validator_status{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1) by (pubkey, status)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 141, - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 191, - "options": { - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 0, - "showHeader": false - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(app_beacon_node_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (version)\n)\n ", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "__auto", - "range": false, - "refId": "B" - } - ], - "transformations": [ - { - "id": "labelsToFields", - "options": { - "keepLabels": [ - "version" - ], - "mode": "rows" - } - } - ], - "type": "table" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Upstream beacon node API request rate per second by endpoint", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "validators_by_pub_key", - "validators" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 29 - }, - "id": 142, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(rate(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", - "interval": "$interval", - "legendFormat": "{{endpoint}}", - "range": true, - "refId": "A" - } - ], - "title": "Beacon Node API requests rate", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Upstream beacon node API request latency (90th percentile) by endpoint by timewindow", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 29 - }, - "id": 138, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.90, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le,endpoint)) ", - "interval": "$interval", - "legendFormat": "{{endpoint}}", - "range": true, - "refId": "A" - } - ], - "title": "Beacon Node API request latency (90%)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of errors per endpoint per time window", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 36 - }, - "id": 144, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", - "interval": "$interval", - "legendFormat": "{{endpoint}}", - "range": true, - "refId": "A" - } - ], - "title": "🚨 Beacon Node API error count (per $interval)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Beacon node score over time", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "#ffffff", - "mode": "fixed" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 32, - "gradientMode": "hue", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "orange", - "value": 0.5 - }, - { - "color": "green", - "value": 0.8 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 36 - }, - "id": 145, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "clamp_min((clamp(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 4 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 5 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 6, 1, 1) OR on() vector(0))\n*\non()\n(\n (\n0.5 * (1.0 - 10*(sum(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) / (sum(increase(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))))) OR on() vector(0.5)\n )\n +\n (\n 0.5 * (1.0 - clamp_max(histogram_quantile(0.99, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le)),1))\n )\n), 0)", - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Beacon Node Score", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of consensus layer P2P peers the beacon node is connected to", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "#fffefe", - "mode": "fixed" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "#EAB839", - "value": 20 - }, - { - "color": "green", - "value": 40 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 43 - }, - "id": 147, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "app_beacon_node_peers{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "interval": "$interval", - "legendFormat": "Peer count", - "range": true, - "refId": "A" - } - ], - "title": "Beacon node peer count", - "type": "timeseries" - } - ], - "title": "Upstream Beacon Node", - "type": "row" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(core_parsigdb_exit_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (pubkey) ", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "C" + } + ], + "title": "Validators", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "pubkey", + "mode": "outer" + } }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 132, - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 158 - }, - "id": 136, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(rate(core_validatorapi_request_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0", - "interval": "$interval", - "legendFormat": "{{endpoint}}", - "range": true, - "refId": "A" - } - ], - "title": "Validator API requests rate", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 158 - }, - "id": 143, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.90, sum(rate(core_validatorapi_request_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le,endpoint)) ", - "interval": "$interval", - "legendFormat": "{{endpoint}}", - "range": true, - "refId": "A" - } - ], - "title": "Validator API request latency (90%)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of errors per endpoint per time window", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 165 - }, - "id": 139, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(increase(core_validatorapi_request_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", - "interval": "$interval", - "legendFormat": "{{endpoint}}", - "range": true, - "refId": "A" - } - ], - "title": "🚨 Validator API error count (per $interval)", - "type": "timeseries" - } - ], - "title": "Downstream Validator", - "type": "row" + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Value #A": true, + "Value #B": false, + "cluster_network": false, + "job": true, + "pubkey_full": false + }, + "indexByName": { + "Time 1": 2, + "Time 2": 7, + "Time 3": 9, + "Value #A": 8, + "Value #B": 6, + "Value #C": 4, + "cluster_network": 3, + "pubkey": 0, + "pubkey_full": 1, + "status": 5 + }, + "renameByName": { + "Value #A": "", + "Value #B": "Balance", + "Value #C": "❗️Peer Exits", + "cluster_network": "__hide__", + "pubkey": " Public Key", + "pubkey_full": "__hide__", + "status": "Status" + } + } + } + ], + "type": "table" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Warning (w) and error (e) logs grouped by topic. \n\nOccasional warns and errors are normal, but a significant increase indicate problems that require operators to investigate by looking at the actual logs entries.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Show in log explorer", + "url": "/explore?orgId=1&left=%7B%22datasource%22:%22loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcompose_service%3D%5C%22$job%5C%22%7D%20%7C%3D%20%60${__field.labels.msg}%60%20%7C%20logfmt%20%7C%20line_format%20%60%7B%7B.pretty%7D%7D%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D" + } + ], + "mappings": [], + "min": 0, + "noValue": "No errors or warnings", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 100 + } + ] + } }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 21 + }, + "id": 117, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "none", + "sort": "none" + } + }, + "targets": [ { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 149, - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "description": "Type of connections to each peer in the cluster during the time window:\n - 🟒 Green: Direct connection (this is the preferred)\n - 🟑 Yellow: Indirect relay connection (this is ok if only temporary)\n - πŸ”΄ Red: No connection to peer (this is a problem)\n\nSee https://docs.obol.tech/docs/charon/networking#external-p2p-network", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "#EAB839", - "value": 1 - }, - { - "color": "green", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 21 - }, - "id": 195, - "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "(max(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer))\n+\n(1 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))\n+ \n(100 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer))", - "format": "time_series", - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Peer Connections (🟒=direct, 🟑=relay, πŸ”΄=not connected)", - "type": "state-timeline" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "yellow", - "value": 0.15 - }, - { - "color": "orange", - "value": 0.3 - }, - { - "color": "red", - "value": 0.5 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 21 - }, - "id": 25, + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(increase(app_log_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (topic) > 0 ", + "hide": false, + "interval": "$interval", + "legendFormat": "e:{{topic}}", + "range": true, + "refId": "B" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(increase(app_log_warn_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (topic) > 0 ", + "hide": false, + "interval": "$interval", + "legendFormat": "w:{{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Warning(w) and Error(e) logs (per $interval)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "- **Peer**: The peer's name (inferred from their *charon-enr-private-key*)\n- **You**: ⭐️ is this local charon node\n- **Connected**: Whether you are currently connected to this peer.\n- **Direct**: Whether the connection is *direct* (πŸ‘) or *relay* (πŸ‘Ž) \n- **Latency**: The time messages take to travel to/from the peer.\n- **Uptime**\" The duration since peer was restarted.\n- **ClockDiff**: Difference between local and peer's clock time. More than 2s is bad.\n- **🚧Attest**: Number of attestation duties missed in the time window.\n- **🚧Propose**: Number of block proposal duties missed in the time window.\n- **🚧Other**: Number of other duties missed in the time window.\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "minWidth": 0, + "width": 90 + }, + "decimals": 0, + "mappings": [], + "noValue": "--", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Latency" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "mappings", + "value": [ + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (le,peer)) ", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "Peer ping latency (90%)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Time windows that contain ping errors per peer", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 27 - }, - "id": 166, + "from": 0, + "result": { + "color": "green", + "index": 0 + }, + "to": 150 + }, + "type": "range" + }, + { "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(increase(p2p_ping_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) > 0", - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Ping Errors", - "type": "state-timeline" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 27 - }, - "id": 160, + "from": 150, + "result": { + "color": "yellow", + "index": 1 + }, + "to": 300 + }, + "type": "range" + }, + { "options": { - "calculate": false, - "cellGap": 5, - "cellValues": { - "unit": "s" - }, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "max": 0.5, - "min": 0, - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdYlGn", - "steps": 82 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "show": true, - "yHistogram": false - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false - } - }, - "pluginVersion": "9.5.2", - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (le,peer))", - "interval": "", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "Peer ping latency (90%) (heatmap)", - "type": "heatmap" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 158, + "from": 300, + "result": { + "color": "orange", + "index": 2 + }, + "to": 500 + }, + "type": "range" + }, + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(rate(p2p_ping_latency_secs_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) ", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "Ping Rate", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of new libp2p connections per peer per time window", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 33 - }, - "id": 163, + "from": 500, + "result": { + "color": "red", + "index": 3 + }, + "to": 100000 + }, + "type": "range" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge" + } + }, + { + "id": "custom.width", + "value": 200 + }, + { + "id": "max", + "value": 700 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "You" + }, + "properties": [ + { + "id": "noValue", + "value": "⭐" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + }, + { + "id": "mappings", + "value": [ + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "exemplar": true, - "expr": "max(increase(p2p_peer_connection_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) > 0 ", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "New P2P Connections (per $interval)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "area" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "yellow", - "value": -0.5 - }, - { - "color": "green", - "value": -0.2 - }, - { - "color": "#EAB839", - "value": 0.2 - }, - { - "color": "red", - "value": 0.5 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 151, + "0": { + "index": 0, + "text": "--" + }, + "1": { + "index": 1, + "text": "--" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.width", + "value": 60 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Connected" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "max(app_peerinfo_clock_offset_seconds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=~\"$cluster_peer\"}) by (peer) ", - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Clock Offset", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Avg Rate of P2P network bytes sent plus received per peer (bytes/second) ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue" - } - ] - }, - "unit": "Bps" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "decbytes" - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 153, + "0": { + "index": 0, + "text": " ❌" + }, + "1": { + "index": 1, + "text": "βœ…" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ClockDiff" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "mappings", + "value": [ + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "0": { + "index": 0, + "text": "πŸ†—" + } }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(rate(p2p_peer_network_receive_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer) + sum(rate(p2p_peer_network_sent_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer)", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "P2P Network Avg Rate Rx+Tx", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 45 - }, - "id": 156, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Direct" + }, + "properties": [ + { + "id": "custom.width", + "value": 55 + }, + { + "id": "mappings", + "value": [ + { "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(p2p_relay_connections{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer) > 0 ", - "interval": "$interval", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Connected Relays", - "type": "state-timeline" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Total P2P network bytes sent plus received per peer per time window (bytes per interval)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue" - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "decbytes" - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 45 - }, - "id": 154, + "0": { + "index": 0, + "text": "--" + } + }, + "type": "value" + }, + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(increase(p2p_peer_network_receive_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer) + sum(increase(p2p_peer_network_sent_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer)", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "P2P Network Bytes Rx+Tx (per $interval)", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "The number of active libp2p streams by peer or relay. Note this can be further grouped by direction and protocol.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "dashed+area" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 64 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 51 - }, - "id": 226, + "from": 1, + "result": { + "index": 1, + "text": "πŸ‘Ž" + }, + "to": 9 + }, + "type": "range" + }, + { "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "${DS_PROMETHEUS}", - "editorMode": "code", - "expr": "sum(p2p_peer_streams{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "interval": "$interval", - "legendFormat": "{{peer}}", - "range": true, - "refId": "A" - } - ], - "title": "Active libp2p streams", - "type": "timeseries" - } - ], - "title": "P2P Networking", - "type": "row" - } - ], - "refresh": "", - "revision": 1, - "schemaVersion": 38, - "style": "dark", - "tags": [ - "cluster-labels", - "charon" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "hide": 0, - "includeAll": false, - "label": "datasource", - "multi": false, - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": false, - "text": "None", - "value": "" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(app_peer_name, cluster_name)", - "description": "", - "hide": 0, - "includeAll": false, - "label": "Cluster Name", - "multi": false, - "name": "cluster_name", - "options": [], - "query": { - "query": "label_values(app_peer_name, cluster_name)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" + "from": 10, + "result": { + "index": 2, + "text": "πŸ‘" + }, + "to": 1000 + }, + "type": "range" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Peer" }, - { - "current": { - "selected": false, - "text": "8439eb3", - "value": "8439eb3" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\"}, cluster_hash)", - "description": "", - "hide": 0, - "includeAll": false, - "label": "Cluster Hash", - "multi": false, - "name": "cluster_hash", - "options": [], - "query": { - "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\"}, cluster_hash)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Uptime" }, - { - "current": { - "selected": false, - "text": "excited-group", - "value": "excited-group" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\"}, cluster_peer)", - "description": "", - "hide": 0, - "includeAll": false, - "label": "Cluster Peer", - "multi": false, - "name": "cluster_peer", - "options": [], - "query": { - "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\"}, cluster_peer)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Index" }, - { - "current": { - "selected": false, - "text": "charon", - "value": "charon" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},job)", - "description": "Prometheus job (can be ignored)", - "hide": 2, - "includeAll": false, - "label": "--", - "multi": false, - "name": "job", - "options": [], - "query": { - "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},job)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" + "properties": [ + { + "id": "custom.width", + "value": 30 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "🚧.*" }, - { - "allValue": "", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\"}, peer)", - "description": "", - "hide": 2, - "includeAll": true, - "label": "P2P Peer", - "multi": false, - "name": "peer", - "options": [], - "query": { - "query": "label_values(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\"}, peer)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - }, - { - "auto": true, - "auto_count": 30, - "auto_min": "1m", - "current": { - "selected": false, - "text": "auto", - "value": "$__auto_interval_interval" - }, - "hide": 0, - "label": "Interval", - "name": "interval", - "options": [ - { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "thresholds" + } + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ { - "selected": false, - "text": "1d", - "value": "1d" + "color": "#585858" }, { - "selected": false, - "text": "7d", - "value": "7d" + "color": "super-light-green", + "value": 0.00001 }, { - "selected": false, - "text": "14d", - "value": "14d" + "color": "yellow", + "value": 0.01 }, { - "selected": false, - "text": "30d", - "value": "30d" + "color": "red", + "value": 0.1 } - ], - "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "queryValue": "", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" + ] + } + }, + { + "id": "noValue", + "value": "0%" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 194, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "You" + } + ] + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__rate_interval])) by (le,peer)) * 1000", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "B" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "F" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(round(app_peerinfo_clock_offset_seconds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},0.1)*1000) by (peer) ", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "J" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer)*10 + \nmax(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "K" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,version) ", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "L" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(app_peerinfo_git_commit{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,git_hash) > 0", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "M" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "time() - max(app_peerinfo_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} > 0) by (peer)", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "Q" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "max(app_peerinfo_index{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "R" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n) > 0 ", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "C" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"proposer\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"proposer\"}[$__range])) by (peer) \n) > 0 ", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "D" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n) > 0 ", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "E" + } + ], + "title": "Peer Connectivity and Missed Duties (🚧)", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "peer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 10": true, + "Time 11": true, + "Time 12": true, + "Time 13": true, + "Time 14": true, + "Time 15": true, + "Time 16": true, + "Time 17": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true, + "Time 8": true, + "Time 9": true, + "Value #F": false, + "Value #L": true, + "Value #M": true + }, + "indexByName": { + "Time 1": 10, + "Time 10": 29, + "Time 11": 30, + "Time 12": 31, + "Time 13": 32, + "Time 14": 33, + "Time 15": 34, + "Time 16": 35, + "Time 17": 36, + "Time 2": 12, + "Time 3": 13, + "Time 4": 14, + "Time 5": 15, + "Time 6": 16, + "Time 7": 17, + "Time 8": 18, + "Time 9": 28, + "Value #A": 3, + "Value #B": 5, + "Value #C": 20, + "Value #D": 21, + "Value #E": 22, + "Value #F": 2, + "Value #G": 23, + "Value #H": 24, + "Value #J": 6, + "Value #K": 4, + "Value #L": 11, + "Value #M": 19, + "Value #N": 25, + "Value #O": 27, + "Value #P": 26, + "Value #Q": 9, + "Value #R": 1, + "git_hash": 8, + "peer": 0, + "version": 7 + }, + "renameByName": { + "Value #A": "Connected", + "Value #B": "Latency", + "Value #C": "🚧Attest", + "Value #D": "🚧Propose", + "Value #E": "🚧Other", + "Value #F": "You", + "Value #G": "PrepareAgg", + "Value #H": "Aggregate", + "Value #I": "Exit", + "Value #J": "ClockDiff", + "Value #K": "Direct", + "Value #L": "", + "Value #N": "SyncMsg", + "Value #O": "SyncContrib", + "Value #P": "PrepareContrib", + "Value #Q": "Uptime", + "Value #R": "Index", + "git_hash": "GitCommit", + "peer": "Peer", + "version": "Version" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 178, + "panels": [], + "title": "Duties", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Completed duties by type over time.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 100, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0 + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 92, + "options": { + "colWidth": 0.5, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.43, + "showValue": "auto", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(core_bcast_broadcast_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) > 0", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "βœ… Completed duties by type ", + "type": "status-history" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Successful duty participation per peer (filtered by selected duty) over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 81, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(core_tracker_participation_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\",duty=~\"$duty\"}[$interval])) by (peer) > 0 ", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "βœ… Successful Peer Participation: Duty=$duty", + "type": "status-history" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Failed duties by type over time. Use data link to show logs.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 100, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0 + }, + "links": [ + { + "targetBlank": true, + "title": "Show in log explorer", + "url": "/explore?orgId=1&left=%7B%22datasource%22:%22loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcompose_service%3D%5C%22$job%5C%22%7D%20%7C%3D%20%60Duty%20failed%60%20%7C%3D%20%60${__field.labels.duty}%60%20%7C%20logfmt%20%7C%20line_format%20%60%7B%7B.pretty%7D%7D%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 91, + "options": { + "colWidth": 0.5, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.43, + "showValue": "auto", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) > 0", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "❌ Failed duties by type (per $interval)", + "type": "status-history" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Missed duty participation per peer (filtered by selected duty) over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 185, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\",duty=~\"$duty\"}[$interval])) by (peer) > 0 ", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "🚧 Missed Peer Participation: Duty=$duty", + "type": "status-history" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Failed duties percentage by type over time", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 180, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty)\n) \n/ \n(\n sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) \n + \n sum(increase(core_bcast_broadcast_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) \n) > 0 ", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "❌ Failed duty percentage by type (per $interval)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Missed duty participation per duty (filtered by peer) over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 186, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "repeat": "peer", + "repeatDirection": "v", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=\"$peer\"}[$interval])) by (duty) > 0 ", + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "🚧 Missed Peer Participation: Peer=$peer", + "type": "status-history" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Number of failed duties by reason per time window. Note this is only available from v0.16.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "noValue": "No failures πŸŽ‰", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 222, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(increase(core_tracker_failed_duty_reasons_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])) by (reason) > 0 ", + "hide": false, + "interval": "$interval", + "legendFormat": "{{.duty}}", + "range": true, + "refId": "A" + } + ], + "title": "🚨 Duty failure reason: $duty (per $interval)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Which operators are having a growing number of relay registrations failures, indicating there may be a risk to an upcoming MEV block production opportunity. Be conscious that absence of an operator from this list may mean their MEV registration is working, or it may mean it is not enabled at all and thus not failing. ", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#ffffff", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "green", + "value": 0.1 + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "orange", + "value": 200 + }, + { + "color": "red", + "value": 500 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "none" + }, + { + "id": "color", + "value": { + "fixedColor": "#808080c4", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 61 + }, + "id": 242, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(increase(app_eth2_errors_total{endpoint=~\"submit_validator_registrations\", cluster_name=~\"$cluster_name\"}[$__range])) by (cluster_peer, cluster_name)", + "hide": false, + "legendFormat": "{{cluster_name}}: {{cluster_peer}}", + "range": true, + "refId": "A" + } + ], + "title": "⚠️ MEV Relay Submit Registrations Failure Growth by Operator", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Duties that was successfully broadcast to the Beacon Node, but that was never included on-chain. This could be due to broadcasting too late (too slow), but it could also just be to aspects of consensus layer out-of-our-control. See https://eth2book.info/bellatrix/part2/incentives/rewards/#remarks.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 203, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(sum(increase(core_tracker_inclusion_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\"$duty\"}[$interval])) by (duty)) > 0 ", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "πŸ’₯ Broadcasted duties never included on-chain (per $interval)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "slots" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 73 + }, + "id": 184, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "sum(core_tracker_inclusion_delay{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "interval": "$interval", + "legendFormat": "Instant", + "range": true, + "refId": "A" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(avg_over_time(core_tracker_inclusion_delay{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))", + "hide": false, + "interval": "$interval", + "legendFormat": "Avg", + "range": true, + "refId": "B" + } + ], + "title": "Attestation Inclusion Delay (slots)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "The delay after the start of the slot when the duty was successfully broadcasted. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "orange", + "value": 6 + }, + { + "color": "red", + "value": 15 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 79 + }, + "id": 218, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(histogram_quantile(0.90, rate(core_bcast_broadcast_delay_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval]))) ", + "hide": false, + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Broadcast delay: $duty", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "orange", + "value": 6 + }, + { + "color": "red", + "value": 15 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 208, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(histogram_quantile(0.90, increase(core_consensus_duration_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval]))) by (timer)", + "hide": false, + "interval": "$interval", + "legendFormat": "{{timer}}", + "range": true, + "refId": "A" + } + ], + "title": "Consensus duration: $duty", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "orange", + "value": 4 + }, + { + "color": "red", + "value": 8 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 91 + }, + "id": 182, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(core_consensus_decided_rounds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}) by (timer)", + "hide": false, + "interval": "$interval", + "legendFormat": "{{timer}}", + "range": true, + "refId": "A" + } + ], + "title": "Consensus Rounds: $duty", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Number of consensus timeouts per time window.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "noValue": "No timeouts πŸŽ‰", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 97 + }, + "id": 189, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(increase(core_consensus_timeout_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])) by (timer) > 0", + "hide": false, + "interval": "$interval", + "legendFormat": "{{.duty}}", + "range": true, + "refId": "B" + } + ], + "title": "🚨 Consensus Timeouts (per $interval)", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 123, + "panels": [ + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 127, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(go_memstats_alloc_bytes{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "interval": "$interval", + "legendFormat": "Allocated Memory", + "range": true, + "refId": "A" }, { - "allValue": ".*", - "current": { - "selected": false, - "text": "All", - "value": "$__all" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(go_memstats_heap_inuse_bytes{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "hide": true, + "legendFormat": "Inuse Heap", + "range": true, + "refId": "B" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "hide": 0, - "includeAll": true, - "label": "Duty", - "multi": false, - "name": "duty", - "options": [ - { - "selected": true, - "text": "All", - "value": "$__all" - }, - { - "selected": false, - "text": "attester", - "value": "attester" - }, - { - "selected": false, - "text": "prepare_aggregator", - "value": "prepare_aggregator" - }, - { - "selected": false, - "text": "aggregator", - "value": "aggregator" - }, - { - "selected": false, - "text": "sync_message", - "value": "sync_message" - }, - { - "selected": false, - "text": "prepare_sync_contribution", - "value": "prepare_sync_contribution" - }, - { - "selected": false, - "text": "sync_contribution", - "value": "sync_contribution" - }, - { - "selected": false, - "text": "proposer", - "value": "proposer" - }, - { - "selected": false, - "text": "randao", - "value": "randao" - }, - { - "selected": false, - "text": "exit", - "value": "exit" - }, - { - "selected": false, - "text": "builder_proposer", - "value": "builder_proposer" - }, - { - "selected": false, - "text": "builder_registration", - "value": "builder_registration" - }, - { - "selected": false, - "text": "info_sync", - "value": "info_sync" - } + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 129, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.4", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(irate(process_cpu_seconds_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "CPU", + "metric": "go_gc_duration_seconds", + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "File Descriptors" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 125, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(go_goroutines{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "interval": "$interval", + "legendFormat": "Go Routines", + "range": true, + "refId": "A" + }, + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(process_open_fds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "hide": false, + "interval": "$interval", + "legendFormat": "File Descriptors", + "range": true, + "refId": "B" + } + ], + "title": "Go Routines and File Descriptors", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Total time spent on garbage collection per interval", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 130, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.4", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(increase(go_gc_duration_seconds_sum{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) ", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "__auto", + "metric": "go_gc_duration_seconds", + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": "Garbage Collection Duration (per $interval)", + "type": "timeseries" + } + ], + "title": "Process", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 104 + }, + "id": 141, + "panels": [], + "title": "Upstream Beacon Node", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 105 + }, + "id": 191, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": false + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(app_beacon_node_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (version)\n)\n ", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "B" + } + ], + "transformations": [ + { + "id": "labelsToFields", + "options": { + "keepLabels": [ + "version" + ], + "mode": "rows" + } + } + ], + "type": "table" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Upstream beacon node API request rate per second by endpoint", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 107 + }, + "id": 142, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(rate(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "Beacon Node API requests rate", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Upstream beacon node API request latency (90th percentile) by endpoint by timewindow", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 107 + }, + "id": 138, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le,endpoint)) ", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "Beacon Node API request latency (90%)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Number of errors per endpoint per time window", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "submit_beacon_block" ], - "query": "attester,prepare_aggregator,aggregator,sync_message,prepare_sync_contribution,sync_contribution,proposer,randao,exit,builder_proposer,builder_registration,info_sync", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 114 + }, + "id": 144, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "🚨 Beacon Node API error count (per $interval)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Beacon node score over time", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#ffffff", + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 114 + }, + "id": 145, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "clamp_min((clamp(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 4 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 5 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 6, 1, 1) OR on() vector(0))\n*\n(\n (\n0.5 * (1.0 - 10*(sum(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) / (sum(increase(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))))) OR on() vector(0.5)\n )\n +\n (\n 0.5 * (1.0 - clamp_max(histogram_quantile(0.99, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le)),1))\n )\n), 0)", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Beacon Node Score", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Number of consensus layer P2P peers the beacon node is connected to", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#fffefe", + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "#EAB839", + "value": 20 + }, + { + "color": "green", + "value": 40 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 121 + }, + "id": 147, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "app_beacon_node_peers{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "interval": "$interval", + "legendFormat": "Peer count", + "range": true, + "refId": "A" + } + ], + "title": "Beacon node peer count", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 128 + }, + "id": 132, + "panels": [ + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 137 + }, + "id": 136, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(rate(core_validatorapi_request_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "Validator API requests rate", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 137 + }, + "id": 143, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ { - "current": { - "selected": false, - "text": "goerli", - "value": "goerli" + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(core_validatorapi_request_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le,endpoint)) ", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "Validator API request latency (90%)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Number of errors per endpoint per time window", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},cluster_network)", - "description": "", - "hide": 2, - "includeAll": false, - "label": "Cluster Network", - "multi": false, - "name": "cluster_network", - "options": [], - "query": { - "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},cluster_network)", - "refId": "StandardVariableQuery" + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 144 + }, + "id": 139, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(increase(core_validatorapi_request_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" } + ], + "title": "🚨 Validator API error count (per $interval)", + "type": "timeseries" + } + ], + "title": "Downstream Validator", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 129 + }, + "id": 149, + "panels": [], + "title": "P2P Networking", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Type of connections to each peer in the cluster during the time window:\n - 🟒 Green: Direct connection (this is the preferred)\n - 🟑 Yellow: Indirect relay connection (this is ok if only temporary)\n - πŸ”΄ Red: No connection to peer (this is a problem)\n\nSee https://docs.obol.tech/docs/charon/networking#external-p2p-network", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "green", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 130 + }, + "id": 195, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "(max(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer))\n+\n(1 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))\n+ \n(100 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer))", + "format": "time_series", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Peer Connections (🟒=direct, 🟑=relay, πŸ”΄=not connected)", + "type": "state-timeline" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.15 + }, + { + "color": "orange", + "value": 0.3 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 130 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (le,peer)) ", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "Peer ping latency (90%)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Time windows that contain ping errors per peer", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 136 + }, + "id": 166, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(increase(p2p_ping_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) > 0", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Ping Errors", + "type": "state-timeline" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 136 + }, + "id": 160, + "options": { + "calculate": false, + "cellGap": 5, + "cellValues": { + "unit": "s" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "max": 0.5, + "min": 0, + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 82 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "10.0.3", + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (le,peer))", + "interval": "", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "Peer ping latency (90%) (heatmap)", + "type": "heatmap" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 142 + }, + "id": 158, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(rate(p2p_ping_latency_secs_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) ", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "Ping Rate", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Number of new libp2p connections per peer per time window", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 142 + }, + "id": 163, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "exemplar": true, + "expr": "max(increase(p2p_peer_connection_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) > 0 ", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "New P2P Connections (per $interval)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "yellow", + "value": -0.5 + }, + { + "color": "green", + "value": -0.2 + }, + { + "color": "#EAB839", + "value": 0.2 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 148 + }, + "id": 151, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "max(app_peerinfo_clock_offset_seconds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=~\"$cluster_peer\"}) by (peer) ", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Clock Offset", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Avg Rate of P2P network bytes sent plus received per peer (bytes/second) ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + } ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 148 + }, + "id": 153, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(rate(p2p_peer_network_receive_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer) + sum(rate(p2p_peer_network_sent_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer)", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "P2P Network Avg Rate Rx+Tx", + "type": "timeseries" }, - "time": { - "from": "now-1h", - "to": "now" + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 154 + }, + "id": 156, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(p2p_relay_connections{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer) > 0 ", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Connected Relays", + "type": "state-timeline" }, - "timepicker": {}, - "timezone": "", - "title": "Charon Overview", - "uid": "charon_overview_dashboard", - "version": 1, - "weekStart": "" + { + "datasource": "${DS_PROMETHEUS}", + "description": "Total P2P network bytes sent plus received per peer per time window (bytes per interval)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 154 + }, + "id": 154, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(increase(p2p_peer_network_receive_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer) + sum(increase(p2p_peer_network_sent_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer)", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "P2P Network Bytes Rx+Tx (per $interval)", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "The number of active libp2p streams by peer or relay. Note this can be further grouped by direction and protocol.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 64 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 160 + }, + "id": 226, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_PROMETHEUS}", + "editorMode": "code", + "expr": "sum(p2p_peer_streams{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "Active libp2p streams", + "type": "timeseries" + } + ], + "refresh": "1m", + "revision": 1, + "schemaVersion": 39, + "tags": [ + "cluster-labels", + "charon" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "description": "", + "hide": 0, + "includeAll": true, + "label": "Cluster Network", + "multi": true, + "name": "cluster_network", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "mainnet", + "value": "mainnet" + }, + { + "selected": false, + "text": "goerli", + "value": "goerli" + }, + { + "selected": false, + "text": "holesky", + "value": "holesky" + }, + { + "selected": false, + "text": "sepolia", + "value": "sepolia" + }, + { + "selected": false, + "text": "gnosis", + "value": "gnosis" + } + ], + "query": "mainnet,goerli,holesky,sepolia,gnosis", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "test-cluster", + "value": "test-cluster" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(app_peer_name{cluster_network=~\"$cluster_network\"},cluster_name)", + "description": "", + "hide": 0, + "includeAll": false, + "label": "Cluster Name", + "multi": false, + "name": "cluster_name", + "options": [], + "query": { + "query": "label_values(app_peer_name{cluster_network=~\"$cluster_network\"},cluster_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "41eb9d9", + "value": "41eb9d9" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\"}, cluster_hash)", + "description": "", + "hide": 0, + "includeAll": false, + "label": "Cluster Hash", + "multi": false, + "name": "cluster_hash", + "options": [], + "query": { + "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\"}, cluster_hash)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "calm-game", + "value": "calm-game" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\"}, cluster_peer)", + "description": "", + "hide": 0, + "includeAll": false, + "label": "Cluster Peer", + "multi": false, + "name": "cluster_peer", + "options": [], + "query": { + "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\"}, cluster_peer)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "charon", + "value": "charon" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},job)", + "description": "Prometheus job (can be ignored)", + "hide": 2, + "includeAll": false, + "label": "--", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},job)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\"}, peer)", + "description": "", + "hide": 2, + "includeAll": true, + "label": "P2P Peer", + "multi": false, + "name": "peer", + "options": [], + "query": { + "query": "label_values(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\"}, peer)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "auto": true, + "auto_count": 30, + "auto_min": "1m", + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "label": "Duty", + "multi": false, + "name": "duty", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "attester", + "value": "attester" + }, + { + "selected": false, + "text": "prepare_aggregator", + "value": "prepare_aggregator" + }, + { + "selected": false, + "text": "aggregator", + "value": "aggregator" + }, + { + "selected": false, + "text": "sync_message", + "value": "sync_message" + }, + { + "selected": false, + "text": "prepare_sync_contribution", + "value": "prepare_sync_contribution" + }, + { + "selected": false, + "text": "sync_contribution", + "value": "sync_contribution" + }, + { + "selected": false, + "text": "proposer", + "value": "proposer" + }, + { + "selected": false, + "text": "randao", + "value": "randao" + }, + { + "selected": false, + "text": "exit", + "value": "exit" + }, + { + "selected": false, + "text": "builder_proposer", + "value": "builder_proposer" + }, + { + "selected": false, + "text": "builder_registration", + "value": "builder_registration" + }, + { + "selected": false, + "text": "info_sync", + "value": "info_sync" + } + ], + "query": "attester,prepare_aggregator,aggregator,sync_message,prepare_sync_contribution,sync_contribution,proposer,randao,exit,builder_proposer,builder_registration,info_sync", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Charon Overview", + "uid": "d6qujIJVk", + "version": 3, + "weekStart": "" } \ No newline at end of file diff --git a/controls/roles/update-changes/tasks/2.1.4/updates-214.yaml b/controls/roles/update-changes/tasks/2.1.4/updates-214.yaml index b9601fce6a..43c828dedc 100644 --- a/controls/roles/update-changes/tasks/2.1.4/updates-214.yaml +++ b/controls/roles/update-changes/tasks/2.1.4/updates-214.yaml @@ -32,7 +32,7 @@ - name: Set Conditional set_fact: - charonFound: "{{ charon_service_files.matched > 0 | default(false) }}" + charonFound: "{{ charon_service_files.matched | default(0) > 0 }}" - name: Set CharonService ID set_fact: diff --git a/controls/roles/update-changes/tasks/2.1.5/updates-215.yaml b/controls/roles/update-changes/tasks/2.1.5/updates-215.yaml index 216a2b937f..854cc3fa22 100644 --- a/controls/roles/update-changes/tasks/2.1.5/updates-215.yaml +++ b/controls/roles/update-changes/tasks/2.1.5/updates-215.yaml @@ -41,7 +41,7 @@ - name: Set Conditional set_fact: - charonFound: "{{ charon_service_files.matched > 0 | default(false) }}" + charonFound: "{{ charon_service_files.matched | default(0) > 0 }}" - name: Set CharonService ID set_fact: