From ed9c253f61ba78edb80b9646c77c7f51e513a1be Mon Sep 17 00:00:00 2001
From: Thomas E Lackey <telackey@bozemanpass.com>
Date: Tue, 19 Sep 2023 20:29:08 -0500
Subject: [PATCH] Simplified, improved default dashboard for
 mainnet-eth-metrics. (#536)

* Simplified, improved default dashboard for mainnet-eth-metrics.

* Add missing lighthouse prometheus config.
---
 .../grafana/etc/dashboards/eth_dashboard.json | 999 ++++++------------
 .../prometheus/etc/prometheus.yml             |   7 +
 2 files changed, 353 insertions(+), 653 deletions(-)

diff --git a/app/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json b/app/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json
index 1946ed37..623c0a56 100644
--- a/app/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json
+++ b/app/data/config/mainnet-eth-metrics/grafana/etc/dashboards/eth_dashboard.json
@@ -24,7 +24,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 1,
+  "id": 45,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -38,7 +38,7 @@
       },
       "id": 16,
       "panels": [],
-      "title": "Row title",
+      "title": "Services",
       "type": "row"
     },
     {
@@ -51,47 +51,13 @@
           "color": {
             "mode": "palette-classic"
           },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
           "mappings": [],
           "thresholds": {
-            "mode": "absolute",
+            "mode": "percentage",
             "steps": [
               {
                 "color": "green",
                 "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
               }
             ]
           }
@@ -100,23 +66,26 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 12,
+        "w": 4,
         "x": 0,
         "y": 1
       },
-      "id": 6,
+      "id": 12,
       "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
         },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
+        "textMode": "auto"
       },
+      "pluginVersion": "10.1.0",
       "targets": [
         {
           "datasource": {
@@ -124,14 +93,29 @@
             "uid": "jZUuGao4k"
           },
           "editorMode": "code",
-          "expr": "rate(nginx_http_requests_total{instance=\"keycloak-nginx-prometheus-exporter:9113\"}[$__rate_interval])",
-          "legendFormat": "__auto",
+          "expr": "chain_head_block",
+          "hide": false,
+          "legendFormat": "geth ({{instance}})",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "jZUuGao4k"
+          },
+          "editorMode": "code",
+          "expr": "beacon_head_slot",
+          "hide": false,
+          "legendFormat": "lighthouse ({{instance}})",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "nginx Requests/sec",
-      "type": "timeseries"
+      "title": "Head Positions",
+      "transformations": [
+      ],
+      "type": "stat"
     },
     {
       "datasource": {
@@ -157,6 +141,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -173,30 +158,29 @@
               "mode": "off"
             }
           },
+          "decimals": 2,
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
                 "color": "green",
                 "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
               }
             ]
-          }
+          },
+          "unit": "none"
         },
         "overrides": []
       },
       "gridPos": {
         "h": 8,
-        "w": 12,
-        "x": 12,
+        "w": 8,
+        "x": 4,
         "y": 1
       },
-      "id": 8,
+      "id": 2,
       "options": {
         "legend": {
           "calcs": [],
@@ -209,20 +193,25 @@
           "sort": "none"
         }
       },
+      "pluginVersion": "9.0.1",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "jZUuGao4k"
           },
-          "editorMode": "builder",
-          "expr": "rate(ipld_eth_server_http_count{instance=\"ipld-eth-server:8090\"}[$__rate_interval])",
-          "legendFormat": "{{method}}",
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "rate(keycloak_logins[$__rate_interval]) * 60",
+          "format": "time_series",
+          "instant": false,
+          "interval": "5m",
+          "legendFormat": "{{client_id}} ({{instance}})",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "ipld-eth-server Requests/sec",
+      "title": "Keycloak Logins/min",
       "transformations": [
         {
           "id": "seriesToColumns",
@@ -234,10 +223,28 @@
           "id": "organize",
           "options": {
             "excludeByName": {
-              "{instance=\"ipld-eth-server:8090\", job=\"ipld-eth-server\"}": true
+              "00c13eee-e48c-4c7f-872b-f0e4c1e67d7f": true,
+              "0cf1b92d-a15c-40a8-b2ae-49a758e658ee": true,
+              "13761bf2-2c0e-42fd-acc8-09a677ab179e": true,
+              "79a56c4e-0bfb-4ef0-87d4-244c4ace9b4b": true,
+              "9c41e289-92f0-456f-bd65-8f73024a9e45": true,
+              "VVQkSrRksVEKAqUmiNh4aMAVD6BcviyYV4b5LpnCrtUkRwTMoE (jackal.lan.vdb.to:8080)": true,
+              "c9718dd1-06d6-4f98-a821-9adbeb4b663b": true,
+              "nYByi9xhWdDHPpStoPLP3hjeJUzyQou8dsjK8aSipOglB3yx75 (jackal.lan.vdb.to:8080)": true,
+              "oA7XX21gTNMFRJJJdbDzjBVERbjrWamzIGhwHUAeMLrd1sXCWx (jackal.lan.vdb.to:8080)": true,
+              "oJf5jLHEqRbZg7BvSboYX1DM2T25K7XS4jgwF8NNzl3AHNWHm2 (jackal.lan.vdb.to:8080)": true
             },
             "indexByName": {},
-            "renameByName": {}
+            "renameByName": {
+              "00c13eee-e48c-4c7f-872b-f0e4c1e67d7f": "",
+              "0cf1b92d-a15c-40a8-b2ae-49a758e658ee": "rickm",
+              "13761bf2-2c0e-42fd-acc8-09a677ab179e": "telackey",
+              "2753f8be-0036-49ba-a53a-4963573fc15c": "cerc-io",
+              "79a56c4e-0bfb-4ef0-87d4-244c4ace9b4b": "",
+              "9c41e289-92f0-456f-bd65-8f73024a9e45": "",
+              "c9718dd1-06d6-4f98-a821-9adbeb4b663b": "dboreham",
+              "f5083aa7-5b5d-4164-b189-d7a559c4aad0": "infura0"
+            }
           }
         }
       ],
@@ -260,13 +267,14 @@
             "axisPlacement": "auto",
             "barAlignment": 0,
             "drawStyle": "line",
-            "fillOpacity": 0,
+            "fillOpacity": 28,
             "gradientMode": "none",
             "hideFrom": {
               "legend": false,
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 1,
             "pointSize": 5,
@@ -277,13 +285,14 @@
             "spanNulls": false,
             "stacking": {
               "group": "A",
-              "mode": "none"
+              "mode": "normal"
             },
             "thresholdsStyle": {
               "mode": "off"
             }
           },
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -303,8 +312,8 @@
       "gridPos": {
         "h": 8,
         "w": 12,
-        "x": 0,
-        "y": 9
+        "x": 12,
+        "y": 1
       },
       "id": 20,
       "options": {
@@ -326,395 +335,109 @@
             "uid": "jZUuGao4k"
           },
           "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_blockNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
+          "expr": "rate(rpc_duration_all_count[$__rate_interval])",
           "hide": false,
-          "legendFormat": "eth_blockNumber",
+          "instant": false,
+          "legendFormat": "Requests ({{instance}})",
           "range": true,
           "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_call_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_call",
-          "range": true,
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_chainId_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_chainId",
-          "range": true,
-          "refId": "C"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_estimateGas_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_estimateGas",
-          "range": true,
-          "refId": "D"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_gasPrice_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_gasPrice",
-          "range": true,
-          "refId": "E"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getBalance_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getBalance",
-          "range": true,
-          "refId": "F"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getBlockByHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getBlockByHash",
-          "range": true,
-          "refId": "G"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getBlockByNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getBlockByNumber",
-          "range": true,
-          "refId": "H"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getBlockReceipts_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getBlockReceipts",
-          "range": true,
-          "refId": "I"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getBlockTransactionCountByHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getBlockTransactionCountByHash",
-          "range": true,
-          "refId": "J"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getBlockTransactionCountByNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getBlockTransactionCountByNumber",
-          "range": true,
-          "refId": "K"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getCode_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getCode",
-          "range": true,
-          "refId": "L"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getFilterChanges_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getFilterChanges",
-          "range": true,
-          "refId": "M"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getFilterLogs_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getFilterLogs",
-          "range": true,
-          "refId": "N"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getLogs_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getLogs",
-          "range": true,
-          "refId": "O"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getStorageAt_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getStorageAt",
-          "range": true,
-          "refId": "P"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getTransactionByBlockHashAndIndex_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getTransactionByBlockHashAndIndex",
-          "range": true,
-          "refId": "Q"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getTransactionByBlockNumberAndIndex_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getTransactionByBlockNumberAndIndex",
-          "range": true,
-          "refId": "R"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getTransactionByHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getTransactionByHash",
-          "range": true,
-          "refId": "S"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getTransactionCount_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getTransactionCount",
-          "range": true,
-          "refId": "T"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getTransactionReceipt_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getTransactionReceipt",
-          "range": true,
-          "refId": "U"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getUncleCountByBlockHash_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getUncleCountByBlockHash",
-          "range": true,
-          "refId": "V"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_getUncleCountByBlockNumber_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_getUncleCountByBlockNumber",
-          "range": true,
-          "refId": "W"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_mining_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_mining_success",
-          "range": true,
-          "refId": "X"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_newBlockFilter_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_newBlockFilter",
-          "range": true,
-          "refId": "Y"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_newFilter_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_newFilter",
-          "range": true,
-          "refId": "Z"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_newPendingTransactionFilter_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_newPendingTransactionFilter",
-          "range": true,
-          "refId": "AA"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_sendRawTransaction_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_sendRawTransaction",
-          "range": true,
-          "refId": "AB"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_signTransaction_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_signTransaction",
-          "range": true,
-          "refId": "AC"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
+        }
+      ],
+      "title": "geth API Requests/sec",
+      "transformations": [
+      ],
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "jZUuGao4k"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
           },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_subscribe_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_subscribe",
-          "range": true,
-          "refId": "AD"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
           },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_syncing_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_syncing",
-          "range": true,
-          "refId": "AE"
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
         },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "jZUuGao4k"
-          },
-          "editorMode": "code",
-          "expr": "rate(rpc_duration_eth_unsubscribe_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "eth_unsubscribe",
-          "range": true,
-          "refId": "AF"
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 9
+      },
+      "id": 67,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
         },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "9.0.1",
+      "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "jZUuGao4k"
           },
           "editorMode": "code",
-          "expr": "rate(rpc_duration_statediff_writeStateDiffFor_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "statediff_writeStateDiffFor",
+          "expr": "p2p_peers{job=\"geth\"}",
+          "legendFormat": "geth ({{instance}})",
           "range": true,
-          "refId": "AG"
+          "refId": "A"
         },
         {
           "datasource": {
@@ -722,14 +445,16 @@
             "uid": "jZUuGao4k"
           },
           "editorMode": "code",
-          "expr": "rate(rpc_duration_statediff_writeStateDiffAt_success_count{instance=\"${node}:6060\"}[$__rate_interval])",
+          "expr": "network_inbound_peers{job=\"lighthouse-beacon\"}",
           "hide": false,
-          "legendFormat": "statediff_writeStateDiffAt",
+          "legendFormat": "lighthouse ({{instance}})",
           "range": true,
-          "refId": "AH"
+          "refId": "B"
         }
       ],
-      "title": "geth API Requests/sec",
+      "title": "P2P Peers",
+      "transformations": [
+      ],
       "type": "timeseries"
     },
     {
@@ -756,6 +481,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -779,9 +505,14 @@
               {
                 "color": "green",
                 "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
               }
             ]
-          }
+          },
+          "unit": "binBps"
         },
         "overrides": []
       },
@@ -791,7 +522,7 @@
         "x": 12,
         "y": 9
       },
-      "id": 2,
+      "id": 32,
       "options": {
         "legend": {
           "calcs": [],
@@ -804,7 +535,6 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "9.0.1",
       "targets": [
         {
           "datasource": {
@@ -812,106 +542,33 @@
             "uid": "jZUuGao4k"
           },
           "editorMode": "code",
-          "exemplar": false,
-          "expr": "rate(keycloak_logins{instance=\"keycloak:8080\"}[90s]) * 60",
-          "format": "time_series",
-          "instant": false,
-          "interval": "2m",
-          "legendFormat": "{{client_id}}",
+          "expr": "rate(system_disk_writebytes[$__rate_interval])",
+          "interval": "",
+          "legendFormat": "Process Write ({{instance}})",
           "range": true,
           "refId": "A"
-        }
-      ],
-      "title": "Keycloak Logins/min",
-      "transformations": [
-        {
-          "id": "seriesToColumns",
-          "options": {
-            "byField": "Time"
-          }
         },
         {
-          "id": "organize",
-          "options": {
-            "excludeByName": {
-              "00c13eee-e48c-4c7f-872b-f0e4c1e67d7f": true,
-              "0cf1b92d-a15c-40a8-b2ae-49a758e658ee": true,
-              "13761bf2-2c0e-42fd-acc8-09a677ab179e": true,
-              "79a56c4e-0bfb-4ef0-87d4-244c4ace9b4b": true,
-              "9c41e289-92f0-456f-bd65-8f73024a9e45": true,
-              "c9718dd1-06d6-4f98-a821-9adbeb4b663b": true
-            },
-            "indexByName": {},
-            "renameByName": {
-              "00c13eee-e48c-4c7f-872b-f0e4c1e67d7f": "",
-              "0cf1b92d-a15c-40a8-b2ae-49a758e658ee": "rickm",
-              "13761bf2-2c0e-42fd-acc8-09a677ab179e": "telackey",
-              "2753f8be-0036-49ba-a53a-4963573fc15c": "cerc-io",
-              "79a56c4e-0bfb-4ef0-87d4-244c4ace9b4b": "",
-              "9c41e289-92f0-456f-bd65-8f73024a9e45": "",
-              "c9718dd1-06d6-4f98-a821-9adbeb4b663b": "dboreham",
-              "f5083aa7-5b5d-4164-b189-d7a559c4aad0": "infura0"
-            }
-          }
-        }
-      ],
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "jZUuGao4k"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
+          "datasource": {
+            "type": "prometheus",
+            "uid": "jZUuGao4k"
           },
-          "mappings": [],
-          "thresholds": {
-            "mode": "percentage",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 7,
-        "w": 6,
-        "x": 0,
-        "y": 17
-      },
-      "id": 12,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "none",
-        "justifyMode": "auto",
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+          "editorMode": "code",
+          "expr": "rate(system_disk_readbytes[$__rate_interval])",
+          "hide": false,
+          "legendFormat": "Process Read ({{instance}})",
+          "range": true,
+          "refId": "B"
         },
-        "textMode": "auto"
-      },
-      "pluginVersion": "9.3.2",
-      "targets": [
         {
           "datasource": {
             "type": "prometheus",
             "uid": "jZUuGao4k"
           },
           "editorMode": "code",
-          "expr": "chain_head_block{instance=\"${node}:6060\"}",
+          "expr": "rate(eth_db_chaindata_disk_read[$__rate_interval])",
           "hide": false,
-          "legendFormat": "geth",
+          "legendFormat": "ethdb Read ({{instance}})",
           "range": true,
           "refId": "C"
         },
@@ -921,25 +578,27 @@
             "uid": "jZUuGao4k"
           },
           "editorMode": "code",
-          "expr": "statediff_service_last_statediff_height{instance=\"${node}:6060\"}",
-          "legendFormat": "statediff",
+          "expr": "rate(eth_db_chaindata_disk_write[$__rate_interval])",
+          "hide": false,
+          "legendFormat": "ethdb Write ({{instance}})",
           "range": true,
-          "refId": "A"
+          "refId": "D"
         }
       ],
-      "title": "Block Positions",
-      "type": "stat"
+      "title": "geth Disk Bytes/sec",
+      "transformations": [
+      ],
+      "type": "timeseries"
     },
     {
       "datasource": {
         "type": "prometheus",
         "uid": "jZUuGao4k"
       },
-      "description": "",
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "continuous-GrYlRd"
+            "mode": "palette-classic"
           },
           "custom": {
             "axisCenteredZero": false,
@@ -955,6 +614,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -986,40 +646,15 @@
             ]
           }
         },
-        "overrides": [
-          {
-            "__systemRef": "hideSeriesFrom",
-            "matcher": {
-              "id": "byNames",
-              "options": {
-                "mode": "exclude",
-                "names": [
-                  "Distance"
-                ],
-                "prefix": "All except:",
-                "readOnly": true
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": true
-                }
-              }
-            ]
-          }
-        ]
+        "overrides": []
       },
       "gridPos": {
-        "h": 7,
-        "w": 6,
-        "x": 6,
+        "h": 8,
+        "w": 12,
+        "x": 0,
         "y": 17
       },
-      "id": 22,
+      "id": 34,
       "options": {
         "legend": {
           "calcs": [],
@@ -1039,14 +674,27 @@
             "uid": "jZUuGao4k"
           },
           "editorMode": "code",
-          "expr": "clamp_min(chain_head_block{instance=\"${node}:6060\"} - on() statediff_service_last_statediff_height{instance=\"${node}:6060\"}, 0)",
-          "interval": "1m",
-          "legendFormat": "Distance",
+          "expr": "rate(system_disk_readcount[$__rate_interval])",
+          "legendFormat": "Process Read ({{instance}})",
           "range": true,
           "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "jZUuGao4k"
+          },
+          "editorMode": "code",
+          "expr": "rate(system_disk_writecount[$__rate_interval])",
+          "hide": false,
+          "legendFormat": "Process Write ({{instance}})",
+          "range": true,
+          "refId": "B"
         }
       ],
-      "title": "Block Distance (geth to statediff)",
+      "title": "geth Disk Operations/sec",
+      "transformations": [
+      ],
       "type": "timeseries"
     },
     {
@@ -1073,6 +721,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -1096,20 +745,23 @@
               {
                 "color": "green",
                 "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
               }
             ]
-          },
-          "unit": "none"
+          }
         },
         "overrides": []
       },
       "gridPos": {
-        "h": 7,
+        "h": 8,
         "w": 12,
         "x": 12,
         "y": 17
       },
-      "id": 10,
+      "id": 61,
       "options": {
         "legend": {
           "calcs": [],
@@ -1122,7 +774,6 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "9.0.1",
       "targets": [
         {
           "datasource": {
@@ -1130,12 +781,8 @@
             "uid": "jZUuGao4k"
           },
           "editorMode": "code",
-          "exemplar": false,
-          "expr": "increase(statediff_service_succeeded{instance=\"${node}:6060\"}[1m])",
-          "format": "time_series",
-          "instant": false,
-          "interval": "1m",
-          "legendFormat": "Succeeded",
+          "expr": "rate(eth_db_chaindata_db_get_time_count[$__rate_interval])",
+          "legendFormat": "get ({{instance}})",
           "range": true,
           "refId": "A"
         },
@@ -1145,10 +792,21 @@
             "uid": "jZUuGao4k"
           },
           "editorMode": "code",
-          "expr": "increase(statediff_service_failed{instance=\"${node}:6060\"}[1m])",
+          "expr": "rate(eth_db_chaindata_db_put_time_count[$__rate_interval])",
+          "hide": false,
+          "legendFormat": "put ({{instance}})",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "jZUuGao4k"
+          },
+          "editorMode": "code",
+          "expr": "rate(eth_db_chaindata_db_has_time_count[$__rate_interval])",
           "hide": false,
-          "interval": "1m",
-          "legendFormat": "Failed",
+          "legendFormat": "has ({{instance}})",
           "range": true,
           "refId": "B"
         },
@@ -1158,15 +816,28 @@
             "uid": "jZUuGao4k"
           },
           "editorMode": "code",
-          "expr": "statediff_service_underway{instance=\"${node}:6060\"}",
+          "expr": "rate(eth_db_chaindata_db_delete_time_count[$__rate_interval])",
           "hide": false,
-          "interval": "1s",
-          "legendFormat": "Underway",
+          "legendFormat": "delete ({{instance}})",
           "range": true,
-          "refId": "C"
+          "refId": "D"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "jZUuGao4k"
+          },
+          "editorMode": "code",
+          "expr": "rate(eth_db_chaindata_db_batch_write_time_count[$__rate_interval])",
+          "hide": false,
+          "legendFormat": "batch_write ({{instance}})",
+          "range": true,
+          "refId": "E"
         }
       ],
-      "title": "Statediff Operations",
+      "title": "geth LevelDB Operations/sec",
+      "transformations": [
+      ],
       "type": "timeseries"
     },
     {
@@ -1174,10 +845,11 @@
         "type": "prometheus",
         "uid": "jZUuGao4k"
       },
+      "description": "",
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "continuous-GrYlRd"
+            "mode": "palette-classic"
           },
           "custom": {
             "axisCenteredZero": false,
@@ -1193,10 +865,8 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
-            "lineStyle": {
-              "fill": "solid"
-            },
             "lineWidth": 1,
             "pointSize": 5,
             "scaleDistribution": {
@@ -1214,23 +884,19 @@
           },
           "mappings": [],
           "thresholds": {
-            "mode": "percentage",
+            "mode": "absolute",
             "steps": [
               {
                 "color": "green",
                 "value": null
               },
-              {
-                "color": "#EAB839",
-                "value": 50
-              },
               {
                 "color": "red",
                 "value": 80
               }
             ]
           },
-          "unit": "ms"
+          "unit": "percent"
         },
         "overrides": []
       },
@@ -1238,9 +904,9 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 24
+        "y": 25
       },
-      "id": 14,
+      "id": 63,
       "options": {
         "legend": {
           "calcs": [],
@@ -1253,7 +919,6 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "9.0.1",
       "targets": [
         {
           "datasource": {
@@ -1261,16 +926,63 @@
             "uid": "jZUuGao4k"
           },
           "editorMode": "code",
-          "exemplar": false,
-          "expr": "rate(statediff_service_total_processing_time{instance=\"${node}:6060\"}[$__rate_interval])",
-          "instant": false,
-          "interval": "1m",
-          "legendFormat": "__auto",
+          "expr": "rate(eth_db_chaindata_db_get_time_total[$__rate_interval]) / 10000000",
+          "legendFormat": "get ({{instance}})",
           "range": true,
           "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "jZUuGao4k"
+          },
+          "editorMode": "code",
+          "expr": "rate(eth_db_chaindata_db_put_time_total[$__rate_interval]) / 10000000",
+          "hide": false,
+          "legendFormat": "put ({{instance}})",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "jZUuGao4k"
+          },
+          "editorMode": "code",
+          "expr": "rate(eth_db_chaindata_db_has_time_total[$__rate_interval]) / 10000000",
+          "hide": false,
+          "legendFormat": "has ({{instance}})",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "jZUuGao4k"
+          },
+          "editorMode": "code",
+          "expr": "rate(eth_db_chaindata_db_delete_time_total[$__rate_interval]) / 10000000",
+          "hide": false,
+          "legendFormat": "delete ({{instance}})",
+          "range": true,
+          "refId": "D"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "jZUuGao4k"
+          },
+          "editorMode": "code",
+          "expr": "rate(eth_db_chaindata_db_batch_write_time_total[$__rate_interval]) / 10000000",
+          "hide": false,
+          "legendFormat": "batch_write ({{instance}})",
+          "range": true,
+          "refId": "E"
         }
       ],
-      "title": "Statediff Duty Cycle",
+      "title": "geth LevelDB Utilization",
+      "transformations": [
+      ],
       "type": "timeseries"
     },
     {
@@ -1297,6 +1009,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -1314,6 +1027,7 @@
             }
           },
           "mappings": [],
+          "min": 0,
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -1334,9 +1048,9 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 24
+        "y": 25
       },
-      "id": 4,
+      "id": 65,
       "options": {
         "legend": {
           "calcs": [],
@@ -1356,54 +1070,33 @@
             "uid": "jZUuGao4k"
           },
           "editorMode": "code",
-          "expr": "rate(statediff_service_api_requests{instance=\"${node}:6060\"}[$__rate_interval])",
-          "legendFormat": "__auto",
+          "expr": "rate(eth_db_chaindata_db_batch_item_count[$__rate_interval])",
+          "legendFormat": "batch_items ({{instance}})",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "Statediff API Requests/sec",
+      "title": "geth LevelDB Batch Items Written/sec",
+      "transformations": [
+      ],
       "type": "timeseries"
     }
   ],
-  "refresh": "5s",
-  "schemaVersion": 37,
+  "refresh": "30s",
+  "schemaVersion": 38,
   "style": "dark",
   "tags": [],
   "templating": {
-    "list": [
-      {
-        "current": {
-          "selected": true,
-          "text": "mainnet-eth-geth-1",
-          "value": "mainnet-eth-geth-1"
-        },
-        "hide": 0,
-        "includeAll": false,
-        "multi": false,
-        "name": "node",
-        "options": [
-          {
-            "selected": true,
-            "text": "mainnet-eth-geth-1",
-            "value": "mainnet-eth-geth-1"
-          }
-        ],
-        "query": "mainnet-eth-geth-1",
-        "queryValue": "",
-        "skipUrlSync": false,
-        "type": "custom"
-      }
-    ]
+    "list": []
   },
   "time": {
-    "from": "now-15m",
+    "from": "now-6h",
     "to": "now"
   },
   "timepicker": {},
   "timezone": "",
-  "title": "cerc-io overview",
-  "uid": "nT9VeZoVk",
-  "version": 2,
+  "title": "eth API Simple",
+  "uid": "b4ba9b2e-965d-4c71-8735-78b6af93156e",
+  "version": 4,
   "weekStart": ""
 }
diff --git a/app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml b/app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml
index e9f23e89..4cc74fc0 100644
--- a/app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml
+++ b/app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml
@@ -10,6 +10,13 @@ scrape_configs:
     static_configs:
     - targets: ['mainnet-eth-geth-1:6060']
 
+  # lighthouse
+  - job_name: 'lighthouse'
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+    - targets: ['mainnet-eth-lighthouse-1:5054']
+
   # keycloak
   - job_name: 'keycloak'
     scrape_interval: 5s