From f0a136a85beb5bc495843b34a7040bc839ba1ad1 Mon Sep 17 00:00:00 2001
From: williamlardier <william.lardier@scality.com>
Date: Fri, 13 Sep 2024 15:09:27 +0200
Subject: [PATCH] Update oplog populator dashboard

- Add a timeserie on the rate of skipped connector reconfiguration

Issue: BB-601
---
 monitoring/oplog-populator/dashboard.json | 107 +++++++++++++++++++---
 monitoring/oplog-populator/dashboard.py   |  15 ++-
 2 files changed, 110 insertions(+), 12 deletions(-)

diff --git a/monitoring/oplog-populator/dashboard.json b/monitoring/oplog-populator/dashboard.json
index d3811e5b5..57c435d28 100644
--- a/monitoring/oplog-populator/dashboard.json
+++ b/monitoring/oplog-populator/dashboard.json
@@ -577,7 +577,7 @@
       },
       "gridPos": {
         "h": 7,
-        "w": 12,
+        "w": 8,
         "x": 0,
         "y": 4
       },
@@ -662,8 +662,8 @@
       },
       "gridPos": {
         "h": 7,
-        "w": 12,
-        "x": 12,
+        "w": 8,
+        "x": 8,
         "y": 4
       },
       "hideTimeOverride": false,
@@ -701,6 +701,91 @@
       "transparent": false,
       "type": "timeseries"
     },
+    {
+      "datasource": "${DS_PROMETHEUS}",
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "opacity",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "log": 2,
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": true,
+            "stacking": {},
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": []
+          },
+          "unit": ""
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 16,
+        "y": 4
+      },
+      "hideTimeOverride": false,
+      "id": 10,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "hidden",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "datasource": null,
+          "expr": "sum (rate(s3_oplog_populator_connectors_reconfiguration_skipped{job=\"${oplog_populator_job}\", namespace=\"${namespace}\", connector=~\"${connector}\"}[$__rate_interval]))",
+          "format": "time_series",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "metric": "",
+          "refId": "",
+          "step": 10,
+          "target": ""
+        }
+      ],
+      "title": "Skipped Connector Reconfigurations",
+      "transformations": [],
+      "transparent": false,
+      "type": "timeseries"
+    },
     {
       "collapsed": false,
       "editable": true,
@@ -720,7 +805,7 @@
         "y": 11
       },
       "hideTimeOverride": false,
-      "id": 10,
+      "id": 11,
       "links": [],
       "maxDataPoints": 100,
       "panels": [],
@@ -757,7 +842,7 @@
         "y": 12
       },
       "hideTimeOverride": false,
-      "id": 11,
+      "id": 12,
       "links": [],
       "maxDataPoints": 100,
       "options": {
@@ -854,7 +939,7 @@
         "y": 12
       },
       "hideTimeOverride": false,
-      "id": 12,
+      "id": 13,
       "links": [],
       "maxDataPoints": 100,
       "options": {
@@ -940,7 +1025,7 @@
         "y": 19
       },
       "hideTimeOverride": false,
-      "id": 13,
+      "id": 14,
       "links": [],
       "maxDataPoints": 100,
       "options": {
@@ -1026,7 +1111,7 @@
         "y": 19
       },
       "hideTimeOverride": false,
-      "id": 14,
+      "id": 15,
       "links": [],
       "maxDataPoints": 100,
       "options": {
@@ -1079,7 +1164,7 @@
         "y": 26
       },
       "hideTimeOverride": false,
-      "id": 15,
+      "id": 16,
       "links": [],
       "maxDataPoints": 100,
       "panels": [],
@@ -1126,7 +1211,7 @@
       "hideTimeOverride": false,
       "hideZeroBuckets": true,
       "highlightCards": true,
-      "id": 16,
+      "id": 17,
       "legend": {
         "show": false
       },
@@ -1210,7 +1295,7 @@
       "hideTimeOverride": false,
       "hideZeroBuckets": true,
       "highlightCards": true,
-      "id": 17,
+      "id": 18,
       "legend": {
         "show": false
       },
diff --git a/monitoring/oplog-populator/dashboard.py b/monitoring/oplog-populator/dashboard.py
index beed6e192..99831b31d 100644
--- a/monitoring/oplog-populator/dashboard.py
+++ b/monitoring/oplog-populator/dashboard.py
@@ -35,7 +35,7 @@ class Metrics:
         'job="${oplog_populator_job}"',
     )
 
-    CONNECTOR_CONFIG, CONNECTOR_CONFIG_APPLIED, BUCKETS, REQUEST_SIZE, PIPELINE_SIZE = [
+    CONNECTOR_CONFIG, CONNECTOR_CONFIG_APPLIED, BUCKETS, REQUEST_SIZE, PIPELINE_SIZE, CONNECTOR_CONFIG_SKIPPED = [
         metrics.CounterMetric(
             name, 'connector', *extraLabels,
             job="${oplog_populator_job}", namespace="${namespace}"
@@ -47,6 +47,7 @@ class Metrics:
             's3_oplog_populator_connector_buckets': [],
             's3_oplog_populator_connector_request_bytes_total': [],
             's3_oplog_populator_connector_pipeline_bytes': [],
+            's3_oplog_populator_connectors_reconfiguration_skipped': ['opType'],
         }.items()
     ]
 
@@ -182,6 +183,17 @@ class Metrics:
         expr='sum (rate(' + Metrics.CONNECTOR_CONFIG_APPLIED('success="false"') + '))',
     )])
 
+reconfig_skipped = TimeSeries(
+    title='Skipped Connector Reconfigurations',
+    dataSource='${DS_PROMETHEUS}',
+    gradientMode='opacity',
+    lineInterpolation='smooth',
+    legendDisplayMode='hidden',
+    fillOpacity=10,
+    spanNulls=True,
+    targets=[Target(
+        expr='sum (rate(' + Metrics.CONNECTOR_CONFIG_SKIPPED() + '))',
+    )])
 
 bucket_distribution = PieChart(
     title='Bucket Distribution',
@@ -340,6 +352,7 @@ class Metrics:
             layout.row([
                 config_applied_success,
                 config_applied_failure,
+                reconfig_skipped,
             ], 7),
             RowPanel(title="Bucket Metrics"),
             layout.row([