update agent-flow-mixin

Signed-off-by: Weifeng Wang <[email protected]>
qclaogui · Dec 1, 2023 · 6c1c0cb · 6c1c0cb
1 parent 70aa1e1
commit 6c1c0cb
Show file tree

Hide file tree

Showing 7 changed files with 107 additions and 9 deletions.
diff --git a/monitoring-mixins/agent-flow-mixin/deploy/alerts.yaml b/monitoring-mixins/agent-flow-mixin/deploy/alerts.yaml
@@ -50,3 +50,15 @@ groups:
                 max by (cluster, namespace, sha256) (agent_config_hash and on(cluster, namespace) cluster_node_info)
             ) > 1
           for: 5m
+    - name: agent_controller
+      rules:
+        - alert: SlowComponentEvaluations
+          annotations:
+            message: Flow component evaluations are taking too long.
+          expr: sum by (cluster, namespace, component_id) (rate(agent_component_evaluation_slow_seconds[10m])) > 0
+          for: 15m
+        - alert: UnhealthyComponents
+          annotations:
+            message: Unhealthy Flow components detected.
+          expr: sum(agent_component_controller_running_components{health_type!="healthy"}) > 0
+          for: 15m
diff --git a/monitoring-mixins/agent-flow-mixin/deploy/dashboards_out/agent-flow-controller.json b/monitoring-mixins/agent-flow-mixin/deploy/dashboards_out/agent-flow-controller.json
@@ -343,13 +343,39 @@
          },
          {
             "datasource": "${datasource}",
-            "description": "Detailed histogram view of how long component evaluations take.\n\nThe goal is to design your config so that evaluations take as little\ntime as possible; under 100ms is a good goal.\n",
+            "description": "The percentage of time spent evaluating 'slow' components - components that took longer than 1 minute to evaluate.\n\nIdeally, no component should take more than 1 minute to evaluate. The components displayed in this chart\nmay be a sign of a problem with the pipeline.\n",
+            "fieldConfig": {
+               "defaults": {
+                  "unit": "percentunit"
+               }
+            },
             "gridPos": {
                "h": 10,
                "w": 8,
                "x": 16,
                "y": 12
             },
+            "targets": [
+               {
+                  "datasource": "${datasource}",
+                  "expr": "sum by (component_id) (rate(agent_component_evaluation_slow_seconds{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n/ scalar(sum(rate(agent_component_evaluation_seconds_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))\n",
+                  "instant": false,
+                  "legendFormat": "{{component_id}}",
+                  "range": true
+               }
+            ],
+            "title": "Slow components evaluation times",
+            "type": "timeseries"
+         },
+         {
+            "datasource": "${datasource}",
+            "description": "Detailed histogram view of how long component evaluations take.\n\nThe goal is to design your config so that evaluations take as little\ntime as possible; under 100ms is a good goal.\n",
+            "gridPos": {
+               "h": 10,
+               "w": 8,
+               "x": 0,
+               "y": 22
+            },
             "maxDataPoints": 30,
             "options": {
                "calculate": false,
@@ -395,7 +421,7 @@
             "gridPos": {
                "h": 10,
                "w": 8,
-               "x": 0,
+               "x": 8,
                "y": 22
             },
             "maxDataPoints": 30,

diff --git a/monitoring-mixins/agent-flow-mixin/deploy/manifests/k8s-all-in-one.yaml b/monitoring-mixins/agent-flow-mixin/deploy/manifests/k8s-all-in-one.yaml
@@ -1196,13 +1196,39 @@ data:
              },
              {
                 "datasource": "${datasource}",
-                "description": "Detailed histogram view of how long component evaluations take.\n\nThe goal is to design your config so that evaluations take as little\ntime as possible; under 100ms is a good goal.\n",
+                "description": "The percentage of time spent evaluating 'slow' components - components that took longer than 1 minute to evaluate.\n\nIdeally, no component should take more than 1 minute to evaluate. The components displayed in this chart\nmay be a sign of a problem with the pipeline.\n",
+                "fieldConfig": {
+                   "defaults": {
+                      "unit": "percentunit"
+                   }
+                },
                 "gridPos": {
                    "h": 10,
                    "w": 8,
                    "x": 16,
                    "y": 12
                 },
+                "targets": [
+                   {
+                      "datasource": "${datasource}",
+                      "expr": "sum by (component_id) (rate(agent_component_evaluation_slow_seconds{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n/ scalar(sum(rate(agent_component_evaluation_seconds_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))\n",
+                      "instant": false,
+                      "legendFormat": "{{component_id}}",
+                      "range": true
+                   }
+                ],
+                "title": "Slow components evaluation times",
+                "type": "timeseries"
+             },
+             {
+                "datasource": "${datasource}",
+                "description": "Detailed histogram view of how long component evaluations take.\n\nThe goal is to design your config so that evaluations take as little\ntime as possible; under 100ms is a good goal.\n",
+                "gridPos": {
+                   "h": 10,
+                   "w": 8,
+                   "x": 0,
+                   "y": 22
+                },
                 "maxDataPoints": 30,
                 "options": {
                    "calculate": false,
@@ -1248,7 +1274,7 @@ data:
                 "gridPos": {
                    "h": 10,
                    "w": 8,
-                   "x": 0,
+                   "x": 8,
                    "y": 22
                 },
                 "maxDataPoints": 30,

diff --git a/monitoring-mixins/agent-flow-mixin/jsonnetfile.lock.json b/monitoring-mixins/agent-flow-mixin/jsonnetfile.lock.json
@@ -8,8 +8,8 @@
           "subdir": "operations/agent-flow-mixin"
         }
       },
-      "version": "5a7c16a8d6511535d815d50d974ca26f4e99524a",
-      "sum": "SoE2uqP81c1mYT38Q39mfWAAuv9ItXKO/hkIJp+hM9E="
+      "version": "5c0fa9bf270beafcabfe46f9966f5b2c2dc84fff",
+      "sum": "TyMxO1sQ2p21MKAYqufOO5HVp/UtTyMBMLmBK4kM6Pw="
     }
   ],
   "legacyImports": false

diff --git a/monitoring-mixins/agent-flow-mixin/prometheus-alert.yaml b/monitoring-mixins/agent-flow-mixin/prometheus-alert.yaml
@@ -57,4 +57,15 @@ spec:
                 max by (cluster, namespace, sha256) (agent_config_hash and on(cluster, namespace) cluster_node_info)
             ) > 1
           for: 5m
-
+    - name: agent_controller
+      rules:
+        - alert: SlowComponentEvaluations
+          annotations:
+            message: Flow component evaluations are taking too long.
+          expr: sum by (cluster, namespace, component_id) (rate(agent_component_evaluation_slow_seconds[10m])) > 0
+          for: 15m
+        - alert: UnhealthyComponents
+          annotations:
+            message: Unhealthy Flow components detected.
+          expr: sum(agent_component_controller_running_components{health_type!="healthy"}) > 0
+          for: 15m
diff --git a/...t-flow-mixin/vendor/github.com/grafana/agent/operations/agent-flow-mixin/alerts.libsonnet b/...t-flow-mixin/vendor/github.com/grafana/agent/operations/agent-flow-mixin/alerts.libsonnet
diff --git a/...ndor/github.com/grafana/agent/operations/agent-flow-mixin/dashboards/controller.libsonnet b/...ndor/github.com/grafana/agent/operations/agent-flow-mixin/dashboards/controller.libsonnet