Namespace-aware Grafana dashboards (#716)

The Grafana dashboards key off of deployment, but had no awareness of namespaces, causing incorrect metrics aggregation and display. This change makes the Grafana dashboards key off of namespaces, and also modifies the Grafana links in the Conduit dashboard to link to namespace+deployment. Fixes #704 Part of #420 Signed-off-by: Andrew Seigner <[email protected]>
linkerd · Apr 6, 2018 · b6bcdcc · b6bcdcc
1 parent baa4d10
commit b6bcdcc
Show file tree

Hide file tree

Showing 3 changed files with 181 additions and 59 deletions.
diff --git a/grafana/dashboards/deployment.json b/grafana/dashboards/deployment.json
@@ -16,7 +16,7 @@
   "gnetId": null,
   "graphTooltip": 1,
   "id": null,
-  "iteration": 1522197746099,
+  "iteration": 1523047156373,
   "links": [],
   "panels": [
     {
@@ -96,7 +96,7 @@
       "tableColumn": "",
       "targets": [
         {
-          "expr": "sum(irate(request_total{deployment=\"$deployment\"}[20s]))",
+          "expr": "sum(irate(request_total{namespace=\"$namespace\", deployment=\"$deployment\", direction=\"inbound\"}[20s]))",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -180,7 +180,7 @@
       "tableColumn": "",
       "targets": [
         {
-          "expr": "(sum(irate(response_total{status_code=\"200\", deployment=\"$deployment\"}[20s])) + sum(irate(response_total{grpc_status_code=\"0\", deployment=\"$deployment\"}[20s]))) / sum(irate(response_total{deployment=\"$deployment\"}[20s]))",
+          "expr": "sum(irate(response_total{classification=\"success\", namespace=\"$namespace\", deployment=\"$deployment\", direction=\"inbound\"}[20s])) / sum(irate(response_total{namespace=\"$namespace\", deployment=\"$deployment\", direction=\"inbound\"}[20s]))",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -264,7 +264,7 @@
       "tableColumn": "",
       "targets": [
         {
-          "expr": "histogram_quantile(0.5, sum(rate(response_latency_ms_bucket{deployment=\"$deployment\"}[20s])) by (le))",
+          "expr": "histogram_quantile(0.5, sum(rate(response_latency_ms_bucket{namespace=\"$namespace\", deployment=\"$deployment\", direction=\"inbound\"}[20s])) by (le))",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -348,7 +348,7 @@
       "tableColumn": "",
       "targets": [
         {
-          "expr": "histogram_quantile(0.95, sum(rate(response_latency_ms_bucket{deployment=\"$deployment\"}[20s])) by (le))",
+          "expr": "histogram_quantile(0.95, sum(rate(response_latency_ms_bucket{namespace=\"$namespace\", deployment=\"$deployment\", direction=\"inbound\"}[20s])) by (le))",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -432,7 +432,7 @@
       "tableColumn": "",
       "targets": [
         {
-          "expr": "histogram_quantile(0.99, sum(rate(response_latency_ms_bucket{deployment=\"$deployment\"}[20s])) by (le))",
+          "expr": "histogram_quantile(0.99, sum(rate(response_latency_ms_bucket{namespace=\"$namespace\", deployment=\"$deployment\", direction=\"inbound\"}[20s])) by (le))",
           "format": "time_series",
           "instant": false,
           "intervalFactor": 1,
@@ -516,7 +516,7 @@
       "tableColumn": "",
       "targets": [
         {
-          "expr": "count(request_total{dst_deployment=\"$deployment\"})",
+          "expr": "count(request_total{dst_namespace=\"$namespace\", dst_deployment=\"$deployment\", direction=\"outbound\"})",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "",
@@ -598,7 +598,7 @@
       "tableColumn": "",
       "targets": [
         {
-          "expr": "count(request_total{deployment=\"$deployment\", dst_deployment=~\"$outbound\"})",
+          "expr": "count(request_total{namespace=\"$namespace\", deployment=\"$deployment\", dst_deployment=~\"$outbound\", direction=\"outbound\"})",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "",
@@ -656,7 +656,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(irate(request_total{deployment=\"$deployment\", direction=\"inbound\"}[20s]))",
+          "expr": "sum(irate(request_total{namespace=\"$namespace\", deployment=\"$deployment\", direction=\"inbound\"}[20s]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "inbound",
@@ -736,7 +736,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(irate(request_total{deployment=\"$deployment\", direction=\"outbound\"}[20s]))",
+          "expr": "sum(irate(request_total{namespace=\"$namespace\", deployment=\"$deployment\", direction=\"outbound\"}[20s]))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "outbound",
@@ -809,7 +809,7 @@
       "type": "row"
     },
     {
-      "content": "<div>\n  <img src=\"https://conduit.io/favicon.png\" style=\"baseline; height:30px;\"/>&nbsp;\n  <a href=\"./dashboard/db/conduit-deployment?var-deployment=$inbound\">\n    <span style=\"font-size: 15px; border-image:none\">$inbound</span>\n  </a>\n</div>",
+      "content": "<div>\n  <img src=\"https://conduit.io/favicon.png\" style=\"baseline; height:30px;\"/>&nbsp;\n  <span style=\"font-size: 15px; border-image:none\">$inbound</span>\n</div>",
       "gridPos": {
         "h": 2,
         "w": 24,
@@ -860,7 +860,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(irate(request_total{deployment=\"$inbound\", dst_deployment=\"$deployment\"}[20s])) by (deployment)",
+          "expr": "sum(irate(request_total{deployment=\"$inbound\", dst_namespace=\"$namespace\", dst_deployment=\"$deployment\", direction=\"outbound\"}[20s])) by (deployment)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{deployment}}",
@@ -940,7 +940,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "(sum(irate(response_total{grpc_status_code=\"0\", deployment=\"$inbound\", dst_deployment=\"$deployment\"}[20s])) by (deployment) +sum(irate(response_total{status_code=\"200\", deployment=\"$inbound\", dst_deployment=\"$deployment\"}[20s])) by (deployment)) / sum(irate(response_total{deployment=\"$inbound\", dst_deployment=\"$deployment\"}[20s])) by (deployment)",
+          "expr": "sum(irate(response_total{classification=\"success\", deployment=\"$inbound\", dst_namespace=\"$namespace\", dst_deployment=\"$deployment\", direction=\"outbound\"}[20s])) by (deployment) / sum(irate(response_total{deployment=\"$inbound\", dst_namespace=\"$namespace\", dst_deployment=\"$deployment\", direction=\"outbound\"}[20s])) by (deployment)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{deployment}}",
@@ -1020,7 +1020,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "histogram_quantile(0.5, sum(rate(response_latency_ms_bucket{deployment=\"$inbound\", dst_deployment=\"$deployment\"}[20s])) by (le, deployment))",
+          "expr": "histogram_quantile(0.5, sum(rate(response_latency_ms_bucket{deployment=\"$inbound\", dst_namespace=\"$namespace\", dst_deployment=\"$deployment\", direction=\"outbound\"}[20s])) by (le, deployment))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{deployment}} P50",
@@ -1100,7 +1100,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "histogram_quantile(0.95, sum(rate(response_latency_ms_bucket{deployment=\"$inbound\", dst_deployment=\"$deployment\"}[20s])) by (le, deployment))",
+          "expr": "histogram_quantile(0.95, sum(rate(response_latency_ms_bucket{deployment=\"$inbound\", dst_namespace=\"$namespace\", dst_deployment=\"$deployment\", direction=\"outbound\"}[20s])) by (le, deployment))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{deployment}} P95",
@@ -1180,7 +1180,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "histogram_quantile(0.99, sum(rate(response_latency_ms_bucket{deployment=\"$inbound\", dst_deployment=\"$deployment\"}[20s])) by (le, deployment))",
+          "expr": "histogram_quantile(0.99, sum(rate(response_latency_ms_bucket{deployment=\"$inbound\", dst_namespace=\"$namespace\", dst_deployment=\"$deployment\", direction=\"outbound\"}[20s])) by (le, deployment))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{deployment}} P99",
@@ -1267,7 +1267,7 @@
       "type": "row"
     },
     {
-      "content": "<div>\n  <img src=\"https://conduit.io/favicon.png\" style=\"baseline; height:30px;\"/>&nbsp;\n  <a href=\"./dashboard/db/conduit-deployment?var-deployment=$outbound\">\n    <span style=\"font-size: 15px; border-image:none\">$outbound</span>\n  </a>\n</div>",
+      "content": "<div>\n  <img src=\"https://conduit.io/favicon.png\" style=\"baseline; height:30px;\"/>&nbsp;\n  <span style=\"font-size: 15px; border-image:none\">$outbound</span>\n</div>",
       "gridPos": {
         "h": 2,
         "w": 24,
@@ -1318,7 +1318,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(irate(request_total{deployment=\"$deployment\", dst_deployment=\"$outbound\"}[20s])) by (dst_deployment)",
+          "expr": "sum(irate(request_total{namespace=\"$namespace\", deployment=\"$deployment\", dst_deployment=\"$outbound\", direction=\"outbound\"}[20s])) by (dst_deployment)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{dst_deployment}}",
@@ -1398,7 +1398,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "(sum(irate(response_total{status_code=\"200\", deployment=\"$deployment\", dst_deployment=\"$outbound\"}[20s])) by (dst_deployment) + sum(irate(response_total{grpc_status_code=\"0\", deployment=\"$deployment\", dst_deployment=\"$outbound\"}[20s])) by (dst_deployment)) / sum(irate(response_total{deployment=\"$deployment\", dst_deployment=\"$outbound\"}[20s])) by (dst_deployment)",
+          "expr": "sum(irate(response_total{classification=\"success\", namespace=\"$namespace\", deployment=\"$deployment\", dst_deployment=\"$outbound\", direction=\"outbound\"}[20s])) by (dst_deployment) / sum(irate(response_total{namespace=\"$namespace\", deployment=\"$deployment\", dst_deployment=\"$outbound\", direction=\"outbound\"}[20s])) by (dst_deployment)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{dst_deployment}}",
@@ -1478,7 +1478,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "histogram_quantile(0.5, sum(rate(response_latency_ms_bucket{deployment=\"$deployment\", dst_deployment=\"$outbound\"}[20s])) by (le, dst_deployment))",
+          "expr": "histogram_quantile(0.5, sum(rate(response_latency_ms_bucket{namespace=\"$namespace\", deployment=\"$deployment\", dst_deployment=\"$outbound\", direction=\"outbound\"}[20s])) by (le, dst_deployment))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{dst_deployment}} P50",
@@ -1558,7 +1558,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "histogram_quantile(0.95, sum(rate(response_latency_ms_bucket{deployment=\"$deployment\", dst_deployment=\"$outbound\"}[20s])) by (le, dst_deployment))",
+          "expr": "histogram_quantile(0.95, sum(rate(response_latency_ms_bucket{namespace=\"$namespace\", deployment=\"$deployment\", dst_deployment=\"$outbound\", direction=\"outbound\"}[20s])) by (le, dst_deployment))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{dst_deployment}} P95",
@@ -1638,7 +1638,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "histogram_quantile(0.99, sum(rate(response_latency_ms_bucket{deployment=\"$deployment\", dst_deployment=\"$outbound\"}[20s])) by (le, dst_deployment))",
+          "expr": "histogram_quantile(0.99, sum(rate(response_latency_ms_bucket{namespace=\"$namespace\", deployment=\"$deployment\", dst_deployment=\"$outbound\", direction=\"outbound\"}[20s])) by (le, dst_deployment))",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{dst_deployment}} P99",
@@ -1688,6 +1688,26 @@
   "tags": [],
   "templating": {
     "list": [
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "prometheus",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Namespace",
+        "multi": false,
+        "name": "namespace",
+        "options": [],
+        "query": "label_values(request_total{conduit_io_control_plane_component=\"\"}, namespace)",
+        "refresh": 2,
+        "regex": "",
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
       {
         "allValue": null,
         "current": {},
@@ -1698,7 +1718,7 @@
         "multi": false,
         "name": "deployment",
         "options": [],
-        "query": "label_values(request_total{conduit_io_control_plane_component=\"\"}, deployment)",
+        "query": "label_values(request_total{namespace=\"$namespace\", conduit_io_control_plane_component=\"\"}, deployment)",
         "refresh": 2,
         "regex": "",
         "sort": 1,
@@ -1718,7 +1738,7 @@
         "multi": false,
         "name": "inbound",
         "options": [],
-        "query": "label_values(request_total{dst_deployment=\"$deployment\"}, deployment)",
+        "query": "label_values(request_total{dst_namespace=\"$namespace\", dst_deployment=\"$deployment\"}, deployment)",
         "refresh": 2,
         "regex": "",
         "sort": 1,
@@ -1738,7 +1758,7 @@
         "multi": false,
         "name": "outbound",
         "options": [],
-        "query": "label_values(request_total{deployment=\"$deployment\"}, dst_deployment)",
+        "query": "label_values(request_total{namespace=\"$namespace\", deployment=\"$deployment\"}, dst_deployment)",
         "refresh": 2,
         "regex": "",
         "sort": 1,