fix: Test grafana dashboards in hubble deploy

Makefile

@@ -537,3 +537,4 @@ quick-deploy-hubble:
 .PHONY: simplify-dashboards
 simplify-dashboards:
 	cd deploy/legacy/grafana/dashboards && go test . -tags=dashboard,simplifydashboard -v && cd $(REPO_ROOT)
+	cd deploy/hubble/grafana/dashboards && go test . -tags=dashboard,simplifydashboard -v && cd $(REPO_ROOT)

deploy/hubble/grafana/dashboards/clusters.json

@@ -1,41 +1,41 @@
 {
-  "__inputs": [],
   "__elements": {},
+  "__inputs": [],
   "__requires": [
     {
-      "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
+      "type": "grafana",
       "version": "9.5.15"
     },
     {
-      "type": "datasource",
       "id": "prometheus",
       "name": "Prometheus",
+      "type": "datasource",
       "version": "1.0.0"
     },
     {
-      "type": "panel",
       "id": "stat",
       "name": "Stat",
+      "type": "panel",
       "version": ""
     },
     {
-      "type": "panel",
       "id": "table",
       "name": "Table",
+      "type": "panel",
       "version": ""
     },
     {
-      "type": "panel",
       "id": "text",
       "name": "Text",
+      "type": "panel",
       "version": ""
     },
     {
-      "type": "panel",
       "id": "timeseries",
       "name": "Time series",
+      "type": "panel",
       "version": ""
     }
   ],

deploy/hubble/grafana/dashboards/dns.json

@@ -1,29 +1,29 @@
 {
-  "__inputs": [],
   "__elements": {},
+  "__inputs": [],
   "__requires": [
     {
-      "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
+      "type": "grafana",
       "version": "9.5.15"
     },
     {
-      "type": "datasource",
       "id": "prometheus",
       "name": "Prometheus",
+      "type": "datasource",
       "version": "1.0.0"
     },
     {
-      "type": "panel",
       "id": "table",
       "name": "Table",
+      "type": "panel",
       "version": ""
     },
     {
-      "type": "panel",
       "id": "timeseries",
       "name": "Time series",
+      "type": "panel",
       "version": ""
     }
   ],

deploy/hubble/grafana/dashboards/pod-flows-namespace.json

@@ -1,41 +1,41 @@
 {
-  "__inputs": [],
   "__elements": {},
+  "__inputs": [],
   "__requires": [
     {
-      "type": "panel",
       "id": "bargauge",
       "name": "Bar gauge",
+      "type": "panel",
       "version": ""
     },
     {
-      "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
+      "type": "grafana",
       "version": "9.5.15"
     },
     {
-      "type": "panel",
       "id": "heatmap",
       "name": "Heatmap",
+      "type": "panel",
       "version": ""
     },
     {
-      "type": "datasource",
       "id": "prometheus",
       "name": "Prometheus",
+      "type": "datasource",
       "version": "1.0.0"
     },
     {
-      "type": "panel",
       "id": "stat",
       "name": "Stat",
+      "type": "panel",
       "version": ""
     },
     {
-      "type": "panel",
       "id": "timeseries",
       "name": "Time series",
+      "type": "panel",
       "version": ""
     }
   ],

deploy/hubble/grafana/dashboards/pod-flows-workload.json

@@ -1,35 +1,35 @@
 {
-  "__inputs": [],
   "__elements": {},
+  "__inputs": [],
   "__requires": [
     {
-      "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
+      "type": "grafana",
       "version": "9.5.15"
     },
     {
-      "type": "panel",
       "id": "heatmap",
       "name": "Heatmap",
+      "type": "panel",
       "version": ""
     },
     {
-      "type": "datasource",
       "id": "prometheus",
       "name": "Prometheus",
+      "type": "datasource",
       "version": "1.0.0"
     },
     {
-      "type": "panel",
       "id": "stat",
       "name": "Stat",
+      "type": "panel",
       "version": ""
     },
     {
-      "type": "panel",
       "id": "timeseries",
       "name": "Time series",
+      "type": "panel",
       "version": ""
     }
   ],

deploy/legacy/grafana/dashboards/simplify-grafana-overwrite_test.go

@@ -0,0 +1,27 @@
+//go:build dashboard && simplifydashboard
+
+package dashboard
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// TestOverwriteDashboards simplifies and overwrites Grafana dashboards in this folder.
+func TestOverwriteDashboards(t *testing.T) {
+	// get all json's in this folder
+	files, err := os.ReadDir("./")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, file := range files {
+		if filepath.Ext(file.Name()) == ".json" {
+			t.Logf("simplifying/overwriting dashboard: %s", file.Name())
+
+			sourcePath := file.Name()
+			_ = SimplifyGrafana(sourcePath, true)
+		}
+	}
+}

deploy/legacy/grafana/dashboards/simplify-grafana.go

@@ -0,0 +1,126 @@
+//go:build dashboard
+
+package dashboard
+
+import (
+	"bytes"
+	"encoding/json"
+	"log"
+	"os"
+)
+
+// SimplifyGrafana parses a json file representing a Grafana dashboard and overwrites it with a simplified version.
+// It removes some unnecessary fields and changes some values so that the dashboard can be used by anyone.
+func SimplifyGrafana(filename string, overwrite bool) map[string]interface{} {
+	dashboard := ParseDashboard(filename)
+
+	// remove unnecessary top-level fields
+	delete(dashboard, "weekStart")
+	delete(dashboard, "fiscalYearStartMonth")
+	delete(dashboard, "graphTooltip")
+	delete(dashboard, "annotations")
+
+	// set empty __inputs
+	if _, ok := dashboard["__inputs"]; ok {
+		dashboard["__inputs"] = []interface{}{}
+	}
+
+	// set current variables to be empty
+	if _, ok := dashboard["templating"]; ok {
+		templating := dashboard["templating"].(map[string]interface{})
+		if _, ok := templating["list"]; ok {
+			list := templating["list"].([]interface{})
+			for _, item := range list {
+				if item, ok := item.(map[string]interface{}); ok {
+					item["current"] = map[string]interface{}{}
+				}
+			}
+		}
+	}
+
+	// remove any pluginVersion
+	removeFieldAnywhere(dashboard, "pluginVersion")
+
+	// change all datasource.uid to "${datasource}"
+	replaceDatasource(dashboard)
+
+	if !overwrite {
+		return dashboard
+	}
+
+	// overwrite the file with the simplified version
+	simplifiedData, err := JSONMarhsalIndent(dashboard, "", "  ")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = os.WriteFile(filename, simplifiedData, 0o644)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return dashboard
+}
+
+func ParseDashboard(filename string) map[string]interface{} {
+	data, err := os.ReadFile(filename)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var dashboard map[string]interface{}
+	err = json.Unmarshal(data, &dashboard)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return dashboard
+}
+
+func removeFieldAnywhere(data map[string]interface{}, field string) {
+	if _, ok := data[field]; ok {
+		delete(data, field)
+	}
+
+	for _, value := range data {
+		if m, ok := value.(map[string]interface{}); ok {
+			removeFieldAnywhere(m, field)
+		} else if l, ok := value.([]interface{}); ok {
+			for _, item := range l {
+				if m, ok := item.(map[string]interface{}); ok {
+					removeFieldAnywhere(m, field)
+				}
+			}
+		}
+	}
+}
+
+func replaceDatasource(data map[string]interface{}) {
+	if datasource, ok := data["datasource"].(map[string]interface{}); ok {
+		if _, ok := datasource["uid"].(string); ok {
+			datasource["uid"] = "${datasource}"
+		}
+	}
+
+	for _, value := range data {
+		if m, ok := value.(map[string]interface{}); ok {
+			replaceDatasource(m)
+		} else if l, ok := value.([]interface{}); ok {
+			for _, item := range l {
+				if m, ok := item.(map[string]interface{}); ok {
+					replaceDatasource(m)
+				}
+			}
+		}
+	}
+}
+
+// JSONMarhsalIndent is json.MarshalIndent without HTML escaping (e.g. converting > to \u003e)
+func JSONMarhsalIndent(t interface{}, prefix, indent string) ([]byte, error) {
+	buffer := &bytes.Buffer{}
+	encoder := json.NewEncoder(buffer)
+	encoder.SetEscapeHTML(false)
+	encoder.SetIndent(prefix, indent)
+	err := encoder.Encode(t)
+	return buffer.Bytes(), err
+}

deploy/legacy/grafana/dashboards/simplify-grafana_test.go

@@ -0,0 +1,33 @@
+//go:build dashboard && !simplifydashboard
+
+package dashboard
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+)
+
+// TestDashboardsAreSimplified ensures that all dashboards are simplified
+func TestDashboardsAreSimplified(t *testing.T) {
+	// get all json's in this folder
+	files, err := os.ReadDir("./")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, file := range files {
+		if filepath.Ext(file.Name()) == ".json" {
+			t.Logf("verifying that dashboard is simplified: %s", file.Name())
+
+			sourcePath := file.Name()
+			simplified := SimplifyGrafana(sourcePath, false)
+			original := ParseDashboard(sourcePath)
+
+			if !reflect.DeepEqual(simplified, original) {
+				t.Errorf("ERROR: dashboard has not been simplified. Please run: make simplify-dashboards")
+			}
+		}
+	}
+}