Addressing multiple dashboard issues: deps loading once, field conversion, etc.

libbeat/dashboards/kibana_loader.go

@@ -44,6 +44,8 @@ type KibanaLoader struct {
 	hostname      string
 	msgOutputter  MessageOutputter
 	defaultLogger *logp.Logger
+
+	loadedAssets map[string]bool
 }
 
 // NewKibanaLoader creates a new loader to load Kibana files
@@ -65,6 +67,7 @@ func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig *
 		hostname:      hostname,
 		msgOutputter:  msgOutputter,
 		defaultLogger: logp.NewLogger("dashboards"),
+		loadedAssets:  make(map[string]bool, 0),
 	}
 
 	version := client.GetVersion()
@@ -150,24 +153,18 @@ func (loader KibanaLoader) ImportDashboard(file string) error {
 		return fmt.Errorf("fail to read dashboard from file %s: %v", file, err)
 	}
 
-	content = ReplaceIndexInDashboardObject(loader.config.Index, content)
-
-	content = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content)
+	content = loader.formatDashboardAssets(content)
 
-	err = loader.importReferences(file, content)
+	dashboardWithReferences, err := loader.addReferences(file, content)
 	if err != nil {
-		return fmt.Errorf("error loading references of dashboard: %+v", err)
+		return fmt.Errorf("error getting references of dashboard: %+v", err)
 	}
 
-	var obj common.MapStr
-	err = json.Unmarshal(content, &obj)
-	if err != nil {
-		return err
-	}
-
-	if err := loader.client.ImportMultiPartFormFile(importAPI, params, correctExtension(file), obj.String()); err != nil {
+	if err := loader.client.ImportMultiPartFormFile(importAPI, params, correctExtension(file), dashboardWithReferences); err != nil {
 		return fmt.Errorf("error dashboard asset: %+v", err)
 	}
+
+	loader.loadedAssets[file] = true
 	return nil
 }
 
@@ -179,25 +176,52 @@ type dashboardReference struct {
 	Type string `json:"type"`
 }
 
-func (loader KibanaLoader) importReferences(path string, dashboard []byte) error {
+func (loader KibanaLoader) addReferences(path string, dashboard []byte) (string, error) {
 	var d dashboardObj
 	err := json.Unmarshal(dashboard, &d)
 	if err != nil {
-		return fmt.Errorf("failed to parse dashboard references: %+v", err)
+		return "", fmt.Errorf("failed to parse dashboard references: %+v", err)
 	}
 
 	base := filepath.Dir(path)
+	var result string
 	for _, ref := range d.References {
 		if ref.Type == "index-pattern" {
 			continue
 		}
 		referencePath := filepath.Join(base, "..", ref.Type, ref.ID+".json")
-		err := loader.ImportDashboard(referencePath)
+		if _, ok := loader.loadedAssets[referencePath]; ok {
+			continue
+		}
+		refContents, err := ioutil.ReadFile(referencePath)
+		if err != nil {
+			return "", fmt.Errorf("fail to read referenced asset from file %s: %v", referencePath, err)
+		}
+		refContents = loader.formatDashboardAssets(refContents)
+		refContentsWithReferences, err := loader.addReferences(referencePath, refContents)
 		if err != nil {
-			return fmt.Errorf("error loading reference of %s: %s %s: %+v", path, ref.Type, ref.ID, err)
+			return "", fmt.Errorf("failed to get references of %s: %+v", referencePath, err)
 		}
+
+		result += refContentsWithReferences
+		loader.loadedAssets[referencePath] = true
 	}
-	return nil
+
+	var res common.MapStr
+	err = json.Unmarshal(dashboard, &res)
+	if err != nil {
+		return "", fmt.Errorf("failed to convert asset: %+v", err)
+	}
+	result += res.String() + "\n"
+
+	return result, nil
+}
+
+func (loader KibanaLoader) formatDashboardAssets(content []byte) []byte {
+	content = ReplaceIndexInDashboardObject(loader.config.Index, content)
+	content = EncodeJSONObjects(content)
+	content = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content)
+	return content
 }
 
 func correctExtension(file string) string {

libbeat/dashboards/modify_json.go

@@ -18,11 +18,9 @@
 package dashboards
 
 import (
-	"bufio"
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
 
 	"github.com/pkg/errors"
 
@@ -40,6 +38,7 @@ type JSONObjectAttribute struct {
 	KibanaSavedObjectMeta map[string]interface{} `json:"kibanaSavedObjectMeta"`
 	Title                 string                 `json:"title"`
 	Type                  string                 `json:"type"`
+	UiStateJSON           map[string]interface{} `json:"uiStateJSON"`
 }
 
 // JSONObject is an Object with a given JSON attribute
@@ -175,37 +174,21 @@ func ReplaceIndexInDashboardObject(index string, content []byte) []byte {
 		return content
 	}
 
-	var result []byte
-	r := bufio.NewReader(bytes.NewReader(content))
-	for {
-		line, err := r.ReadBytes('\n')
-		if err != nil {
-			if err == io.EOF {
-				return append(result, replaceInNDJSON(logger, index, line)...)
-			}
-			logger.Error("Error reading bytes from raw dashboard object: %+v", err)
-			return content
-		}
-		result = append(result, replaceInNDJSON(logger, index, line)...)
-	}
-}
-
-func replaceInNDJSON(logger *logp.Logger, index string, line []byte) []byte {
-	if len(bytes.TrimSpace(line)) == 0 {
-		return line
+	if len(bytes.TrimSpace(content)) == 0 {
+		return content
 	}
 
 	objectMap := make(map[string]interface{}, 0)
-	err := json.Unmarshal(line, &objectMap)
+	err := json.Unmarshal(content, &objectMap)
 	if err != nil {
 		logger.Errorf("Failed to convert bytes to map[string]interface: %+v", err)
-		return line
+		return content
 	}
 
 	attributes, ok := objectMap["attributes"].(map[string]interface{})
 	if !ok {
 		logger.Errorf("Object does not have attributes key")
-		return line
+		return content
 	}
 
 	if kibanaSavedObject, ok := attributes["kibanaSavedObjectMeta"].(map[string]interface{}); ok {
@@ -219,10 +202,69 @@ func replaceInNDJSON(logger *logp.Logger, index string, line []byte) []byte {
 	b, err := json.Marshal(objectMap)
 	if err != nil {
 		logger.Error("Error marshaling modified dashboard: %+v", err)
-		return line
+		return content
+	}
+
+	return b
+}
+
+func EncodeJSONObjects(content []byte) []byte {
+	logger := logp.NewLogger("dashboards")
+
+	if len(bytes.TrimSpace(content)) == 0 {
+		return content
+	}
+
+	objectMap := make(map[string]interface{}, 0)
+	err := json.Unmarshal(content, &objectMap)
+	if err != nil {
+		logger.Errorf("Failed to convert bytes to map[string]interface: %+v", err)
+		return content
+	}
+
+	attributes, ok := objectMap["attributes"].(map[string]interface{})
+	if !ok {
+		logger.Errorf("Object does not have attributes key")
+		return content
+	}
+
+	if kibanaSavedObject, ok := attributes["kibanaSavedObjectMeta"].(map[string]interface{}); ok {
+		if searchSourceJSON, ok := kibanaSavedObject["searchSourceJSON"].(map[string]interface{}); ok {
+			b, err := json.Marshal(searchSourceJSON)
+			if err != nil {
+				return content
+			}
+			kibanaSavedObject["searchSourceJSON"] = string(b)
+		}
+	}
+
+	fieldsToStr := []string{"visState", "uiStateJSON", "optionsJSON"}
+	for _, field := range fieldsToStr {
+		if rootField, ok := attributes[field].(map[string]interface{}); ok {
+			b, err := json.Marshal(rootField)
+			if err != nil {
+				return content
+			}
+			attributes[field] = string(b)
+		}
+	}
+
+	if panelsJSON, ok := attributes["panelsJSON"].([]interface{}); ok {
+		b, err := json.Marshal(panelsJSON)
+		if err != nil {
+			return content
+		}
+		attributes["panelsJSON"] = string(b)
+
+	}
+
+	b, err := json.Marshal(objectMap)
+	if err != nil {
+		logger.Error("Error marshaling modified dashboard: %+v", err)
+		return content
 	}
 
-	return append(b, newline...)
+	return b
 
 }
 

libbeat/dashboards/modify_json_test.go

@@ -72,18 +72,14 @@ func TestReplaceIndexInDashboardObject(t *testing.T) {
 		expected  []byte
 	}{
 		{
-			[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"metricbeat-*\"}"}}}
-`),
+			[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"metricbeat-*\"}"}}}`),
 			"otherindex-*",
-			[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"otherindex-*\"}"}}}
-`),
+			[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"otherindex-*\"}"}}}`),
 		},
 		{
-			[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"metricbeat-*\"}}"}}}
-`),
+			[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"metricbeat-*\"}}"}}}`),
 			"otherindex-*",
-			[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"otherindex-*\"}}"}}}
-`),
+			[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"otherindex-*\"}}"}}}`),
 		},
 	}