Avoid circular deps when loading dashboard in setup

libbeat/dashboards/kibana_loader.go

@@ -44,6 +44,8 @@ type KibanaLoader struct {
 	hostname      string
 	msgOutputter  MessageOutputter
 	defaultLogger *logp.Logger
+
+	loadedAssets map[string]bool
 }
 
 // NewKibanaLoader creates a new loader to load Kibana files
@@ -65,6 +67,7 @@ func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig *
 		hostname:      hostname,
 		msgOutputter:  msgOutputter,
 		defaultLogger: logp.NewLogger("dashboards"),
+		loadedAssets:  make(map[string]bool, 0),
 	}
 
 	version := client.GetVersion()
@@ -151,7 +154,7 @@ func (loader KibanaLoader) ImportDashboard(file string) error {
 	}
 
 	content = ReplaceIndexInDashboardObject(loader.config.Index, content)
-
+	content = ConvertToStr(content)
 	content = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content)
 
 	err = loader.importReferences(file, content)
@@ -168,6 +171,8 @@ func (loader KibanaLoader) ImportDashboard(file string) error {
 	if err := loader.client.ImportMultiPartFormFile(importAPI, params, correctExtension(file), obj.String()); err != nil {
 		return fmt.Errorf("error dashboard asset: %+v", err)
 	}
+
+	loader.loadedAssets[file] = true
 	return nil
 }
 
@@ -192,6 +197,9 @@ func (loader KibanaLoader) importReferences(path string, dashboard []byte) error
 			continue
 		}
 		referencePath := filepath.Join(base, "..", ref.Type, ref.ID+".json")
+		if _, ok := loader.loadedAssets[referencePath]; ok {
+			continue
+		}
 		err := loader.ImportDashboard(referencePath)
 		if err != nil {
 			return fmt.Errorf("error loading reference of %s: %s %s: %+v", path, ref.Type, ref.ID, err)

libbeat/dashboards/modify_json.go

@@ -18,11 +18,9 @@
 package dashboards
 
 import (
-	"bufio"
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
 
 	"github.com/pkg/errors"
 
@@ -40,6 +38,7 @@ type JSONObjectAttribute struct {
 	KibanaSavedObjectMeta map[string]interface{} `json:"kibanaSavedObjectMeta"`
 	Title                 string                 `json:"title"`
 	Type                  string                 `json:"type"`
+	UiStateJSON           map[string]interface{} `json:"uiStateJSON"`
 }
 
 // JSONObject is an Object with a given JSON attribute
@@ -175,54 +174,99 @@ func ReplaceIndexInDashboardObject(index string, content []byte) []byte {
 		return content
 	}
 
-	var result []byte
-	r := bufio.NewReader(bytes.NewReader(content))
-	for {
-		line, err := r.ReadBytes('\n')
-		if err != nil {
-			if err == io.EOF {
-				return append(result, replaceInNDJSON(logger, index, line)...)
-			}
-			logger.Error("Error reading bytes from raw dashboard object: %+v", err)
-			return content
-		}
-		result = append(result, replaceInNDJSON(logger, index, line)...)
+	if len(bytes.TrimSpace(content)) == 0 {
+		return content
+	}
+
+	objectMap := make(map[string]interface{}, 0)
+	err := json.Unmarshal(content, &objectMap)
+	if err != nil {
+		logger.Errorf("Failed to convert bytes to map[string]interface: %+v", err)
+		return content
+	}
+
+	attributes, ok := objectMap["attributes"].(map[string]interface{})
+	if !ok {
+		logger.Errorf("Object does not have attributes key")
+		return content
+	}
+
+	if kibanaSavedObject, ok := attributes["kibanaSavedObjectMeta"].(map[string]interface{}); ok {
+		ba := ReplaceIndexInSavedObject(logger, index, kibanaSavedObject)
+		attributes["kibanaSavedObjectMeta"] = ba
+	}
+
+	if visState, ok := attributes["visState"].(string); ok {
+		nya := ReplaceIndexInVisState(logger, index, visState)
+		attributes["visState"] = nya
 	}
+
+	b, err := json.Marshal(objectMap)
+	if err != nil {
+		logger.Error("Error marshaling modified dashboard: %+v", err)
+		return content
+	}
+
+	return b
 }
 
-func replaceInNDJSON(logger *logp.Logger, index string, line []byte) []byte {
-	if len(bytes.TrimSpace(line)) == 0 {
-		return line
+func ConvertToStr(content []byte) []byte {
+	logger := logp.NewLogger("dashboards")
+
+	if len(bytes.TrimSpace(content)) == 0 {
+		return content
 	}
 
 	objectMap := make(map[string]interface{}, 0)
-	err := json.Unmarshal(line, &objectMap)
+	err := json.Unmarshal(content, &objectMap)
 	if err != nil {
 		logger.Errorf("Failed to convert bytes to map[string]interface: %+v", err)
-		return line
+		return content
 	}
 
 	attributes, ok := objectMap["attributes"].(map[string]interface{})
 	if !ok {
 		logger.Errorf("Object does not have attributes key")
-		return line
+		return content
 	}
 
 	if kibanaSavedObject, ok := attributes["kibanaSavedObjectMeta"].(map[string]interface{}); ok {
-		attributes["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(logger, index, kibanaSavedObject)
+		if searchSourceJSON, ok := kibanaSavedObject["searchSourceJSON"].(map[string]interface{}); ok {
+			b, err := json.Marshal(searchSourceJSON)
+			if err != nil {
+				return content
+			}
+			kibanaSavedObject["searchSourceJSON"] = string(b)
+		}
 	}
 
-	if visState, ok := attributes["visState"].(string); ok {
-		attributes["visState"] = ReplaceIndexInVisState(logger, index, visState)
+	fieldsToStr := []string{"visState", "uiStateJSON", "optionsJSON"}
+	for _, field := range fieldsToStr {
+		if rootField, ok := attributes[field].(map[string]interface{}); ok {
+			b, err := json.Marshal(rootField)
+			if err != nil {
+				return content
+			}
+			attributes[field] = string(b)
+		}
+	}
+
+	if panelsJSON, ok := attributes["panelsJSON"].([]interface{}); ok {
+		b, err := json.Marshal(panelsJSON)
+		if err != nil {
+			return content
+		}
+		attributes["panelsJSON"] = string(b)
+
 	}
 
 	b, err := json.Marshal(objectMap)
 	if err != nil {
 		logger.Error("Error marshaling modified dashboard: %+v", err)
-		return line
+		return content
 	}
 
-	return append(b, newline...)
+	return b
 
 }