diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index 95f8a688223..7c4897df021 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -44,6 +44,8 @@ type KibanaLoader struct { hostname string msgOutputter MessageOutputter defaultLogger *logp.Logger + + loadedAssets map[string]bool } // NewKibanaLoader creates a new loader to load Kibana files @@ -65,6 +67,7 @@ func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig * hostname: hostname, msgOutputter: msgOutputter, defaultLogger: logp.NewLogger("dashboards"), + loadedAssets: make(map[string]bool, 0), } version := client.GetVersion() @@ -150,24 +153,18 @@ func (loader KibanaLoader) ImportDashboard(file string) error { return fmt.Errorf("fail to read dashboard from file %s: %v", file, err) } - content = ReplaceIndexInDashboardObject(loader.config.Index, content) - - content = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content) + content = loader.formatDashboardAssets(content) - err = loader.importReferences(file, content) + dashboardWithReferences, err := loader.addReferences(file, content) if err != nil { - return fmt.Errorf("error loading references of dashboard: %+v", err) + return fmt.Errorf("error getting references of dashboard: %+v", err) } - var obj common.MapStr - err = json.Unmarshal(content, &obj) - if err != nil { - return err - } - - if err := loader.client.ImportMultiPartFormFile(importAPI, params, correctExtension(file), obj.String()); err != nil { + if err := loader.client.ImportMultiPartFormFile(importAPI, params, correctExtension(file), dashboardWithReferences); err != nil { return fmt.Errorf("error dashboard asset: %+v", err) } + + loader.loadedAssets[file] = true return nil } @@ -179,25 +176,52 @@ type dashboardReference struct { Type string `json:"type"` } -func (loader KibanaLoader) importReferences(path string, dashboard []byte) error { +func (loader KibanaLoader) addReferences(path string, dashboard []byte) (string, error) { var d dashboardObj err := json.Unmarshal(dashboard, &d) if err != nil { - return fmt.Errorf("failed to parse dashboard references: %+v", err) + return "", fmt.Errorf("failed to parse dashboard references: %+v", err) } base := filepath.Dir(path) + var result string for _, ref := range d.References { if ref.Type == "index-pattern" { continue } referencePath := filepath.Join(base, "..", ref.Type, ref.ID+".json") - err := loader.ImportDashboard(referencePath) + if _, ok := loader.loadedAssets[referencePath]; ok { + continue + } + refContents, err := ioutil.ReadFile(referencePath) + if err != nil { + return "", fmt.Errorf("fail to read referenced asset from file %s: %v", referencePath, err) + } + refContents = loader.formatDashboardAssets(refContents) + refContentsWithReferences, err := loader.addReferences(referencePath, refContents) if err != nil { - return fmt.Errorf("error loading reference of %s: %s %s: %+v", path, ref.Type, ref.ID, err) + return "", fmt.Errorf("failed to get references of %s: %+v", referencePath, err) } + + result += refContentsWithReferences + loader.loadedAssets[referencePath] = true } - return nil + + var res common.MapStr + err = json.Unmarshal(dashboard, &res) + if err != nil { + return "", fmt.Errorf("failed to convert asset: %+v", err) + } + result += res.String() + "\n" + + return result, nil +} + +func (loader KibanaLoader) formatDashboardAssets(content []byte) []byte { + content = ReplaceIndexInDashboardObject(loader.config.Index, content) + content = EncodeJSONObjects(content) + content = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content) + return content } func correctExtension(file string) string { diff --git a/libbeat/dashboards/modify_json.go b/libbeat/dashboards/modify_json.go index 92dcf5f59c9..3178d6b2382 100644 --- a/libbeat/dashboards/modify_json.go +++ b/libbeat/dashboards/modify_json.go @@ -18,11 +18,9 @@ package dashboards import ( - "bufio" "bytes" "encoding/json" "fmt" - "io" "github.com/pkg/errors" @@ -40,6 +38,7 @@ type JSONObjectAttribute struct { KibanaSavedObjectMeta map[string]interface{} `json:"kibanaSavedObjectMeta"` Title string `json:"title"` Type string `json:"type"` + UiStateJSON map[string]interface{} `json:"uiStateJSON"` } // JSONObject is an Object with a given JSON attribute @@ -175,37 +174,21 @@ func ReplaceIndexInDashboardObject(index string, content []byte) []byte { return content } - var result []byte - r := bufio.NewReader(bytes.NewReader(content)) - for { - line, err := r.ReadBytes('\n') - if err != nil { - if err == io.EOF { - return append(result, replaceInNDJSON(logger, index, line)...) - } - logger.Error("Error reading bytes from raw dashboard object: %+v", err) - return content - } - result = append(result, replaceInNDJSON(logger, index, line)...) - } -} - -func replaceInNDJSON(logger *logp.Logger, index string, line []byte) []byte { - if len(bytes.TrimSpace(line)) == 0 { - return line + if len(bytes.TrimSpace(content)) == 0 { + return content } objectMap := make(map[string]interface{}, 0) - err := json.Unmarshal(line, &objectMap) + err := json.Unmarshal(content, &objectMap) if err != nil { logger.Errorf("Failed to convert bytes to map[string]interface: %+v", err) - return line + return content } attributes, ok := objectMap["attributes"].(map[string]interface{}) if !ok { logger.Errorf("Object does not have attributes key") - return line + return content } if kibanaSavedObject, ok := attributes["kibanaSavedObjectMeta"].(map[string]interface{}); ok { @@ -219,10 +202,69 @@ func replaceInNDJSON(logger *logp.Logger, index string, line []byte) []byte { b, err := json.Marshal(objectMap) if err != nil { logger.Error("Error marshaling modified dashboard: %+v", err) - return line + return content + } + + return b +} + +func EncodeJSONObjects(content []byte) []byte { + logger := logp.NewLogger("dashboards") + + if len(bytes.TrimSpace(content)) == 0 { + return content + } + + objectMap := make(map[string]interface{}, 0) + err := json.Unmarshal(content, &objectMap) + if err != nil { + logger.Errorf("Failed to convert bytes to map[string]interface: %+v", err) + return content + } + + attributes, ok := objectMap["attributes"].(map[string]interface{}) + if !ok { + logger.Errorf("Object does not have attributes key") + return content + } + + if kibanaSavedObject, ok := attributes["kibanaSavedObjectMeta"].(map[string]interface{}); ok { + if searchSourceJSON, ok := kibanaSavedObject["searchSourceJSON"].(map[string]interface{}); ok { + b, err := json.Marshal(searchSourceJSON) + if err != nil { + return content + } + kibanaSavedObject["searchSourceJSON"] = string(b) + } + } + + fieldsToStr := []string{"visState", "uiStateJSON", "optionsJSON"} + for _, field := range fieldsToStr { + if rootField, ok := attributes[field].(map[string]interface{}); ok { + b, err := json.Marshal(rootField) + if err != nil { + return content + } + attributes[field] = string(b) + } + } + + if panelsJSON, ok := attributes["panelsJSON"].([]interface{}); ok { + b, err := json.Marshal(panelsJSON) + if err != nil { + return content + } + attributes["panelsJSON"] = string(b) + + } + + b, err := json.Marshal(objectMap) + if err != nil { + logger.Error("Error marshaling modified dashboard: %+v", err) + return content } - return append(b, newline...) + return b } diff --git a/libbeat/dashboards/modify_json_test.go b/libbeat/dashboards/modify_json_test.go index 367b07f2b3d..48f0fe972c9 100644 --- a/libbeat/dashboards/modify_json_test.go +++ b/libbeat/dashboards/modify_json_test.go @@ -72,18 +72,14 @@ func TestReplaceIndexInDashboardObject(t *testing.T) { expected []byte }{ { - []byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"metricbeat-*\"}"}}} -`), + []byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"metricbeat-*\"}"}}}`), "otherindex-*", - []byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"otherindex-*\"}"}}} -`), + []byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"otherindex-*\"}"}}}`), }, { - []byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"metricbeat-*\"}}"}}} -`), + []byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"metricbeat-*\"}}"}}}`), "otherindex-*", - []byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"otherindex-*\"}}"}}} -`), + []byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"otherindex-*\"}}"}}}`), }, }