From 4d91ae7b53ed8a5355aa97dc8563375c14988a69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 31 Aug 2021 17:29:58 +0200 Subject: [PATCH] Avoid circular deps when loading dashboard in setup --- libbeat/dashboards/kibana_loader.go | 10 ++- libbeat/dashboards/modify_json.go | 94 +++++++++++++++++++++-------- 2 files changed, 78 insertions(+), 26 deletions(-) diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index 95f8a688223..129e2e3f248 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -44,6 +44,8 @@ type KibanaLoader struct { hostname string msgOutputter MessageOutputter defaultLogger *logp.Logger + + loadedAssets map[string]bool } // NewKibanaLoader creates a new loader to load Kibana files @@ -65,6 +67,7 @@ func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig * hostname: hostname, msgOutputter: msgOutputter, defaultLogger: logp.NewLogger("dashboards"), + loadedAssets: make(map[string]bool, 0), } version := client.GetVersion() @@ -151,7 +154,7 @@ func (loader KibanaLoader) ImportDashboard(file string) error { } content = ReplaceIndexInDashboardObject(loader.config.Index, content) - + content = ConvertToStr(content) content = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content) err = loader.importReferences(file, content) @@ -168,6 +171,8 @@ func (loader KibanaLoader) ImportDashboard(file string) error { if err := loader.client.ImportMultiPartFormFile(importAPI, params, correctExtension(file), obj.String()); err != nil { return fmt.Errorf("error dashboard asset: %+v", err) } + + loader.loadedAssets[file] = true return nil } @@ -192,6 +197,9 @@ func (loader KibanaLoader) importReferences(path string, dashboard []byte) error continue } referencePath := filepath.Join(base, "..", ref.Type, ref.ID+".json") + if _, ok := loader.loadedAssets[referencePath]; ok { + continue + } err := loader.ImportDashboard(referencePath) if err != nil { return fmt.Errorf("error loading reference of %s: %s %s: %+v", path, ref.Type, ref.ID, err) diff --git a/libbeat/dashboards/modify_json.go b/libbeat/dashboards/modify_json.go index 92dcf5f59c9..8fcb90c6b59 100644 --- a/libbeat/dashboards/modify_json.go +++ b/libbeat/dashboards/modify_json.go @@ -18,11 +18,9 @@ package dashboards import ( - "bufio" "bytes" "encoding/json" "fmt" - "io" "github.com/pkg/errors" @@ -40,6 +38,7 @@ type JSONObjectAttribute struct { KibanaSavedObjectMeta map[string]interface{} `json:"kibanaSavedObjectMeta"` Title string `json:"title"` Type string `json:"type"` + UiStateJSON map[string]interface{} `json:"uiStateJSON"` } // JSONObject is an Object with a given JSON attribute @@ -175,54 +174,99 @@ func ReplaceIndexInDashboardObject(index string, content []byte) []byte { return content } - var result []byte - r := bufio.NewReader(bytes.NewReader(content)) - for { - line, err := r.ReadBytes('\n') - if err != nil { - if err == io.EOF { - return append(result, replaceInNDJSON(logger, index, line)...) - } - logger.Error("Error reading bytes from raw dashboard object: %+v", err) - return content - } - result = append(result, replaceInNDJSON(logger, index, line)...) + if len(bytes.TrimSpace(content)) == 0 { + return content + } + + objectMap := make(map[string]interface{}, 0) + err := json.Unmarshal(content, &objectMap) + if err != nil { + logger.Errorf("Failed to convert bytes to map[string]interface: %+v", err) + return content + } + + attributes, ok := objectMap["attributes"].(map[string]interface{}) + if !ok { + logger.Errorf("Object does not have attributes key") + return content + } + + if kibanaSavedObject, ok := attributes["kibanaSavedObjectMeta"].(map[string]interface{}); ok { + ba := ReplaceIndexInSavedObject(logger, index, kibanaSavedObject) + attributes["kibanaSavedObjectMeta"] = ba + } + + if visState, ok := attributes["visState"].(string); ok { + nya := ReplaceIndexInVisState(logger, index, visState) + attributes["visState"] = nya } + + b, err := json.Marshal(objectMap) + if err != nil { + logger.Error("Error marshaling modified dashboard: %+v", err) + return content + } + + return b } -func replaceInNDJSON(logger *logp.Logger, index string, line []byte) []byte { - if len(bytes.TrimSpace(line)) == 0 { - return line +func ConvertToStr(content []byte) []byte { + logger := logp.NewLogger("dashboards") + + if len(bytes.TrimSpace(content)) == 0 { + return content } objectMap := make(map[string]interface{}, 0) - err := json.Unmarshal(line, &objectMap) + err := json.Unmarshal(content, &objectMap) if err != nil { logger.Errorf("Failed to convert bytes to map[string]interface: %+v", err) - return line + return content } attributes, ok := objectMap["attributes"].(map[string]interface{}) if !ok { logger.Errorf("Object does not have attributes key") - return line + return content } if kibanaSavedObject, ok := attributes["kibanaSavedObjectMeta"].(map[string]interface{}); ok { - attributes["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(logger, index, kibanaSavedObject) + if searchSourceJSON, ok := kibanaSavedObject["searchSourceJSON"].(map[string]interface{}); ok { + b, err := json.Marshal(searchSourceJSON) + if err != nil { + return content + } + kibanaSavedObject["searchSourceJSON"] = string(b) + } } - if visState, ok := attributes["visState"].(string); ok { - attributes["visState"] = ReplaceIndexInVisState(logger, index, visState) + fieldsToStr := []string{"visState", "uiStateJSON", "optionsJSON"} + for _, field := range fieldsToStr { + if rootField, ok := attributes[field].(map[string]interface{}); ok { + b, err := json.Marshal(rootField) + if err != nil { + return content + } + attributes[field] = string(b) + } + } + + if panelsJSON, ok := attributes["panelsJSON"].([]interface{}); ok { + b, err := json.Marshal(panelsJSON) + if err != nil { + return content + } + attributes["panelsJSON"] = string(b) + } b, err := json.Marshal(objectMap) if err != nil { logger.Error("Error marshaling modified dashboard: %+v", err) - return line + return content } - return append(b, newline...) + return b }