From 89330833ea8777469c11dbc96fc704f37e276092 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 1 Sep 2021 12:25:03 +0200 Subject: [PATCH] Addressing multiple dashboard issues: deps loading once, field conversion, etc. (#27669) (#27682) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR addresses 3 minor issues in dashboard loading: 1. Everything should be loaded once 2. Some fields have to be strings that were objects before 3. Replacing index names in dashboards is no longer an NDJSON This prevented setup from working properly. (cherry picked from commit 42ca950e76e0bebffdc76b5fba2c97891d47e011) Co-authored-by: Noémi Ványi --- libbeat/dashboards/kibana_loader.go | 58 +++++++++++----- libbeat/dashboards/modify_json.go | 92 +++++++++++++++++++------- libbeat/dashboards/modify_json_test.go | 12 ++-- 3 files changed, 112 insertions(+), 50 deletions(-) diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index bec6107cd00..719ac7f2952 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -42,6 +42,8 @@ type KibanaLoader struct { version common.Version hostname string msgOutputter MessageOutputter + + loadedAssets map[string]bool } // NewKibanaLoader creates a new loader to load Kibana files @@ -62,6 +64,7 @@ func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig * version: client.GetVersion(), hostname: hostname, msgOutputter: msgOutputter, + loadedAssets: make(map[string]bool, 0), } version := client.GetVersion() @@ -147,24 +150,18 @@ func (loader KibanaLoader) ImportDashboard(file string) error { return fmt.Errorf("fail to read dashboard from file %s: %v", file, err) } - content = ReplaceIndexInDashboardObject(loader.config.Index, content) - - content = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content) + content = loader.formatDashboardAssets(content) - err = loader.importReferences(file, content) + dashboardWithReferences, err := loader.addReferences(file, content) if err != nil { - return fmt.Errorf("error loading references of dashboard: %+v", err) + return fmt.Errorf("error getting references of dashboard: %+v", err) } - var obj common.MapStr - err = json.Unmarshal(content, &obj) - if err != nil { - return err - } - - if err := loader.client.ImportMultiPartFormFile(importAPI, params, correctExtension(file), obj.String()); err != nil { + if err := loader.client.ImportMultiPartFormFile(importAPI, params, correctExtension(file), dashboardWithReferences); err != nil { return fmt.Errorf("error dashboard asset: %+v", err) } + + loader.loadedAssets[file] = true return nil } @@ -176,25 +173,52 @@ type dashboardReference struct { Type string `json:"type"` } -func (loader KibanaLoader) importReferences(path string, dashboard []byte) error { +func (loader KibanaLoader) addReferences(path string, dashboard []byte) (string, error) { var d dashboardObj err := json.Unmarshal(dashboard, &d) if err != nil { - return fmt.Errorf("failed to parse dashboard references: %+v", err) + return "", fmt.Errorf("failed to parse dashboard references: %+v", err) } base := filepath.Dir(path) + var result string for _, ref := range d.References { if ref.Type == "index-pattern" { continue } referencePath := filepath.Join(base, "..", ref.Type, ref.ID+".json") - err := loader.ImportDashboard(referencePath) + if _, ok := loader.loadedAssets[referencePath]; ok { + continue + } + refContents, err := ioutil.ReadFile(referencePath) + if err != nil { + return "", fmt.Errorf("fail to read referenced asset from file %s: %v", referencePath, err) + } + refContents = loader.formatDashboardAssets(refContents) + refContentsWithReferences, err := loader.addReferences(referencePath, refContents) if err != nil { - return fmt.Errorf("error loading reference of %s: %s %s: %+v", path, ref.Type, ref.ID, err) + return "", fmt.Errorf("failed to get references of %s: %+v", referencePath, err) } + + result += refContentsWithReferences + loader.loadedAssets[referencePath] = true } - return nil + + var res common.MapStr + err = json.Unmarshal(dashboard, &res) + if err != nil { + return "", fmt.Errorf("failed to convert asset: %+v", err) + } + result += res.String() + "\n" + + return result, nil +} + +func (loader KibanaLoader) formatDashboardAssets(content []byte) []byte { + content = ReplaceIndexInDashboardObject(loader.config.Index, content) + content = EncodeJSONObjects(content) + content = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content) + return content } func correctExtension(file string) string { diff --git a/libbeat/dashboards/modify_json.go b/libbeat/dashboards/modify_json.go index dd1332e5873..b6b89e36ed0 100644 --- a/libbeat/dashboards/modify_json.go +++ b/libbeat/dashboards/modify_json.go @@ -18,11 +18,9 @@ package dashboards import ( - "bufio" "bytes" "encoding/json" "fmt" - "io" "github.com/pkg/errors" @@ -40,6 +38,7 @@ type JSONObjectAttribute struct { KibanaSavedObjectMeta map[string]interface{} `json:"kibanaSavedObjectMeta"` Title string `json:"title"` Type string `json:"type"` + UiStateJSON map[string]interface{} `json:"uiStateJSON"` } type JSONObject struct { @@ -170,37 +169,21 @@ func ReplaceIndexInDashboardObject(index string, content []byte) []byte { return content } - var result []byte - r := bufio.NewReader(bytes.NewReader(content)) - for { - line, err := r.ReadBytes('\n') - if err != nil { - if err == io.EOF { - return append(result, replaceInNDJSON(index, line)...) - } - logp.Err("Error reading bytes from raw dashboard object: %+v", err) - return content - } - result = append(result, replaceInNDJSON(index, line)...) - } -} - -func replaceInNDJSON(index string, line []byte) []byte { - if len(bytes.TrimSpace(line)) == 0 { - return line + if len(bytes.TrimSpace(content)) == 0 { + return content } objectMap := make(map[string]interface{}, 0) - err := json.Unmarshal(line, &objectMap) + err := json.Unmarshal(content, &objectMap) if err != nil { logp.Err("Failed to convert bytes to map[string]interface: %+v", err) - return line + return content } attributes, ok := objectMap["attributes"].(map[string]interface{}) if !ok { logp.Err("Object does not have attributes key") - return line + return content } if kibanaSavedObject, ok := attributes["kibanaSavedObjectMeta"].(map[string]interface{}); ok { @@ -214,10 +197,69 @@ func replaceInNDJSON(index string, line []byte) []byte { b, err := json.Marshal(objectMap) if err != nil { logp.Err("Error marshaling modified dashboard: %+v", err) - return line + return content + } + + return b +} + +func EncodeJSONObjects(content []byte) []byte { + logger := logp.NewLogger("dashboards") + + if len(bytes.TrimSpace(content)) == 0 { + return content + } + + objectMap := make(map[string]interface{}, 0) + err := json.Unmarshal(content, &objectMap) + if err != nil { + logger.Errorf("Failed to convert bytes to map[string]interface: %+v", err) + return content + } + + attributes, ok := objectMap["attributes"].(map[string]interface{}) + if !ok { + logger.Errorf("Object does not have attributes key") + return content + } + + if kibanaSavedObject, ok := attributes["kibanaSavedObjectMeta"].(map[string]interface{}); ok { + if searchSourceJSON, ok := kibanaSavedObject["searchSourceJSON"].(map[string]interface{}); ok { + b, err := json.Marshal(searchSourceJSON) + if err != nil { + return content + } + kibanaSavedObject["searchSourceJSON"] = string(b) + } + } + + fieldsToStr := []string{"visState", "uiStateJSON", "optionsJSON"} + for _, field := range fieldsToStr { + if rootField, ok := attributes[field].(map[string]interface{}); ok { + b, err := json.Marshal(rootField) + if err != nil { + return content + } + attributes[field] = string(b) + } + } + + if panelsJSON, ok := attributes["panelsJSON"].([]interface{}); ok { + b, err := json.Marshal(panelsJSON) + if err != nil { + return content + } + attributes["panelsJSON"] = string(b) + + } + + b, err := json.Marshal(objectMap) + if err != nil { + logger.Error("Error marshaling modified dashboard: %+v", err) + return content } - return append(b, newline...) + return b } diff --git a/libbeat/dashboards/modify_json_test.go b/libbeat/dashboards/modify_json_test.go index 367b07f2b3d..48f0fe972c9 100644 --- a/libbeat/dashboards/modify_json_test.go +++ b/libbeat/dashboards/modify_json_test.go @@ -72,18 +72,14 @@ func TestReplaceIndexInDashboardObject(t *testing.T) { expected []byte }{ { - []byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"metricbeat-*\"}"}}} -`), + []byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"metricbeat-*\"}"}}}`), "otherindex-*", - []byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"otherindex-*\"}"}}} -`), + []byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"otherindex-*\"}"}}}`), }, { - []byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"metricbeat-*\"}}"}}} -`), + []byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"metricbeat-*\"}}"}}}`), "otherindex-*", - []byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"otherindex-*\"}}"}}} -`), + []byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"otherindex-*\"}}"}}}`), }, }