Skip to content

Commit

Permalink
Addressing multiple dashboard issues: deps loading once, field conver…
Browse files Browse the repository at this point in the history 
…sion, etc. (#27669) (#27682)

This PR addresses 3 minor issues in dashboard loading:
1. Everything should be loaded once
2. Some fields have to be strings that were objects before
3. Replacing index names in dashboards is no longer an NDJSON

This prevented setup from working properly.

(cherry picked from commit 42ca950)

Co-authored-by: Noémi Ványi <[email protected]>
  • Loading branch information
@mergify @kvch
mergify[bot] and kvch authored Sep 1, 2021
1 parent 7ab0719 commit 8933083
Show file tree
Hide file tree
Showing 3 changed files with 112 additions and 50 deletions.
58 changes: 41 additions & 17 deletions libbeat/dashboards/kibana_loader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ type KibanaLoader struct {
version common.Version
hostname string
msgOutputter MessageOutputter

loadedAssets map[string]bool
}

// NewKibanaLoader creates a new loader to load Kibana files
Expand All @@ -62,6 +64,7 @@ func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig *
version: client.GetVersion(),
hostname: hostname,
msgOutputter: msgOutputter,
loadedAssets: make(map[string]bool, 0),
}

version := client.GetVersion()
Expand Down Expand Up @@ -147,24 +150,18 @@ func (loader KibanaLoader) ImportDashboard(file string) error {
return fmt.Errorf("fail to read dashboard from file %s: %v", file, err)
}

content = ReplaceIndexInDashboardObject(loader.config.Index, content)

content = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content)
content = loader.formatDashboardAssets(content)

err = loader.importReferences(file, content)
dashboardWithReferences, err := loader.addReferences(file, content)
if err != nil {
return fmt.Errorf("error loading references of dashboard: %+v", err)
return fmt.Errorf("error getting references of dashboard: %+v", err)
}

var obj common.MapStr
err = json.Unmarshal(content, &obj)
if err != nil {
return err
}

if err := loader.client.ImportMultiPartFormFile(importAPI, params, correctExtension(file), obj.String()); err != nil {
if err := loader.client.ImportMultiPartFormFile(importAPI, params, correctExtension(file), dashboardWithReferences); err != nil {
return fmt.Errorf("error dashboard asset: %+v", err)
}

loader.loadedAssets[file] = true
return nil
}

Expand All @@ -176,25 +173,52 @@ type dashboardReference struct {
Type string `json:"type"`
}

func (loader KibanaLoader) importReferences(path string, dashboard []byte) error {
func (loader KibanaLoader) addReferences(path string, dashboard []byte) (string, error) {
var d dashboardObj
err := json.Unmarshal(dashboard, &d)
if err != nil {
return fmt.Errorf("failed to parse dashboard references: %+v", err)
return "", fmt.Errorf("failed to parse dashboard references: %+v", err)
}

base := filepath.Dir(path)
var result string
for _, ref := range d.References {
if ref.Type == "index-pattern" {
continue
}
referencePath := filepath.Join(base, "..", ref.Type, ref.ID+".json")
err := loader.ImportDashboard(referencePath)
if _, ok := loader.loadedAssets[referencePath]; ok {
continue
}
refContents, err := ioutil.ReadFile(referencePath)
if err != nil {
return "", fmt.Errorf("fail to read referenced asset from file %s: %v", referencePath, err)
}
refContents = loader.formatDashboardAssets(refContents)
refContentsWithReferences, err := loader.addReferences(referencePath, refContents)
if err != nil {
return fmt.Errorf("error loading reference of %s: %s %s: %+v", path, ref.Type, ref.ID, err)
return "", fmt.Errorf("failed to get references of %s: %+v", referencePath, err)
}

result += refContentsWithReferences
loader.loadedAssets[referencePath] = true
}
return nil

var res common.MapStr
err = json.Unmarshal(dashboard, &res)
if err != nil {
return "", fmt.Errorf("failed to convert asset: %+v", err)
}
result += res.String() + "\n"

return result, nil
}

func (loader KibanaLoader) formatDashboardAssets(content []byte) []byte {
content = ReplaceIndexInDashboardObject(loader.config.Index, content)
content = EncodeJSONObjects(content)
content = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content)
return content
}

func correctExtension(file string) string {
Expand Down
92 changes: 67 additions & 25 deletions libbeat/dashboards/modify_json.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,9 @@
package dashboards

import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"

"github.com/pkg/errors"

Expand All @@ -40,6 +38,7 @@ type JSONObjectAttribute struct {
KibanaSavedObjectMeta map[string]interface{} `json:"kibanaSavedObjectMeta"`
Title string `json:"title"`
Type string `json:"type"`
UiStateJSON map[string]interface{} `json:"uiStateJSON"`
}

type JSONObject struct {
Expand Down Expand Up @@ -170,37 +169,21 @@ func ReplaceIndexInDashboardObject(index string, content []byte) []byte {
return content
}

var result []byte
r := bufio.NewReader(bytes.NewReader(content))
for {
line, err := r.ReadBytes('\n')
if err != nil {
if err == io.EOF {
return append(result, replaceInNDJSON(index, line)...)
}
logp.Err("Error reading bytes from raw dashboard object: %+v", err)
return content
}
result = append(result, replaceInNDJSON(index, line)...)
}
}

func replaceInNDJSON(index string, line []byte) []byte {
if len(bytes.TrimSpace(line)) == 0 {
return line
if len(bytes.TrimSpace(content)) == 0 {
return content
}

objectMap := make(map[string]interface{}, 0)
err := json.Unmarshal(line, &objectMap)
err := json.Unmarshal(content, &objectMap)
if err != nil {
logp.Err("Failed to convert bytes to map[string]interface: %+v", err)
return line
return content
}

attributes, ok := objectMap["attributes"].(map[string]interface{})
if !ok {
logp.Err("Object does not have attributes key")
return line
return content
}

if kibanaSavedObject, ok := attributes["kibanaSavedObjectMeta"].(map[string]interface{}); ok {
Expand All @@ -214,10 +197,69 @@ func replaceInNDJSON(index string, line []byte) []byte {
b, err := json.Marshal(objectMap)
if err != nil {
logp.Err("Error marshaling modified dashboard: %+v", err)
return line
return content
}

return b
}

func EncodeJSONObjects(content []byte) []byte {
logger := logp.NewLogger("dashboards")

if len(bytes.TrimSpace(content)) == 0 {
return content
}

objectMap := make(map[string]interface{}, 0)
err := json.Unmarshal(content, &objectMap)
if err != nil {
logger.Errorf("Failed to convert bytes to map[string]interface: %+v", err)
return content
}

attributes, ok := objectMap["attributes"].(map[string]interface{})
if !ok {
logger.Errorf("Object does not have attributes key")
return content
}

if kibanaSavedObject, ok := attributes["kibanaSavedObjectMeta"].(map[string]interface{}); ok {
if searchSourceJSON, ok := kibanaSavedObject["searchSourceJSON"].(map[string]interface{}); ok {
b, err := json.Marshal(searchSourceJSON)
if err != nil {
return content
}
kibanaSavedObject["searchSourceJSON"] = string(b)
}
}

fieldsToStr := []string{"visState", "uiStateJSON", "optionsJSON"}
for _, field := range fieldsToStr {
if rootField, ok := attributes[field].(map[string]interface{}); ok {
b, err := json.Marshal(rootField)
if err != nil {
return content
}
attributes[field] = string(b)
}
}

if panelsJSON, ok := attributes["panelsJSON"].([]interface{}); ok {
b, err := json.Marshal(panelsJSON)
if err != nil {
return content
}
attributes["panelsJSON"] = string(b)

}

b, err := json.Marshal(objectMap)
if err != nil {
logger.Error("Error marshaling modified dashboard: %+v", err)
return content
}

return append(b, newline...)
return b

}

Expand Down
12 changes: 4 additions & 8 deletions libbeat/dashboards/modify_json_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,18 +72,14 @@ func TestReplaceIndexInDashboardObject(t *testing.T) {
expected []byte
}{
{
[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"metricbeat-*\"}"}}}
`),
[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"metricbeat-*\"}"}}}`),
"otherindex-*",
[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"otherindex-*\"}"}}}
`),
[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"otherindex-*\"}"}}}`),
},
{
[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"metricbeat-*\"}}"}}}
`),
[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"metricbeat-*\"}}"}}}`),
"otherindex-*",
[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"otherindex-*\"}}"}}}
`),
[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"otherindex-*\"}}"}}}`),
},
}

Expand Down

0 comments on commit 8933083

Please sign in to comment.