From b90370d14e5829184813b3c42f351b57aa282325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 31 Aug 2021 12:10:44 +0200 Subject: [PATCH] Import the references of dashboard assets using the Saved Objects API (#27647) ## What does this PR do? This PR changes the dashboard loading by first loading its references and then loading the dashboards. ## Why is it important? If the assets are not loaded in the proper order, the import can fail with unknown references. --- dev-tools/mage/kibana.go | 2 +- libbeat/dashboards/importer.go | 2 +- libbeat/dashboards/kibana_loader.go | 34 +++++++++++++++++++++++++++++ libbeat/kibana/client.go | 32 +++++++++++++++++++++++++++ libbeat/kibana/client_test.go | 15 +++++++++++++ 5 files changed, 83 insertions(+), 2 deletions(-) diff --git a/dev-tools/mage/kibana.go b/dev-tools/mage/kibana.go index 4a087ac4982..cbd9da6ed07 100644 --- a/dev-tools/mage/kibana.go +++ b/dev-tools/mage/kibana.go @@ -69,7 +69,7 @@ func KibanaDashboards(moduleDirs ...string) error { // Convert 7.x dashboards to strings. err = sh.Run(pythonExe, filepath.Join(esBeatsDir, "libbeat/scripts/unpack_dashboards.py"), - "--glob="+filepath.Join(kibanaBuildDir, "7/*/*.json")) + "--glob="+filepath.Join(kibanaBuildDir, "7/dashboards/*.json")) if err != nil { return err } diff --git a/libbeat/dashboards/importer.go b/libbeat/dashboards/importer.go index d86eda91abf..b27ec695cad 100644 --- a/libbeat/dashboards/importer.go +++ b/libbeat/dashboards/importer.go @@ -116,7 +116,7 @@ func (imp Importer) ImportDir(dirType string, dir string) error { var errors []string - files, err := filepath.Glob(path.Join(dir, "*", "*.json")) + files, err := filepath.Glob(path.Join(dir, dirType, "*.json")) if err != nil { return fmt.Errorf("Failed to read directory %s. Error: %s", dir, err) } diff --git a/libbeat/dashboards/kibana_loader.go b/libbeat/dashboards/kibana_loader.go index 7ad25991a80..95f8a688223 100644 --- a/libbeat/dashboards/kibana_loader.go +++ b/libbeat/dashboards/kibana_loader.go @@ -154,6 +154,11 @@ func (loader KibanaLoader) ImportDashboard(file string) error { content = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content) + err = loader.importReferences(file, content) + if err != nil { + return fmt.Errorf("error loading references of dashboard: %+v", err) + } + var obj common.MapStr err = json.Unmarshal(content, &obj) if err != nil { @@ -166,6 +171,35 @@ func (loader KibanaLoader) ImportDashboard(file string) error { return nil } +type dashboardObj struct { + References []dashboardReference `json:"references"` +} +type dashboardReference struct { + ID string `json:"id"` + Type string `json:"type"` +} + +func (loader KibanaLoader) importReferences(path string, dashboard []byte) error { + var d dashboardObj + err := json.Unmarshal(dashboard, &d) + if err != nil { + return fmt.Errorf("failed to parse dashboard references: %+v", err) + } + + base := filepath.Dir(path) + for _, ref := range d.References { + if ref.Type == "index-pattern" { + continue + } + referencePath := filepath.Join(base, "..", ref.Type, ref.ID+".json") + err := loader.ImportDashboard(referencePath) + if err != nil { + return fmt.Errorf("error loading reference of %s: %s %s: %+v", path, ref.Type, ref.ID, err) + } + } + return nil +} + func correctExtension(file string) string { return filepath.Base(file[:len(file)-len("json")]) + "ndjson" } diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index 2c5e9e93ab5..5a3b37d9625 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -94,6 +94,36 @@ func extractError(result []byte) error { return nil } +func extractMessage(result []byte) error { + var kibanaResult struct { + Success bool + Errors []struct { + Id string + Type string + Error struct { + Type string + References []struct { + Type string + Id string + } + } + } + } + if err := json.Unmarshal(result, &kibanaResult); err != nil { + return nil + } + + if !kibanaResult.Success { + var errs multierror.Errors + for _, err := range kibanaResult.Errors { + errs = append(errs, fmt.Errorf("error: %s, asset ID=%s; asset type=%s; references=%+v", err.Error.Type, err.Id, err.Type, err.Error.References)) + } + return errs.Err() + } + + return nil +} + // NewKibanaClient builds and returns a new Kibana client func NewKibanaClient(cfg *common.Config) (*Client, error) { config := DefaultClientConfig() @@ -200,6 +230,8 @@ func (conn *Connection) Request(method, extraPath string, if resp.StatusCode >= 300 { retError = extractError(result) + } else { + retError = extractMessage(result) } return resp.StatusCode, result, retError } diff --git a/libbeat/kibana/client_test.go b/libbeat/kibana/client_test.go index 1f103f387e4..9cb5d0d47ab 100644 --- a/libbeat/kibana/client_test.go +++ b/libbeat/kibana/client_test.go @@ -63,6 +63,21 @@ func TestErrorBadJson(t *testing.T) { assert.Error(t, err) } +func TestErrorJsonWithHTTPOK(t *testing.T) { + kibanaTs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"successCount":0,"success":false,"warnings":[],"errors":[{"id":"abcf35b0-0a82-11e8-bffe-ff7d4f68cf94-ecs","type":"dashboard","title":"[Filebeat MongoDB] Overview ECS","meta":{"title":"[Filebeat MongoDB] Overview ECS","icon":"dashboardApp"},"error":{"type":"missing_references","references":[{"type":"search","id":"e49fe000-0a7e-11e8-bffe-ff7d4f68cf94-ecs"},{"type":"search","id":"bfc96a60-0a80-11e8-bffe-ff7d4f68cf94-ecs"}]}}]}`)) + })) + defer kibanaTs.Close() + + conn := Connection{ + URL: kibanaTs.URL, + HTTP: http.DefaultClient, + } + code, _, err := conn.Request(http.MethodPost, "", url.Values{}, nil, nil) + assert.Equal(t, http.StatusOK, code) + assert.Error(t, err) +} + func TestSuccess(t *testing.T) { kibanaTs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`{"objects":[{"id":"test-*","type":"index-pattern","updated_at":"2018-01-24T19:04:13.371Z","version":1}]}`))