Import the references of dashboard assets using the Saved Objects API (…

…#27647) (#27665)

This PR changes the dashboard loading by first loading its references and then loading the dashboards.

If the assets are not loaded in the proper order, the import can fail with unknown references.

(cherry picked from commit b90370d)

Co-authored-by: Noémi Ványi <[email protected]>

dev-tools/mage/kibana.go

@@ -69,7 +69,7 @@ func KibanaDashboards(moduleDirs ...string) error {
 	// Convert 7.x dashboards to strings.
 	err = sh.Run(pythonExe,
 		filepath.Join(esBeatsDir, "libbeat/scripts/unpack_dashboards.py"),
-		"--glob="+filepath.Join(kibanaBuildDir, "7/*/*.json"))
+		"--glob="+filepath.Join(kibanaBuildDir, "7/dashboard/*.json"))
 	if err != nil {
 		return err
 	}

libbeat/dashboards/importer.go

@@ -112,7 +112,7 @@ func (imp Importer) ImportDir(dirType string, dir string) error {
 
 	var errors []string
 
-	files, err := filepath.Glob(path.Join(dir, "*", "*.json"))
+	files, err := filepath.Glob(path.Join(dir, dirType, "*.json"))
 	if err != nil {
 		return fmt.Errorf("Failed to read directory %s. Error: %s", dir, err)
 	}

libbeat/dashboards/kibana_loader.go

@@ -151,6 +151,11 @@ func (loader KibanaLoader) ImportDashboard(file string) error {
 
 	content = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content)
 
+	err = loader.importReferences(file, content)
+	if err != nil {
+		return fmt.Errorf("error loading references of dashboard: %+v", err)
+	}
+
 	var obj common.MapStr
 	err = json.Unmarshal(content, &obj)
 	if err != nil {
@@ -163,6 +168,35 @@ func (loader KibanaLoader) ImportDashboard(file string) error {
 	return nil
 }
 
+type dashboardObj struct {
+	References []dashboardReference `json:"references"`
+}
+type dashboardReference struct {
+	ID   string `json:"id"`
+	Type string `json:"type"`
+}
+
+func (loader KibanaLoader) importReferences(path string, dashboard []byte) error {
+	var d dashboardObj
+	err := json.Unmarshal(dashboard, &d)
+	if err != nil {
+		return fmt.Errorf("failed to parse dashboard references: %+v", err)
+	}
+
+	base := filepath.Dir(path)
+	for _, ref := range d.References {
+		if ref.Type == "index-pattern" {
+			continue
+		}
+		referencePath := filepath.Join(base, "..", ref.Type, ref.ID+".json")
+		err := loader.ImportDashboard(referencePath)
+		if err != nil {
+			return fmt.Errorf("error loading reference of %s: %s %s: %+v", path, ref.Type, ref.ID, err)
+		}
+	}
+	return nil
+}
+
 func correctExtension(file string) string {
 	return filepath.Base(file[:len(file)-len("json")]) + "ndjson"
 }

libbeat/kibana/client.go

@@ -91,6 +91,36 @@ func extractError(result []byte) error {
 	return nil
 }
 
+func extractMessage(result []byte) error {
+	var kibanaResult struct {
+		Success bool
+		Errors  []struct {
+			Id    string
+			Type  string
+			Error struct {
+				Type       string
+				References []struct {
+					Type string
+					Id   string
+				}
+			}
+		}
+	}
+	if err := json.Unmarshal(result, &kibanaResult); err != nil {
+		return nil
+	}
+
+	if !kibanaResult.Success {
+		var errs multierror.Errors
+		for _, err := range kibanaResult.Errors {
+			errs = append(errs, fmt.Errorf("error: %s, asset ID=%s; asset type=%s; references=%+v", err.Error.Type, err.Id, err.Type, err.Error.References))
+		}
+		return errs.Err()
+	}
+
+	return nil
+}
+
 // NewKibanaClient builds and returns a new Kibana client
 func NewKibanaClient(cfg *common.Config) (*Client, error) {
 	config := DefaultClientConfig()
@@ -186,6 +216,8 @@ func (conn *Connection) Request(method, extraPath string,
 
 	if resp.StatusCode >= 300 {
 		retError = extractError(result)
+	} else {
+		retError = extractMessage(result)
 	}
 	return resp.StatusCode, result, retError
 }

libbeat/kibana/client_test.go

@@ -63,6 +63,21 @@ func TestErrorBadJson(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestErrorJsonWithHTTPOK(t *testing.T) {
+	kibanaTs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`{"successCount":0,"success":false,"warnings":[],"errors":[{"id":"abcf35b0-0a82-11e8-bffe-ff7d4f68cf94-ecs","type":"dashboard","title":"[Filebeat MongoDB] Overview ECS","meta":{"title":"[Filebeat MongoDB] Overview ECS","icon":"dashboardApp"},"error":{"type":"missing_references","references":[{"type":"search","id":"e49fe000-0a7e-11e8-bffe-ff7d4f68cf94-ecs"},{"type":"search","id":"bfc96a60-0a80-11e8-bffe-ff7d4f68cf94-ecs"}]}}]}`))
+	}))
+	defer kibanaTs.Close()
+
+	conn := Connection{
+		URL:  kibanaTs.URL,
+		HTTP: http.DefaultClient,
+	}
+	code, _, err := conn.Request(http.MethodPost, "", url.Values{}, nil, nil)
+	assert.Equal(t, http.StatusOK, code)
+	assert.Error(t, err)
+}
+
 func TestSuccess(t *testing.T) {
 	kibanaTs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Write([]byte(`{"objects":[{"id":"test-*","type":"index-pattern","updated_at":"2018-01-24T19:04:13.371Z","version":1}]}`))