Import the references of dashboard assets using the Saved Objects API (…

…#27647)

## What does this PR do?

This PR changes the dashboard loading by first loading its references and then loading the dashboards.

## Why is it important?

If the assets are not loaded in the proper order, the import can fail with unknown references.

dev-tools/mage/kibana.go

@@ -69,7 +69,7 @@ func KibanaDashboards(moduleDirs ...string) error {
 	// Convert 7.x dashboards to strings.
 	err = sh.Run(pythonExe,
 		filepath.Join(esBeatsDir, "libbeat/scripts/unpack_dashboards.py"),
-		"--glob="+filepath.Join(kibanaBuildDir, "7/*/*.json"))
+		"--glob="+filepath.Join(kibanaBuildDir, "7/dashboards/*.json"))
 	if err != nil {
 		return err
 	}

libbeat/dashboards/importer.go

@@ -116,7 +116,7 @@ func (imp Importer) ImportDir(dirType string, dir string) error {
 
 	var errors []string
 
-	files, err := filepath.Glob(path.Join(dir, "*", "*.json"))
+	files, err := filepath.Glob(path.Join(dir, dirType, "*.json"))
 	if err != nil {
 		return fmt.Errorf("Failed to read directory %s. Error: %s", dir, err)
 	}

libbeat/dashboards/kibana_loader.go

@@ -154,6 +154,11 @@ func (loader KibanaLoader) ImportDashboard(file string) error {
 
 	content = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content)
 
+	err = loader.importReferences(file, content)
+	if err != nil {
+		return fmt.Errorf("error loading references of dashboard: %+v", err)
+	}
+
 	var obj common.MapStr
 	err = json.Unmarshal(content, &obj)
 	if err != nil {
@@ -166,6 +171,35 @@ func (loader KibanaLoader) ImportDashboard(file string) error {
 	return nil
 }
 
+type dashboardObj struct {
+	References []dashboardReference `json:"references"`
+}
+type dashboardReference struct {
+	ID   string `json:"id"`
+	Type string `json:"type"`
+}
+
+func (loader KibanaLoader) importReferences(path string, dashboard []byte) error {
+	var d dashboardObj
+	err := json.Unmarshal(dashboard, &d)
+	if err != nil {
+		return fmt.Errorf("failed to parse dashboard references: %+v", err)
+	}
+
+	base := filepath.Dir(path)
+	for _, ref := range d.References {
+		if ref.Type == "index-pattern" {
+			continue
+		}
+		referencePath := filepath.Join(base, "..", ref.Type, ref.ID+".json")
+		err := loader.ImportDashboard(referencePath)
+		if err != nil {
+			return fmt.Errorf("error loading reference of %s: %s %s: %+v", path, ref.Type, ref.ID, err)
+		}
+	}
+	return nil
+}
+
 func correctExtension(file string) string {
 	return filepath.Base(file[:len(file)-len("json")]) + "ndjson"
 }

libbeat/kibana/client.go

@@ -94,6 +94,36 @@ func extractError(result []byte) error {
 	return nil
 }
 
+func extractMessage(result []byte) error {
+	var kibanaResult struct {
+		Success bool
+		Errors  []struct {
+			Id    string
+			Type  string
+			Error struct {
+				Type       string
+				References []struct {
+					Type string
+					Id   string
+				}
+			}
+		}
+	}
+	if err := json.Unmarshal(result, &kibanaResult); err != nil {
+		return nil
+	}
+
+	if !kibanaResult.Success {
+		var errs multierror.Errors
+		for _, err := range kibanaResult.Errors {
+			errs = append(errs, fmt.Errorf("error: %s, asset ID=%s; asset type=%s; references=%+v", err.Error.Type, err.Id, err.Type, err.Error.References))
+		}
+		return errs.Err()
+	}
+
+	return nil
+}
+
 // NewKibanaClient builds and returns a new Kibana client
 func NewKibanaClient(cfg *common.Config) (*Client, error) {
 	config := DefaultClientConfig()
@@ -200,6 +230,8 @@ func (conn *Connection) Request(method, extraPath string,
 
 	if resp.StatusCode >= 300 {
 		retError = extractError(result)
+	} else {
+		retError = extractMessage(result)
 	}
 	return resp.StatusCode, result, retError
 }

libbeat/kibana/client_test.go

@@ -63,6 +63,21 @@ func TestErrorBadJson(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func TestErrorJsonWithHTTPOK(t *testing.T) {
+	kibanaTs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`{"successCount":0,"success":false,"warnings":[],"errors":[{"id":"abcf35b0-0a82-11e8-bffe-ff7d4f68cf94-ecs","type":"dashboard","title":"[Filebeat MongoDB] Overview ECS","meta":{"title":"[Filebeat MongoDB] Overview ECS","icon":"dashboardApp"},"error":{"type":"missing_references","references":[{"type":"search","id":"e49fe000-0a7e-11e8-bffe-ff7d4f68cf94-ecs"},{"type":"search","id":"bfc96a60-0a80-11e8-bffe-ff7d4f68cf94-ecs"}]}}]}`))
+	}))
+	defer kibanaTs.Close()
+
+	conn := Connection{
+		URL:  kibanaTs.URL,
+		HTTP: http.DefaultClient,
+	}
+	code, _, err := conn.Request(http.MethodPost, "", url.Values{}, nil, nil)
+	assert.Equal(t, http.StatusOK, code)
+	assert.Error(t, err)
+}
+
 func TestSuccess(t *testing.T) {
 	kibanaTs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Write([]byte(`{"objects":[{"id":"test-*","type":"index-pattern","updated_at":"2018-01-24T19:04:13.371Z","version":1}]}`))