Import "beats" dashboards

dev/import-beats/common.go

@@ -0,0 +1,136 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+// Source code origin:
+// github.com/elastic/beats/libbeat/common/mapstr.go
+
+var (
+	// errKeyNotFound indicates that the specified key was not found.
+	errKeyNotFound = errors.New("key not found")
+)
+
+type mapStr map[string]interface{}
+
+// getValue gets a value from the map. If the key does not exist then an error
+// is returned.
+func (m mapStr) getValue(key string) (interface{}, error) {
+	_, _, v, found, err := mapFind(key, m, false)
+	if err != nil {
+		return nil, err
+	}
+	if !found {
+		return nil, errKeyNotFound
+	}
+	return v, nil
+}
+
+// put associates the specified value with the specified key. If the map
+// previously contained a mapping for the key, the old value is replaced and
+// returned. The key can be expressed in dot-notation (e.g. x.y) to put a value
+// into a nested map.
+//
+// If you need insert keys containing dots then you must use bracket notation
+// to insert values (e.g. m[key] = value).
+func (m mapStr) put(key string, value interface{}) (interface{}, error) {
+	// XXX `safemapstr.Put` mimics this implementation, both should be updated to have similar behavior
+	k, d, old, _, err := mapFind(key, m, true)
+	if err != nil {
+		return nil, err
+	}
+
+	d[k] = value
+	return old, nil
+}
+
+// delete deletes the given key from the map.
+func (m mapStr) delete(key string) error {
+	k, d, _, found, err := mapFind(key, m, false)
+	if err != nil {
+		return err
+	}
+	if !found {
+		return errKeyNotFound
+	}
+
+	delete(d, k)
+	return nil
+}
+
+// mapFind iterates a mapStr based on a the given dotted key, finding the final
+// subMap and subKey to operate on.
+// An error is returned if some intermediate is no map or the key doesn't exist.
+// If createMissing is set to true, intermediate maps are created.
+// The final map and un-dotted key to run further operations on are returned in
+// subKey and subMap. The subMap already contains a value for subKey, the
+// present flag is set to true and the oldValue return will hold
+// the original value.
+func mapFind(
+	key string,
+	data mapStr,
+	createMissing bool,
+) (subKey string, subMap mapStr, oldValue interface{}, present bool, err error) {
+	// XXX `safemapstr.mapFind` mimics this implementation, both should be updated to have similar behavior
+
+	for {
+		// Fast path, key is present as is.
+		if v, exists := data[key]; exists {
+			return key, data, v, true, nil
+		}
+
+		idx := strings.IndexRune(key, '.')
+		if idx < 0 {
+			return key, data, nil, false, nil
+		}
+
+		k := key[:idx]
+		d, exists := data[k]
+		if !exists {
+			if createMissing {
+				d = mapStr{}
+				data[k] = d
+			} else {
+				return "", nil, nil, false, errKeyNotFound
+			}
+		}
+
+		v, err := toMapStr(d)
+		if err != nil {
+			return "", nil, nil, false, err
+		}
+
+		// advance to sub-map
+		key = key[idx+1:]
+		data = v
+	}
+}
+
+// tomapStr performs a type assertion on v and returns a mapStr. v can be either
+// a mapStr or a map[string]interface{}. If it's any other type or nil then
+// an error is returned.
+func toMapStr(v interface{}) (mapStr, error) {
+	m, ok := tryTomapStr(v)
+	if !ok {
+		return nil, fmt.Errorf("expected map but type is %v", v)
+	}
+	return m, nil
+}
+
+func tryTomapStr(v interface{}) (mapStr, bool) {
+	switch m := v.(type) {
+	case mapStr:
+		return m, true
+	case map[string]interface{}:
+		return mapStr(m), true
+	default:
+		return nil, false
+	}
+}

dev/import-beats/kibana.go

@@ -0,0 +1,248 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	encodedFields = []string{
+		"attributes.uiStateJSON",
+		"attributes.visState",
+		"attributes.optionsJSON",
+		"attributes.panelsJSON",
+		"attributes.kibanaSavedObjectMeta.searchSourceJSON",
+	}
+)
+
+type kibanaContent struct {
+	dashboardFiles     map[string][]byte
+	visualizationFiles map[string][]byte
+}
+
+type kibanaMigrator struct {
+	hostPort string
+}
+
+type kibanaDocuments struct {
+	Objects []mapStr `json:"objects"`
+}
+
+func newKibanaMigrator(hostPort string) *kibanaMigrator {
+	return &kibanaMigrator{
+		hostPort: hostPort,
+	}
+}
+
+func (km *kibanaMigrator) migrateDashboardFile(dashboardFile []byte) ([]byte, error) {
+	dashboardFile, err := prepareDashboardFile(dashboardFile)
+	if err != nil {
+		return nil, errors.Wrapf(err, "preparing file failed")
+	}
+
+	request, err := http.NewRequest("POST",
+		fmt.Sprintf("http://%s/api/kibana/dashboards/import?force=true", km.hostPort),
+		bytes.NewReader(dashboardFile))
+	if err != nil {
+		return nil, errors.Wrapf(err, "creating POST request failed")
+	}
+	request.Header.Add("kbn-xsrf", "8.0.0")
+	response, err := http.DefaultClient.Do(request)
+	if err != nil {
+		return nil, errors.Wrapf(err, "making POST request to Kibana failed")
+	}
+	defer response.Body.Close()
+
+	saved, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return nil, errors.Wrapf(err, "reading saved object failed")
+	}
+
+	if response.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("making POST request failed: %s", string(saved))
+	}
+	return saved, nil
+}
+
+func prepareDashboardFile(dashboardFile []byte) ([]byte, error) {
+	var documents kibanaDocuments
+
+	err := json.Unmarshal(dashboardFile, &documents)
+	if err != nil {
+		return nil, errors.Wrapf(err, "unmarshalling dashboard file failed")
+	}
+
+	for i, object := range documents.Objects {
+		object, err = encodeFields(object)
+		if err != nil {
+			return nil, errors.Wrapf(err, "encoding fields failed")
+		}
+		documents.Objects[i] = object
+	}
+
+	data, err := json.Marshal(&documents)
+	if err != nil {
+		return nil, errors.Wrapf(err, "marshalling dashboard file failed")
+	}
+	return data, nil
+}
+
+func encodeFields(ms mapStr) (mapStr, error) {
+	for _, field := range encodedFields {
+		v, err := ms.getValue(field)
+		if err == errKeyNotFound {
+			continue
+		} else if err != nil {
+			return mapStr{}, errors.Wrapf(err, "retrieving value failed (key: %s)", field)
+		}
+
+		ve, err := json.Marshal(v)
+		if err != nil {
+			return mapStr{}, errors.Wrapf(err, "marshalling value failed (key: %s)", field)
+		}
+
+		_, err = ms.put(field, string(ve))
+		if err != nil {
+			return mapStr{}, errors.Wrapf(err, "putting value failed (key: %s)", field)
+		}
+	}
+	return ms, nil
+}
+
+func createKibanaContent(kibanaMigrator *kibanaMigrator, modulePath string) (kibanaContent, error) {
+	moduleDashboardPath := path.Join(modulePath, "_meta", "kibana", "7", "dashboard")
+	moduleDashboards, err := ioutil.ReadDir(moduleDashboardPath)
+	if os.IsNotExist(err) {
+		log.Printf("\tno dashboards present, skipped (modulePath: %s)", modulePath)
+		return kibanaContent{}, nil
+	}
+	if err != nil {
+		return kibanaContent{}, errors.Wrapf(err, "reading module dashboard directory failed (path: %s)",
+			moduleDashboardPath)
+	}
+
+	kibana := kibanaContent{
+		dashboardFiles:     map[string][]byte{},
+		visualizationFiles: map[string][]byte{},
+	}
+	for _, moduleDashboard := range moduleDashboards {
+		log.Printf("\tdashboard found: %s", moduleDashboard.Name())
+
+		dashboardFilePath := path.Join(moduleDashboardPath, moduleDashboard.Name())
+		dashboardFile, err := ioutil.ReadFile(dashboardFilePath)
+		if err != nil {
+			return kibanaContent{}, errors.Wrapf(err, "reading dashboard file failed (path: %s)",
+				dashboardFilePath)
+		}
+
+		migrated, err := kibanaMigrator.migrateDashboardFile(dashboardFile)
+		if err != nil {
+			return kibanaContent{}, errors.Wrapf(err, "migrating dashboard file failed (path: %s)",
+				dashboardFilePath)
+		}
+
+		extractedDashboards, err := extractKibanaObjects(migrated, "dashboard")
+		if err != nil {
+			return kibanaContent{}, errors.Wrapf(err, "extracting kibana dashboards failed")
+		}
+
+		for k, v := range extractedDashboards {
+			kibana.dashboardFiles[k] = v
+		}
+
+		extractedVisualizations, err := extractKibanaObjects(migrated, "visualization")
+		if err != nil {
+			return kibanaContent{}, errors.Wrapf(err, "extracting kibana visualizations failed")
+		}
+
+		for k, v := range extractedVisualizations {
+			kibana.visualizationFiles[k] = v
+		}
+	}
+	return kibana, nil
+}
+
+func extractKibanaObjects(dashboardFile []byte, objectType string) (map[string][]byte, error) {
+	var documents kibanaDocuments
+
+	err := json.Unmarshal(dashboardFile, &documents)
+	if err != nil {
+		return nil, errors.Wrapf(err, "unmarshalling migrated dashboard file failed")
+	}
+
+	extracted := map[string][]byte{}
+	for _, object := range documents.Objects {
+		aType, err := object.getValue("type")
+		if err != nil {
+			return nil, errors.Wrapf(err, "retrieving type failed")
+		}
+
+		if aType != objectType {
+			continue
+		}
+
+		err = object.delete("updated_at")
+		if err != nil {
+			return nil, errors.Wrapf(err, "removing field updated_at failed")
+		}
+
+		err = object.delete("version")
+		if err != nil {
+			return nil, errors.Wrapf(err, "removing field version failed")
+		}
+
+		object, err = decodeFields(object)
+		if err != nil {
+			return nil, errors.Wrapf(err, "decoding fields failed")
+		}
+
+		data, err := json.MarshalIndent(object, "", "    ")
+		if err != nil {
+			return nil, errors.Wrapf(err, "marshalling object failed")
+		}
+
+		id, err := object.getValue("id")
+		if err != nil {
+			return nil, errors.Wrapf(err, "retrieving id failed")
+		}
+
+		extracted[id.(string)+".json"] = data
+	}
+
+	return extracted, nil
+}
+
+func decodeFields(ms mapStr) (mapStr, error) {
+	for _, field := range encodedFields {
+		v, err := ms.getValue(field)
+		if err == errKeyNotFound {
+			continue
+		} else if err != nil {
+			return mapStr{}, errors.Wrapf(err, "retrieving value failed (key: %s)", field)
+		}
+
+		var vd interface{}
+		err = json.Unmarshal([]byte(v.(string)), &vd)
+		if err != nil {
+			return mapStr{}, errors.Wrapf(err, "unmarshalling value failed (key: %s)", field)
+		}
+
+		_, err = ms.put(field, vd)
+		if err != nil {
+			return mapStr{}, errors.Wrapf(err, "putting value failed (key: %s)", field)
+		}
+	}
+	return ms, nil
+}