Encode the Kibana saved objects during packaging (#157)

In the packages, the saved objects are stored with decoded JSON fields. The reason is that this makes versioning of it much easier and the diffs are simpler. But inside Kibana some of the fields are stored as encoded json strings (this might change in the future elastic/kibana#43673). To not require special logic on the Package Manager to encode the strings, this is done directly during packaging.

One thing not too nice about this PR is that it includes now a dependency on `common.MapStr` from Beats. Reason is that it makes the code much simpler.

Part of #42

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Breaking changes
 
 * Change `requirements.kibana.version.min/max` to `requirements.kibana.versions: {semver-range}`
+* Encode Kibana objects during packaging. [#157](https://github.com/elastic/integrations-registry/pull/157)
 
 ### Bugfixes
 

dev/generator/.gitignore

@@ -0,0 +1 @@
+generator

dev/generator/main.go

@@ -119,13 +119,6 @@ func buildPackage(packagesBasePath string, p util.Package) error {
 	}
 	defer os.Chdir(currentPath)
 
-	if tarGz {
-		err = sh.RunV("tar", "cvzf", p.GetPath()+".tar.gz", filepath.Base(p.GetPath())+"/")
-		if err != nil {
-			return fmt.Errorf("Error creating package: %s: %s", p.GetPath(), err)
-		}
-	}
-
 	// Checks if the package is valid
 	err = p.Validate()
 	if err != nil {
@@ -141,6 +134,43 @@ func buildPackage(packagesBasePath string, p util.Package) error {
 	if err != nil {
 		return err
 	}
+
+	// Get all Kibana files
+	savedObjects1, err := filepath.Glob(p.GetPath() + "/dataset/*/kibana/*/*")
+	if err != nil {
+		return err
+	}
+	savedObjects2, err := filepath.Glob(p.GetPath() + "/kibana/*/*")
+	if err != nil {
+		return err
+	}
+	savedObjects := append(savedObjects1, savedObjects2...)
+
+	// Run each file through the saved object encoder
+	for _, file := range savedObjects {
+
+		data, err := ioutil.ReadFile(file)
+		if err != nil {
+			return err
+		}
+		output, err := encodedSavedObject(data)
+		if err != nil {
+			return err
+		}
+
+		err = ioutil.WriteFile(file, []byte(output), 0644)
+		if err != nil {
+			return err
+		}
+	}
+
+	if tarGz {
+		err = sh.RunV("tar", "cvzf", p.GetPath()+".tar.gz", filepath.Base(p.GetPath())+"/")
+		if err != nil {
+			return fmt.Errorf("Error creating package: %s: %s", p.GetPath(), err)
+		}
+	}
+
 	return nil
 }
 
@@ -152,3 +182,40 @@ func writeJsonFile(v interface{}, path string) error {
 
 	return ioutil.WriteFile(path, data, 0644)
 }
+
+var (
+	fieldsToEncode = []string{
+		"attributes.uiStateJSON",
+		"attributes.visState",
+		"attributes.optionsJSON",
+		"attributes.panelsJSON",
+		"attributes.kibanaSavedObjectMeta.searchSourceJSON",
+	}
+)
+
+// encodeSavedObject encodes all the fields inside a saved object
+// which are stored in encoded JSON in Kibana.
+// The reason is that for versioning it is much nicer to have the full
+// json so only on packaging this is changed.
+func encodedSavedObject(data []byte) (string, error) {
+
+	savedObject := MapStr{}
+	json.Unmarshal(data, &savedObject)
+
+	for _, v := range fieldsToEncode {
+		out, err := savedObject.GetValue(v)
+		// This means the key did not exists, no conversion needed
+		if err != nil {
+			continue
+		}
+
+		// Marshal the value to encode it properly
+		r, err := json.Marshal(&out)
+		if err != nil {
+			return "", err
+		}
+		savedObject.Put(v, string(r))
+	}
+
+	return savedObject.StringToPrint(), nil
+}

dev/generator/main_test.go

@@ -0,0 +1,18 @@
+package main
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEncodeSavedObject(t *testing.T) {
+	file := "../package-examples/auditd-2.0.4/kibana/dashboard/7de391b0-c1ca-11e7-8995-936807a28b16-ecs.json"
+
+	data, err := ioutil.ReadFile(file)
+	assert.NoError(t, err)
+
+	_, err = encodedSavedObject(data)
+	assert.NoError(t, err)
+}