Add missing assets from datasets (#146)

The current code was only going 4 levels deep when looking for assets. But since we introduce datasets more level exists. This code makes sure when the package output is generated it goes into all directories.

docs/api/package.json

@@ -35,18 +35,19 @@
     "/package/example-1.0.0/docs/README.md",
     "/package/example-1.0.0/img/icon.png",
     "/package/example-1.0.0/img/kibana-envoyproxy.jpg",
-    "/package/example-1.0.0/elasticsearch/ingest-pipeline/pipeline-entry.json",
-    "/package/example-1.0.0/elasticsearch/ingest-pipeline/pipeline-http.json",
-    "/package/example-1.0.0/elasticsearch/ingest-pipeline/pipeline-json.json",
-    "/package/example-1.0.0/elasticsearch/ingest-pipeline/pipeline-plaintext.json",
-    "/package/example-1.0.0/elasticsearch/ingest-pipeline/pipeline-tcp.json",
+    "/package/example-1.0.0/dataset/foo/manifest.yml",
     "/package/example-1.0.0/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json",
     "/package/example-1.0.0/kibana/index-pattern/filebeat-*.json",
     "/package/example-1.0.0/kibana/visualization/0a994af0-5c9d-11e9-8477-077ec9664dbd.json",
     "/package/example-1.0.0/kibana/visualization/36f872a0-5c03-11e9-85b4-19d0072eb4f2.json",
     "/package/example-1.0.0/kibana/visualization/38f96190-5c99-11e9-8477-077ec9664dbd.json",
     "/package/example-1.0.0/kibana/visualization/7e4084e0-5c99-11e9-8477-077ec9664dbd.json",
     "/package/example-1.0.0/kibana/visualization/80844540-5c97-11e9-8477-077ec9664dbd.json",
-    "/package/example-1.0.0/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json"
+    "/package/example-1.0.0/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json",
+    "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-entry.json",
+    "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-http.json",
+    "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-json.json",
+    "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-plaintext.json",
+    "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-tcp.json"
   ]
 }

testdata/package/example-1.0.0/dataset/foo/manifest.yml

@@ -0,0 +1,5 @@
+title: Foo
+
+# Needs to describe the type of this input
+type: logs
+ingest_pipline: pipeline-entry

testdata/package/example-1.0.0/index.json

@@ -35,18 +35,19 @@
     "/package/example-1.0.0/docs/README.md",
     "/package/example-1.0.0/img/icon.png",
     "/package/example-1.0.0/img/kibana-envoyproxy.jpg",
-    "/package/example-1.0.0/elasticsearch/ingest-pipeline/pipeline-entry.json",
-    "/package/example-1.0.0/elasticsearch/ingest-pipeline/pipeline-http.json",
-    "/package/example-1.0.0/elasticsearch/ingest-pipeline/pipeline-json.json",
-    "/package/example-1.0.0/elasticsearch/ingest-pipeline/pipeline-plaintext.json",
-    "/package/example-1.0.0/elasticsearch/ingest-pipeline/pipeline-tcp.json",
+    "/package/example-1.0.0/dataset/foo/manifest.yml",
     "/package/example-1.0.0/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json",
     "/package/example-1.0.0/kibana/index-pattern/filebeat-*.json",
     "/package/example-1.0.0/kibana/visualization/0a994af0-5c9d-11e9-8477-077ec9664dbd.json",
     "/package/example-1.0.0/kibana/visualization/36f872a0-5c03-11e9-85b4-19d0072eb4f2.json",
     "/package/example-1.0.0/kibana/visualization/38f96190-5c99-11e9-8477-077ec9664dbd.json",
     "/package/example-1.0.0/kibana/visualization/7e4084e0-5c99-11e9-8477-077ec9664dbd.json",
     "/package/example-1.0.0/kibana/visualization/80844540-5c97-11e9-8477-077ec9664dbd.json",
-    "/package/example-1.0.0/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json"
+    "/package/example-1.0.0/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json",
+    "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-entry.json",
+    "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-http.json",
+    "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-json.json",
+    "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-plaintext.json",
+    "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-tcp.json"
   ]
 }

util/package.go

@@ -182,23 +182,14 @@ func (p *Package) LoadAssets(packagePath string) (err error) {
 		return err
 	}
 
-	assets, err := filepath.Glob("*")
+	// Iterates recursively through all the levels to find assets
+	// If we need more complex matching a library like https://github.com/bmatcuk/doublestar
+	// could be used but the below works and is pretty simple.
+	assets, err := collectAssets("*")
 	if err != nil {
 		return err
 	}
 
-	a, err := filepath.Glob("*/*")
-	if err != nil {
-		return err
-	}
-	assets = append(assets, a...)
-
-	a, err = filepath.Glob("*/*/*")
-	if err != nil {
-		return err
-	}
-	assets = append(assets, a...)
-
 	for _, a := range assets {
 		// Unfortunately these files keep sneaking in
 		if strings.Contains(a, ".DS_Store") {
@@ -220,6 +211,21 @@ func (p *Package) LoadAssets(packagePath string) (err error) {
 	return nil
 }
 
+func collectAssets(pattern string) ([]string, error) {
+	assets, err := filepath.Glob(pattern)
+	if err != nil {
+		return nil, err
+	}
+	if len(assets) != 0 {
+		a, err := collectAssets(pattern + "/*")
+		if err != nil {
+			return nil, err
+		}
+		return append(assets, a...), nil
+	}
+	return nil, nil
+}
+
 func (p *Package) Validate() error {
 
 	if p.Title == nil || *p.Title == "" {