Add multiple config for datasource

CHANGELOG.md

@@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Make README.md file a required file for a package. [#287](https://github.com/elastic/integrations-registry/pull/287)
 *  Add stream fields to each dataset [#296](https://github.com/elastic/integrations-registry/pull/296)
 * Add `all` query param to return all packages. By default is set false. [#301](https://github.com/elastic/integrations-registry/pull/301)
+* Add `multiple` config for datasource. By default true. [#361](https://github.com/elastic/integrations-registry/pull/361)
 * Add `removable` flag to package manifest. Default is true. [#359](https://github.com/elastic/integrations-registry/pull/359)
 
 ### Deprecated

docs/api/categories.json

@@ -2,7 +2,7 @@
   {
     "id": "logs",
     "title": "Logs",
-    "count": 3
+    "count": 4
   },
   {
     "id": "metrics",

docs/api/package.json

@@ -107,7 +107,8 @@
             }
           ]
         }
-      ]
+      ],
+      "multiple": true
     }
   ],
   "download": "/epr/example/example-1.0.0.tar.gz",

docs/api/search-all.json

@@ -43,5 +43,14 @@
     "title": "Foo",
     "type": "solution",
     "version": "1.0.0"
+  },
+  {
+    "description": "Tests that multiple can be set to false",
+    "download": "/epr/multiple-false/multiple-false-0.0.1.tar.gz",
+    "name": "multiple-false",
+    "path": "/package/multiple-false/0.0.1",
+    "title": "Multiple false",
+    "type": "integration",
+    "version": "0.0.1"
   }
 ]

docs/api/search-category-logs.json

@@ -25,5 +25,14 @@
     "title": "Example Integration",
     "type": "integration",
     "version": "1.0.0"
+  },
+  {
+    "description": "Tests that multiple can be set to false",
+    "download": "/epr/multiple-false/multiple-false-0.0.1.tar.gz",
+    "name": "multiple-false",
+    "path": "/package/multiple-false/0.0.1",
+    "title": "Multiple false",
+    "type": "integration",
+    "version": "0.0.1"
   }
 ]

docs/api/search-kibana652.json

@@ -25,5 +25,14 @@
     "title": "Example",
     "type": "integration",
     "version": "0.0.2"
+  },
+  {
+    "description": "Tests that multiple can be set to false",
+    "download": "/epr/multiple-false/multiple-false-0.0.1.tar.gz",
+    "name": "multiple-false",
+    "path": "/package/multiple-false/0.0.1",
+    "title": "Multiple false",
+    "type": "integration",
+    "version": "0.0.1"
   }
 ]

docs/api/search-kibana721.json

@@ -34,5 +34,14 @@
     "title": "Foo",
     "type": "solution",
     "version": "1.0.0"
+  },
+  {
+    "description": "Tests that multiple can be set to false",
+    "download": "/epr/multiple-false/multiple-false-0.0.1.tar.gz",
+    "name": "multiple-false",
+    "path": "/package/multiple-false/0.0.1",
+    "title": "Multiple false",
+    "type": "integration",
+    "version": "0.0.1"
   }
 ]

docs/api/search-package-internal.json

@@ -44,5 +44,14 @@
     "title": "Internal",
     "type": "integration",
     "version": "1.2.0"
+  },
+  {
+    "description": "Tests that multiple can be set to false",
+    "download": "/epr/multiple-false/multiple-false-0.0.1.tar.gz",
+    "name": "multiple-false",
+    "path": "/package/multiple-false/0.0.1",
+    "title": "Multiple false",
+    "type": "integration",
+    "version": "0.0.1"
   }
 ]

docs/api/search.json

@@ -34,5 +34,14 @@
     "title": "Foo",
     "type": "solution",
     "version": "1.0.0"
+  },
+  {
+    "description": "Tests that multiple can be set to false",
+    "download": "/epr/multiple-false/multiple-false-0.0.1.tar.gz",
+    "name": "multiple-false",
+    "path": "/package/multiple-false/0.0.1",
+    "title": "Multiple false",
+    "type": "integration",
+    "version": "0.0.1"
   }
 ]

testdata/package/multiple-false-0.0.1/dataset/foo/agent/stream/stream.yml

@@ -0,0 +1 @@
+foo: bar

testdata/package/multiple-false-0.0.1/dataset/foo/elasticsearch/ingest-pipeline/default.json

@@ -0,0 +1,4 @@
+{
+  "description": "Pipeline for normalizing envoyproxy logs",
+  "processors": []
+}

testdata/package/multiple-false-0.0.1/dataset/foo/manifest.yml

@@ -0,0 +1,13 @@
+title: Foo
+
+# Needs to describe the type of this input
+type: logs
+
+streams:
+  - input: logs
+    vars:
+      - name: paths
+        required: true
+        description: Path to log files to be collected
+        type: text
+        multi: true

testdata/package/multiple-false-0.0.1/manifest.yml

@@ -0,0 +1,18 @@
+format_version: 1.0.0
+
+name: multiple-false
+description: Tests that multiple can be set to false
+version: 0.0.1
+title: Multiple false
+categories: ["logs"]
+type: integration
+
+
+datasources:
+  - name: logs
+    title: Logs datasource
+    description: Datasource for your log files.
+    inputs:
+      - type: logs
+    multiple: false
+

testdata/public/package/datasources/1.0.0/index.json

@@ -275,7 +275,8 @@
         {
           "type": "syslog"
         }
-      ]
+      ],
+      "multiple": true
     }
   ],
   "download": "/epr/datasources/datasources-1.0.0.tar.gz",

testdata/public/package/default-pipeline/0.0.2/index.json

@@ -75,7 +75,8 @@
             }
           ]
         }
-      ]
+      ],
+      "multiple": true
     }
   ],
   "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",

testdata/public/package/example/1.0.0/index.json

@@ -107,7 +107,8 @@
             }
           ]
         }
-      ]
+      ],
+      "multiple": true
     }
   ],
   "download": "/epr/example/example-1.0.0.tar.gz",

testdata/public/package/multiple-false/0.0.1/dataset/foo/agent/stream/stream.yml

@@ -0,0 +1 @@
+foo: bar

testdata/public/package/multiple-false/0.0.1/dataset/foo/elasticsearch/ingest-pipeline/default.json

@@ -0,0 +1,4 @@
+{
+  "description": "Pipeline for normalizing envoyproxy logs",
+  "processors": []
+}

testdata/public/package/multiple-false/0.0.1/dataset/foo/fields/stream.yml

@@ -0,0 +1,13 @@
+
+- name: stream.type
+  type: constant_keyword
+  description: >
+    Stream type
+- name: stream.dataset
+  type: constant_keyword
+  description: >
+    Stream dataset.
+- name: stream.namespace
+  type: constant_keyword
+  description: >
+    Stream namespace.

testdata/public/package/multiple-false/0.0.1/dataset/foo/manifest.yml

@@ -0,0 +1,13 @@
+title: Foo
+
+# Needs to describe the type of this input
+type: logs
+
+streams:
+  - input: logs
+    vars:
+      - name: paths
+        required: true
+        description: Path to log files to be collected
+        type: text
+        multi: true

testdata/public/package/multiple-false/0.0.1/index.json

@@ -0,0 +1,84 @@
+{
+  "format_version": "1.0.0",
+  "name": "multiple-false",
+  "title": "Multiple false",
+  "version": "0.0.1",
+  "readme": "/package/multiple-false/0.0.1/docs/README.md",
+  "license": "basic",
+  "description": "Tests that multiple can be set to false",
+  "type": "integration",
+  "categories": [
+    "logs"
+  ],
+  "removable": true,
+  "requirement": {
+    "kibana": {},
+    "elasticsearch": {}
+  },
+  "assets": [
+    "/package/multiple-false/0.0.1/manifest.yml",
+    "/package/multiple-false/0.0.1/docs/README.md",
+    "/package/multiple-false/0.0.1/dataset/foo/manifest.yml",
+    "/package/multiple-false/0.0.1/dataset/foo/fields/stream.yml",
+    "/package/multiple-false/0.0.1/dataset/foo/agent/stream/stream.yml",
+    "/package/multiple-false/0.0.1/dataset/foo/elasticsearch/ingest-pipeline/default.json"
+  ],
+  "datasets": [
+    {
+      "id": "multiple-false.foo",
+      "title": "Foo",
+      "release": "beta",
+      "type": "logs",
+      "ingest_pipeline": "default",
+      "streams": [
+        {
+          "input": "logs",
+          "vars": [
+            {
+              "name": "paths",
+              "type": "text",
+              "description": "Path to log files to be collected",
+              "multi": true,
+              "required": true,
+              "show_user": false
+            }
+          ]
+        }
+      ],
+      "package": "multiple-false",
+      "path": "foo"
+    }
+  ],
+  "datasources": [
+    {
+      "name": "logs",
+      "title": "Logs datasource",
+      "description": "Datasource for your log files.",
+      "inputs": [
+        {
+          "type": "logs",
+          "streams": [
+            {
+              "input": "logs",
+              "vars": [
+                {
+                  "name": "paths",
+                  "type": "text",
+                  "description": "Path to log files to be collected",
+                  "multi": true,
+                  "required": true,
+                  "show_user": false
+                }
+              ],
+              "dataset": "multiple-false.foo",
+              "template": "foo: bar\n"
+            }
+          ]
+        }
+      ],
+      "multiple": false
+    }
+  ],
+  "download": "/epr/multiple-false/multiple-false-0.0.1.tar.gz",
+  "path": "/package/multiple-false/0.0.1"
+}

testdata/public/package/multiple-false/0.0.1/manifest.yml

@@ -0,0 +1,18 @@
+format_version: 1.0.0
+
+name: multiple-false
+description: Tests that multiple can be set to false
+version: 0.0.1
+title: Multiple false
+categories: ["logs"]
+type: integration
+
+
+datasources:
+  - name: logs
+    title: Logs datasource
+    description: Datasource for your log files.
+    inputs:
+      - type: logs
+    multiple: false
+

util/package.go

@@ -62,6 +62,7 @@ type Datasource struct {
 	Description string  `config:"description" json:"description" validate:"required"`
 	Solution    string  `config:"solution" json:"solution,omitempty" yaml:"solution,omitempty"`
 	Inputs      []Input `config:"inputs" json:"inputs"`
+	Multiple    *bool   `config:"multiple" json:"multiple" yaml:"multiple"`
 }
 
 type Requirement struct {
@@ -108,6 +109,13 @@ func NewPackage(basePath string) (*Package, error) {
 		return nil, err
 	}
 
+	// Default for the multiple flags is true.
+	trueValue := true
+	for i, _ := range p.Datasources {
+		if p.Datasources[i].Multiple == nil {
+			p.Datasources[i].Multiple = &trueValue
+		}
+	}
 	if p.Type == "" {
 		p.Type = defaultType
 	}