Add code to expose datasource configs in API

CHANGELOG.md

@@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Add path field to search and package info endpoint. [#174](https://github.com/elastic/integrations-registry/pull/174)
 * Add download field to package info endpoint. [#174](https://github.com/elastic/integrations-registry/pull/174)
 * Add `package` field to dataset. [#189](https://github.com/elastic/integrations-registry/pull/189)
+* Add support for datasources. [#216](https://github.com/elastic/integrations-registry/pull/216) [#212](https://github.com/elastic/integrations-registry/pull/212)
 
 ### Deprecated
 

docs/api/categories.json

@@ -2,7 +2,7 @@
   {
     "id": "logs",
     "title": "Logs",
-    "count": 2
+    "count": 3
   },
   {
     "id": "metrics",

docs/api/search-category-logs.json

@@ -1,4 +1,13 @@
 [
+  {
+    "description": "Package with data sources",
+    "download": "/epr/datasources/datasources-1.0.0.tar.gz",
+    "name": "datasources",
+    "path": "/package/datasources-1.0.0",
+    "title": "Default datasource Integration",
+    "type": "integration",
+    "version": "1.0.0"
+  },
   {
     "description": "Tests if no pipeline is set, it defaults to the default one",
     "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",

docs/api/search-kibana652.json

@@ -1,4 +1,13 @@
 [
+  {
+    "description": "Package with data sources",
+    "download": "/epr/datasources/datasources-1.0.0.tar.gz",
+    "name": "datasources",
+    "path": "/package/datasources-1.0.0",
+    "title": "Default datasource Integration",
+    "type": "integration",
+    "version": "1.0.0"
+  },
   {
     "description": "Tests if no pipeline is set, it defaults to the default one",
     "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",

docs/api/search-kibana721.json

@@ -1,4 +1,13 @@
 [
+  {
+    "description": "Package with data sources",
+    "download": "/epr/datasources/datasources-1.0.0.tar.gz",
+    "name": "datasources",
+    "path": "/package/datasources-1.0.0",
+    "title": "Default datasource Integration",
+    "type": "integration",
+    "version": "1.0.0"
+  },
   {
     "description": "Tests if no pipeline is set, it defaults to the default one",
     "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",

docs/api/search-package-internal.json

@@ -1,4 +1,13 @@
 [
+  {
+    "description": "Package with data sources",
+    "download": "/epr/datasources/datasources-1.0.0.tar.gz",
+    "name": "datasources",
+    "path": "/package/datasources-1.0.0",
+    "title": "Default datasource Integration",
+    "type": "integration",
+    "version": "1.0.0"
+  },
   {
     "description": "Tests if no pipeline is set, it defaults to the default one",
     "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",

docs/api/search.json

@@ -1,4 +1,13 @@
 [
+  {
+    "description": "Package with data sources",
+    "download": "/epr/datasources/datasources-1.0.0.tar.gz",
+    "name": "datasources",
+    "path": "/package/datasources-1.0.0",
+    "title": "Default datasource Integration",
+    "type": "integration",
+    "version": "1.0.0"
+  },
   {
     "description": "Tests if no pipeline is set, it defaults to the default one",
     "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",

testdata/package/datasources-1.0.0/dataset/example/manifest.yml

@@ -0,0 +1,23 @@
+title: Example dataset with inputs
+type: logs
+
+# List of supported inputs
+inputs:
+  - type: log
+    vars:
+      - name: paths
+        required: true
+        # Should we define this as array? How will the UI best make sense of it?
+        description: Paths to the nginx access log file.
+        type: text
+        multi: true
+        default:
+          - /var/log/nginx/access.log*
+        # I suggest to use ECS fields for this config options here: https://github.com/elastic/ecs/blob/master/schemas/os.yml
+        # This would need to be based on a predefined definition on what can be filtered on
+        os.darwin:
+          default:
+            - /usr/local/var/log/nginx/access.log*
+        os.windows:
+          default:
+            - c:/programdata/nginx/logs/*access.log*

testdata/package/datasources-1.0.0/manifest.yml

@@ -0,0 +1,56 @@
+format_version: 1.0.0
+
+name: datasources
+description: Package with data sources
+version: 1.0.0
+title: Default datasource Integration
+categories: ["logs"]
+type: integration
+
+
+datasources:
+  -
+    # Do we need a name for the data source?
+    name: nginx
+
+    # List of inputs this datasource supports
+    inputs:
+      -
+        # An id can be given, in case the type used here is not unique
+        # This is for selection in the stream
+        # id: nginx
+        type: nginx/metrics
+        descrition: Collecting metrics for nginx.
+
+        # Common configuration options for this input
+        vars:
+          - name: hosts
+            description: Nginx hosts
+            default:
+              ["http://127.0.0.1"]
+            # All the config options that are required should be shown in the UI
+            required: true
+            multi: true
+            type: text
+          - name: period
+            description: "Collection period. Valid values: 10s, 5m, 2h"
+            default: "10s"
+            type: duration
+          - name: username
+            type: text
+          - name: password
+            # This is the html input type?
+            type: password
+
+      -
+        type: logs
+        description: Collect nginx logs.
+
+        # Common configuration options for this input
+        vars:
+
+      -
+        type: syslog
+
+        # Common configuration options for this input
+        vars:

testdata/public/package/datasources-1.0.0/dataset/example/manifest.yml

@@ -0,0 +1,23 @@
+title: Example dataset with inputs
+type: logs
+
+# List of supported inputs
+inputs:
+  - type: log
+    vars:
+      - name: paths
+        required: true
+        # Should we define this as array? How will the UI best make sense of it?
+        description: Paths to the nginx access log file.
+        type: text
+        multi: true
+        default:
+          - /var/log/nginx/access.log*
+        # I suggest to use ECS fields for this config options here: https://github.com/elastic/ecs/blob/master/schemas/os.yml
+        # This would need to be based on a predefined definition on what can be filtered on
+        os.darwin:
+          default:
+            - /usr/local/var/log/nginx/access.log*
+        os.windows:
+          default:
+            - c:/programdata/nginx/logs/*access.log*

testdata/public/package/datasources-1.0.0/index.json

@@ -0,0 +1,104 @@
+{
+  "name": "datasources",
+  "title": "Default datasource Integration",
+  "version": "1.0.0",
+  "description": "Package with data sources",
+  "type": "integration",
+  "categories": [
+    "logs"
+  ],
+  "requirement": {
+    "kibana": {}
+  },
+  "assets": [
+    "/package/datasources-1.0.0/manifest.yml",
+    "/package/datasources-1.0.0/dataset/example/manifest.yml"
+  ],
+  "format_version": "1.0.0",
+  "datasets": [
+    {
+      "title": "Example dataset with inputs",
+      "name": "example",
+      "release": "beta",
+      "type": "logs",
+      "inputs": [
+        {
+          "type": "log",
+          "vars": [
+            {
+              "default": [
+                "/var/log/nginx/access.log*"
+              ],
+              "description": "Paths to the nginx access log file.",
+              "multi": true,
+              "name": "paths",
+              "os": {
+                "darwin": {
+                  "default": [
+                    "/usr/local/var/log/nginx/access.log*"
+                  ]
+                },
+                "windows": {
+                  "default": [
+                    "c:/programdata/nginx/logs/*access.log*"
+                  ]
+                }
+              },
+              "required": true,
+              "type": "text"
+            }
+          ]
+        }
+      ],
+      "package": "datasources"
+    }
+  ],
+  "datasources": [
+    {
+      "name": "nginx",
+      "inputs": [
+        {
+          "descrition": "Collecting metrics for nginx.",
+          "type": "nginx/metrics",
+          "vars": [
+            {
+              "default": [
+                "http://127.0.0.1"
+              ],
+              "description": "Nginx hosts",
+              "multi": true,
+              "name": "hosts",
+              "required": true,
+              "type": "text"
+            },
+            {
+              "default": "10s",
+              "description": "Collection period. Valid values: 10s, 5m, 2h",
+              "name": "period",
+              "type": "duration"
+            },
+            {
+              "name": "username",
+              "type": "text"
+            },
+            {
+              "name": "password",
+              "type": "password"
+            }
+          ]
+        },
+        {
+          "description": "Collect nginx logs.",
+          "type": "logs",
+          "vars": null
+        },
+        {
+          "type": "syslog",
+          "vars": null
+        }
+      ]
+    }
+  ],
+  "download": "/epr/datasources/datasources-1.0.0.tar.gz",
+  "path": "/package/datasources-1.0.0"
+}

testdata/public/package/datasources-1.0.0/manifest.yml

@@ -0,0 +1,56 @@
+format_version: 1.0.0
+
+name: datasources
+description: Package with data sources
+version: 1.0.0
+title: Default datasource Integration
+categories: ["logs"]
+type: integration
+
+
+datasources:
+  -
+    # Do we need a name for the data source?
+    name: nginx
+
+    # List of inputs this datasource supports
+    inputs:
+      -
+        # An id can be given, in case the type used here is not unique
+        # This is for selection in the stream
+        # id: nginx
+        type: nginx/metrics
+        descrition: Collecting metrics for nginx.
+
+        # Common configuration options for this input
+        vars:
+          - name: hosts
+            description: Nginx hosts
+            default:
+              ["http://127.0.0.1"]
+            # All the config options that are required should be shown in the UI
+            required: true
+            multi: true
+            type: text
+          - name: period
+            description: "Collection period. Valid values: 10s, 5m, 2h"
+            default: "10s"
+            type: duration
+          - name: username
+            type: text
+          - name: password
+            # This is the html input type?
+            type: password
+
+      -
+        type: logs
+        description: Collect nginx logs.
+
+        # Common configuration options for this input
+        vars:
+
+      -
+        type: syslog
+
+        # Common configuration options for this input
+        vars:

util/dataset.go

@@ -18,6 +18,7 @@ type DataSet struct {
 	Type           string                   `config:"type" json:"type" validate:"required"`
 	IngestPipeline string                   `config:"ingest_pipeline,omitempty" config:"ingest_pipeline" json:"ingest_pipeline,omitempty"`
 	Vars           []map[string]interface{} `config:"vars" json:"vars,omitempty"`
+	Inputs         []map[string]interface{} `config:"inputs" json:"inputs,omitempty"`
 	Package        string                   `json:"package"`
 }
 

util/package.go

@@ -31,18 +31,24 @@ type Package struct {
 	Version       string  `config:"version" json:"version"`
 	Readme        *string `config:"readme,omitempty" json:"readme,omitempty"`
 	versionSemVer semver.Version
-	Description   string      `config:"description" json:"description"`
-	Type          string      `config:"type" json:"type"`
-	Categories    []string    `config:"categories" json:"categories"`
-	Requirement   Requirement `config:"requirement" json:"requirement"`
-	Screenshots   []Image     `config:"screenshots,omitempty" json:"screenshots,omitempty"`
-	Icons         []Image     `config:"icons,omitempty" json:"icons,omitempty"`
-	Assets        []string    `config:"assets,omitempty" json:"assets,omitempty"`
-	Internal      bool        `config:"internal,omitempty" json:"internal,omitempty"`
-	FormatVersion string      `config:"format_version" json:"format_version"`
-	DataSets      []*DataSet  `config:"datasets,omitempty" json:"datasets,omitempty"`
-	Download      string      `json:"download"`
-	Path          string      `json:"path"`
+	Description   string       `config:"description" json:"description"`
+	Type          string       `config:"type" json:"type"`
+	Categories    []string     `config:"categories" json:"categories"`
+	Requirement   Requirement  `config:"requirement" json:"requirement"`
+	Screenshots   []Image      `config:"screenshots,omitempty" json:"screenshots,omitempty"`
+	Icons         []Image      `config:"icons,omitempty" json:"icons,omitempty"`
+	Assets        []string     `config:"assets,omitempty" json:"assets,omitempty"`
+	Internal      bool         `config:"internal,omitempty" json:"internal,omitempty"`
+	FormatVersion string       `config:"format_version" json:"format_version"`
+	DataSets      []*DataSet   `config:"datasets,omitempty" json:"datasets,omitempty"`
+	Datasources   []Datasource `config:"datasources,omitempty" json:"datasources,omitempty"`
+	Download      string       `json:"download"`
+	Path          string       `json:"path"`
+}
+
+type Datasource struct {
+	Name   string        `config:"name" json:"name"`
+	Inputs []interface{} `config:"inputs" json:"inputs"`
 }
 
 type Requirement struct {