diff --git a/CHANGELOG.md b/CHANGELOG.md index e05a49b23..2562326bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Add path field to search and package info endpoint. [#174](https://github.com/elastic/integrations-registry/pull/174) * Add download field to package info endpoint. [#174](https://github.com/elastic/integrations-registry/pull/174) * Add `package` field to dataset. [#189](https://github.com/elastic/integrations-registry/pull/189) +* Add support for datasources. [#216](https://github.com/elastic/integrations-registry/pull/216) [#212](https://github.com/elastic/integrations-registry/pull/212) ### Deprecated diff --git a/docs/api/categories.json b/docs/api/categories.json index 207f7e869..c98666479 100644 --- a/docs/api/categories.json +++ b/docs/api/categories.json @@ -2,7 +2,7 @@ { "id": "logs", "title": "Logs", - "count": 2 + "count": 3 }, { "id": "metrics", diff --git a/docs/api/search-category-logs.json b/docs/api/search-category-logs.json index 0aad02b8c..c131d66fa 100644 --- a/docs/api/search-category-logs.json +++ b/docs/api/search-category-logs.json @@ -1,4 +1,13 @@ [ + { + "description": "Package with data sources", + "download": "/epr/datasources/datasources-1.0.0.tar.gz", + "name": "datasources", + "path": "/package/datasources-1.0.0", + "title": "Default datasource Integration", + "type": "integration", + "version": "1.0.0" + }, { "description": "Tests if no pipeline is set, it defaults to the default one", "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz", diff --git a/docs/api/search-kibana652.json b/docs/api/search-kibana652.json index 9d6c3b6f4..dc596e3e9 100644 --- a/docs/api/search-kibana652.json +++ b/docs/api/search-kibana652.json @@ -1,4 +1,13 @@ [ + { + "description": "Package with data sources", + "download": "/epr/datasources/datasources-1.0.0.tar.gz", + "name": "datasources", + "path": "/package/datasources-1.0.0", + "title": "Default datasource Integration", + "type": "integration", + "version": "1.0.0" + }, { "description": "Tests if no pipeline is set, it defaults to the default one", "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz", diff --git a/docs/api/search-kibana721.json b/docs/api/search-kibana721.json index 2315ef242..6cbc4a36c 100644 --- a/docs/api/search-kibana721.json +++ b/docs/api/search-kibana721.json @@ -1,4 +1,13 @@ [ + { + "description": "Package with data sources", + "download": "/epr/datasources/datasources-1.0.0.tar.gz", + "name": "datasources", + "path": "/package/datasources-1.0.0", + "title": "Default datasource Integration", + "type": "integration", + "version": "1.0.0" + }, { "description": "Tests if no pipeline is set, it defaults to the default one", "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz", diff --git a/docs/api/search-package-internal.json b/docs/api/search-package-internal.json index a9a56db2e..f1222ed06 100644 --- a/docs/api/search-package-internal.json +++ b/docs/api/search-package-internal.json @@ -1,4 +1,13 @@ [ + { + "description": "Package with data sources", + "download": "/epr/datasources/datasources-1.0.0.tar.gz", + "name": "datasources", + "path": "/package/datasources-1.0.0", + "title": "Default datasource Integration", + "type": "integration", + "version": "1.0.0" + }, { "description": "Tests if no pipeline is set, it defaults to the default one", "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz", diff --git a/docs/api/search.json b/docs/api/search.json index 2315ef242..6cbc4a36c 100644 --- a/docs/api/search.json +++ b/docs/api/search.json @@ -1,4 +1,13 @@ [ + { + "description": "Package with data sources", + "download": "/epr/datasources/datasources-1.0.0.tar.gz", + "name": "datasources", + "path": "/package/datasources-1.0.0", + "title": "Default datasource Integration", + "type": "integration", + "version": "1.0.0" + }, { "description": "Tests if no pipeline is set, it defaults to the default one", "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz", diff --git a/testdata/package/datasources-1.0.0/dataset/example/manifest.yml b/testdata/package/datasources-1.0.0/dataset/example/manifest.yml new file mode 100644 index 000000000..96926e5a6 --- /dev/null +++ b/testdata/package/datasources-1.0.0/dataset/example/manifest.yml @@ -0,0 +1,23 @@ +title: Example dataset with inputs +type: logs + +# List of supported inputs +inputs: + - type: log + vars: + - name: paths + required: true + # Should we define this as array? How will the UI best make sense of it? + description: Paths to the nginx access log file. + type: text + multi: true + default: + - /var/log/nginx/access.log* + # I suggest to use ECS fields for this config options here: https://github.com/elastic/ecs/blob/master/schemas/os.yml + # This would need to be based on a predefined definition on what can be filtered on + os.darwin: + default: + - /usr/local/var/log/nginx/access.log* + os.windows: + default: + - c:/programdata/nginx/logs/*access.log* diff --git a/testdata/package/datasources-1.0.0/manifest.yml b/testdata/package/datasources-1.0.0/manifest.yml new file mode 100644 index 000000000..3007c585a --- /dev/null +++ b/testdata/package/datasources-1.0.0/manifest.yml @@ -0,0 +1,56 @@ +format_version: 1.0.0 + +name: datasources +description: Package with data sources +version: 1.0.0 +title: Default datasource Integration +categories: ["logs"] +type: integration + + +datasources: + - + # Do we need a name for the data source? + name: nginx + + # List of inputs this datasource supports + inputs: + - + # An id can be given, in case the type used here is not unique + # This is for selection in the stream + # id: nginx + type: nginx/metrics + descrition: Collecting metrics for nginx. + + # Common configuration options for this input + vars: + - name: hosts + description: Nginx hosts + default: + ["http://127.0.0.1"] + # All the config options that are required should be shown in the UI + required: true + multi: true + type: text + - name: period + description: "Collection period. Valid values: 10s, 5m, 2h" + default: "10s" + type: duration + - name: username + type: text + - name: password + # This is the html input type? + type: password + + - + type: logs + description: Collect nginx logs. + + # Common configuration options for this input + vars: + + - + type: syslog + + # Common configuration options for this input + vars: diff --git a/testdata/public/package/datasources-1.0.0/dataset/example/manifest.yml b/testdata/public/package/datasources-1.0.0/dataset/example/manifest.yml new file mode 100644 index 000000000..96926e5a6 --- /dev/null +++ b/testdata/public/package/datasources-1.0.0/dataset/example/manifest.yml @@ -0,0 +1,23 @@ +title: Example dataset with inputs +type: logs + +# List of supported inputs +inputs: + - type: log + vars: + - name: paths + required: true + # Should we define this as array? How will the UI best make sense of it? + description: Paths to the nginx access log file. + type: text + multi: true + default: + - /var/log/nginx/access.log* + # I suggest to use ECS fields for this config options here: https://github.com/elastic/ecs/blob/master/schemas/os.yml + # This would need to be based on a predefined definition on what can be filtered on + os.darwin: + default: + - /usr/local/var/log/nginx/access.log* + os.windows: + default: + - c:/programdata/nginx/logs/*access.log* diff --git a/testdata/public/package/datasources-1.0.0/index.json b/testdata/public/package/datasources-1.0.0/index.json new file mode 100644 index 000000000..eb8bcfb5e --- /dev/null +++ b/testdata/public/package/datasources-1.0.0/index.json @@ -0,0 +1,104 @@ +{ + "name": "datasources", + "title": "Default datasource Integration", + "version": "1.0.0", + "description": "Package with data sources", + "type": "integration", + "categories": [ + "logs" + ], + "requirement": { + "kibana": {} + }, + "assets": [ + "/package/datasources-1.0.0/manifest.yml", + "/package/datasources-1.0.0/dataset/example/manifest.yml" + ], + "format_version": "1.0.0", + "datasets": [ + { + "title": "Example dataset with inputs", + "name": "example", + "release": "beta", + "type": "logs", + "inputs": [ + { + "type": "log", + "vars": [ + { + "default": [ + "/var/log/nginx/access.log*" + ], + "description": "Paths to the nginx access log file.", + "multi": true, + "name": "paths", + "os": { + "darwin": { + "default": [ + "/usr/local/var/log/nginx/access.log*" + ] + }, + "windows": { + "default": [ + "c:/programdata/nginx/logs/*access.log*" + ] + } + }, + "required": true, + "type": "text" + } + ] + } + ], + "package": "datasources" + } + ], + "datasources": [ + { + "name": "nginx", + "inputs": [ + { + "descrition": "Collecting metrics for nginx.", + "type": "nginx/metrics", + "vars": [ + { + "default": [ + "http://127.0.0.1" + ], + "description": "Nginx hosts", + "multi": true, + "name": "hosts", + "required": true, + "type": "text" + }, + { + "default": "10s", + "description": "Collection period. Valid values: 10s, 5m, 2h", + "name": "period", + "type": "duration" + }, + { + "name": "username", + "type": "text" + }, + { + "name": "password", + "type": "password" + } + ] + }, + { + "description": "Collect nginx logs.", + "type": "logs", + "vars": null + }, + { + "type": "syslog", + "vars": null + } + ] + } + ], + "download": "/epr/datasources/datasources-1.0.0.tar.gz", + "path": "/package/datasources-1.0.0" +} \ No newline at end of file diff --git a/testdata/public/package/datasources-1.0.0/manifest.yml b/testdata/public/package/datasources-1.0.0/manifest.yml new file mode 100644 index 000000000..3007c585a --- /dev/null +++ b/testdata/public/package/datasources-1.0.0/manifest.yml @@ -0,0 +1,56 @@ +format_version: 1.0.0 + +name: datasources +description: Package with data sources +version: 1.0.0 +title: Default datasource Integration +categories: ["logs"] +type: integration + + +datasources: + - + # Do we need a name for the data source? + name: nginx + + # List of inputs this datasource supports + inputs: + - + # An id can be given, in case the type used here is not unique + # This is for selection in the stream + # id: nginx + type: nginx/metrics + descrition: Collecting metrics for nginx. + + # Common configuration options for this input + vars: + - name: hosts + description: Nginx hosts + default: + ["http://127.0.0.1"] + # All the config options that are required should be shown in the UI + required: true + multi: true + type: text + - name: period + description: "Collection period. Valid values: 10s, 5m, 2h" + default: "10s" + type: duration + - name: username + type: text + - name: password + # This is the html input type? + type: password + + - + type: logs + description: Collect nginx logs. + + # Common configuration options for this input + vars: + + - + type: syslog + + # Common configuration options for this input + vars: diff --git a/util/dataset.go b/util/dataset.go index 0e071c2cb..91072db5b 100644 --- a/util/dataset.go +++ b/util/dataset.go @@ -18,6 +18,7 @@ type DataSet struct { Type string `config:"type" json:"type" validate:"required"` IngestPipeline string `config:"ingest_pipeline,omitempty" config:"ingest_pipeline" json:"ingest_pipeline,omitempty"` Vars []map[string]interface{} `config:"vars" json:"vars,omitempty"` + Inputs []map[string]interface{} `config:"inputs" json:"inputs,omitempty"` Package string `json:"package"` } diff --git a/util/package.go b/util/package.go index cbf90857d..9345ae83a 100644 --- a/util/package.go +++ b/util/package.go @@ -31,18 +31,24 @@ type Package struct { Version string `config:"version" json:"version"` Readme *string `config:"readme,omitempty" json:"readme,omitempty"` versionSemVer semver.Version - Description string `config:"description" json:"description"` - Type string `config:"type" json:"type"` - Categories []string `config:"categories" json:"categories"` - Requirement Requirement `config:"requirement" json:"requirement"` - Screenshots []Image `config:"screenshots,omitempty" json:"screenshots,omitempty"` - Icons []Image `config:"icons,omitempty" json:"icons,omitempty"` - Assets []string `config:"assets,omitempty" json:"assets,omitempty"` - Internal bool `config:"internal,omitempty" json:"internal,omitempty"` - FormatVersion string `config:"format_version" json:"format_version"` - DataSets []*DataSet `config:"datasets,omitempty" json:"datasets,omitempty"` - Download string `json:"download"` - Path string `json:"path"` + Description string `config:"description" json:"description"` + Type string `config:"type" json:"type"` + Categories []string `config:"categories" json:"categories"` + Requirement Requirement `config:"requirement" json:"requirement"` + Screenshots []Image `config:"screenshots,omitempty" json:"screenshots,omitempty"` + Icons []Image `config:"icons,omitempty" json:"icons,omitempty"` + Assets []string `config:"assets,omitempty" json:"assets,omitempty"` + Internal bool `config:"internal,omitempty" json:"internal,omitempty"` + FormatVersion string `config:"format_version" json:"format_version"` + DataSets []*DataSet `config:"datasets,omitempty" json:"datasets,omitempty"` + Datasources []Datasource `config:"datasources,omitempty" json:"datasources,omitempty"` + Download string `json:"download"` + Path string `json:"path"` +} + +type Datasource struct { + Name string `config:"name" json:"name"` + Inputs []interface{} `config:"inputs" json:"inputs"` } type Requirement struct {