From e97377d00399e41cd21397db6e7cc31e5c2856a4 Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Fri, 21 Apr 2017 12:55:50 -0700 Subject: [PATCH 01/40] Warn Beats users not to do multiline handling in Logstash --- filebeat/docs/multiline.asciidoc | 11 +++++++++-- .../reference/configuration/filebeat-options.asciidoc | 4 ++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/filebeat/docs/multiline.asciidoc b/filebeat/docs/multiline.asciidoc index ebaf979ec7a..34ae182c275 100644 --- a/filebeat/docs/multiline.asciidoc +++ b/filebeat/docs/multiline.asciidoc @@ -1,8 +1,15 @@ [[multiline-examples]] == Managing Multiline Messages -You can specify `multiline` settings in the +{beatname_lc}.yml+ file to control how Filebeat deals with messages that -span multiple lines. At a minimum, you need to configure: +The files harvested by {beatname_uc} may contain messages that span multiple lines of text. In order to correctly handle +these multiline events, you need to configure `multiline` settings in the +{beatname_lc}.yml+ file to specify which +lines are part of a single event. + +IMPORTANT: If you are sending multiline events to Logstash, use the options described here to handle multiline events +before sending the event data to Logstash. Trying to implement multiline event handling in Logstash (for example, by +using the Logstash multiline codec) may result in the mixing of streams and corrupted data. + +At a minimum, you need to configure these `multiline` options: * the `pattern` option, which specifies a regular expression. Depending on how you configure other multiline options, lines that match the specified regular expression are considered either continuations of a previous line or the start of a new multiline event. You can set the `negate` option to negate the pattern. diff --git a/filebeat/docs/reference/configuration/filebeat-options.asciidoc b/filebeat/docs/reference/configuration/filebeat-options.asciidoc index 342ee30e21b..dd072016d7e 100644 --- a/filebeat/docs/reference/configuration/filebeat-options.asciidoc +++ b/filebeat/docs/reference/configuration/filebeat-options.asciidoc @@ -346,6 +346,10 @@ occur. [[multiline]] ===== multiline +IMPORTANT: If you are sending multiline events to Logstash, use the options described here to handle multiline events +before sending the event data to Logstash. Trying to implement multiline event handling in Logstash (for example, by +using the Logstash multiline codec) may result in the mixing of streams and corrupted data. + Options that control how Filebeat deals with log messages that span multiple lines. Multiline messages are common in files that contain Java stack traces. The following example shows how to configure Filebeat to handle a multiline message where the first line of the message begins with a bracket (`[`). From 4f061e2464fd0044fef51b0e3979cfeecdced0f9 Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Fri, 21 Apr 2017 16:44:24 +0200 Subject: [PATCH 02/40] Initial commit http metricbeat module --- metricbeat/docs/modules_list.asciidoc | 2 ++ metricbeat/include/list.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 418728dc554..dcab36e12ab 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -10,6 +10,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -37,6 +38,7 @@ include::modules/dropwizard.asciidoc[] include::modules/elasticsearch.asciidoc[] include::modules/golang.asciidoc[] include::modules/haproxy.asciidoc[] +include::modules/http.asciidoc[] include::modules/jolokia.asciidoc[] include::modules/kafka.asciidoc[] include::modules/kibana.asciidoc[] diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 565b1b5df1e..c3da777c3b3 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -39,6 +39,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/haproxy" _ "github.com/elastic/beats/metricbeat/module/haproxy/info" _ "github.com/elastic/beats/metricbeat/module/haproxy/stat" + _ "github.com/elastic/beats/metricbeat/module/http" + _ "github.com/elastic/beats/metricbeat/module/http/json" _ "github.com/elastic/beats/metricbeat/module/jolokia" _ "github.com/elastic/beats/metricbeat/module/jolokia/jmx" _ "github.com/elastic/beats/metricbeat/module/kafka" From e750f90c9cb1264ddca0f1b66e2df897bad72270 Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Fri, 21 Apr 2017 16:44:39 +0200 Subject: [PATCH 03/40] Initial commit http metricbeat module --- metricbeat/docs/modules/http.asciidoc | 37 +++++++++++++ metricbeat/docs/modules/http/json.asciidoc | 19 +++++++ metricbeat/module/http/_meta/config.yml | 6 +++ metricbeat/module/http/_meta/docs.asciidoc | 4 ++ metricbeat/module/http/_meta/fields.yml | 9 ++++ metricbeat/module/http/doc.go | 4 ++ metricbeat/module/http/json/_meta/data.json | 19 +++++++ .../module/http/json/_meta/docs.asciidoc | 3 ++ metricbeat/module/http/json/_meta/fields.yml | 9 ++++ metricbeat/module/http/json/json.go | 53 +++++++++++++++++++ 10 files changed, 163 insertions(+) create mode 100644 metricbeat/docs/modules/http.asciidoc create mode 100644 metricbeat/docs/modules/http/json.asciidoc create mode 100644 metricbeat/module/http/_meta/config.yml create mode 100644 metricbeat/module/http/_meta/docs.asciidoc create mode 100644 metricbeat/module/http/_meta/fields.yml create mode 100644 metricbeat/module/http/doc.go create mode 100644 metricbeat/module/http/json/_meta/data.json create mode 100644 metricbeat/module/http/json/_meta/docs.asciidoc create mode 100644 metricbeat/module/http/json/_meta/fields.yml create mode 100644 metricbeat/module/http/json/json.go diff --git a/metricbeat/docs/modules/http.asciidoc b/metricbeat/docs/modules/http.asciidoc new file mode 100644 index 00000000000..e4f6e470323 --- /dev/null +++ b/metricbeat/docs/modules/http.asciidoc @@ -0,0 +1,37 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-module-http]] +== http Module + +This is the http Module. + + + +[float] +=== Example Configuration + +The http module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: http + metricsets: ["json"] + enabled: true + period: 10s + hosts: ["localhost"] + +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::http/json.asciidoc[] + diff --git a/metricbeat/docs/modules/http/json.asciidoc b/metricbeat/docs/modules/http/json.asciidoc new file mode 100644 index 00000000000..f9173b08d8a --- /dev/null +++ b/metricbeat/docs/modules/http/json.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-http-json]] +include::../../../module/http/json/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/http/json/_meta/data.json[] +---- diff --git a/metricbeat/module/http/_meta/config.yml b/metricbeat/module/http/_meta/config.yml new file mode 100644 index 00000000000..b74b5214045 --- /dev/null +++ b/metricbeat/module/http/_meta/config.yml @@ -0,0 +1,6 @@ +- module: http + metricsets: ["json"] + enabled: true + period: 10s + hosts: ["localhost"] + diff --git a/metricbeat/module/http/_meta/docs.asciidoc b/metricbeat/module/http/_meta/docs.asciidoc new file mode 100644 index 00000000000..72f5d9cae11 --- /dev/null +++ b/metricbeat/module/http/_meta/docs.asciidoc @@ -0,0 +1,4 @@ +== http Module + +This is the http Module. + diff --git a/metricbeat/module/http/_meta/fields.yml b/metricbeat/module/http/_meta/fields.yml new file mode 100644 index 00000000000..82c8ea4aee8 --- /dev/null +++ b/metricbeat/module/http/_meta/fields.yml @@ -0,0 +1,9 @@ +- key: http + title: "http" + description: > + http Module + fields: + - name: http + type: group + description: > + fields: diff --git a/metricbeat/module/http/doc.go b/metricbeat/module/http/doc.go new file mode 100644 index 00000000000..fadec87d6e7 --- /dev/null +++ b/metricbeat/module/http/doc.go @@ -0,0 +1,4 @@ +/* +Package http is a Metricbeat module that contains MetricSets. +*/ +package http diff --git a/metricbeat/module/http/json/_meta/data.json b/metricbeat/module/http/json/_meta/data.json new file mode 100644 index 00000000000..7324e6a3c68 --- /dev/null +++ b/metricbeat/module/http/json/_meta/data.json @@ -0,0 +1,19 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"http", + "name":"json", + "rtt":44269 + }, + "http":{ + "json":{ + "example": "json" + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/http/json/_meta/docs.asciidoc b/metricbeat/module/http/json/_meta/docs.asciidoc new file mode 100644 index 00000000000..8094e53468c --- /dev/null +++ b/metricbeat/module/http/json/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +=== http json MetricSet + +This is the json metricset of the module http. diff --git a/metricbeat/module/http/json/_meta/fields.yml b/metricbeat/module/http/json/_meta/fields.yml new file mode 100644 index 00000000000..29e0c11cb8f --- /dev/null +++ b/metricbeat/module/http/json/_meta/fields.yml @@ -0,0 +1,9 @@ +- name: json + type: group + description: > + json + fields: + - name: example + type: keyword + description: > + Example field diff --git a/metricbeat/module/http/json/json.go b/metricbeat/module/http/json/json.go new file mode 100644 index 00000000000..9893064ad6e --- /dev/null +++ b/metricbeat/module/http/json/json.go @@ -0,0 +1,53 @@ +package json + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + if err := mb.Registry.AddMetricSet("http", "json", New); err != nil { + panic(err) + } +} + +// MetricSet type defines all fields of the MetricSet +// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with +// additional entries. These variables can be used to persist data or configuration between +// multiple fetch calls. +type MetricSet struct { + mb.BaseMetricSet + counter int +} + +// New create a new instance of the MetricSet +// Part of new is also setting up the configuration by processing additional +// configuration entries if needed. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + + config := struct{}{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + counter: 1, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch() (common.MapStr, error) { + + event := common.MapStr{ + "counter": m.counter, + } + m.counter++ + + return event, nil +} From 1977b337855150007900bada03da16011ac5fa16 Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Sun, 23 Apr 2017 07:50:23 +0200 Subject: [PATCH 04/40] Add first fields information Added first request/event --- metricbeat/module/http/json/_meta/fields.yml | 22 ++++++++-- metricbeat/module/http/json/json.go | 44 +++++++++++++++++--- 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/metricbeat/module/http/json/_meta/fields.yml b/metricbeat/module/http/json/_meta/fields.yml index 29e0c11cb8f..ef432c3dbda 100644 --- a/metricbeat/module/http/json/_meta/fields.yml +++ b/metricbeat/module/http/json/_meta/fields.yml @@ -1,9 +1,23 @@ - name: json type: group description: > - json + json metricset fields: - - name: example - type: keyword + - name: request + type: group description: > - Example field + HTTP request information + fields: + - name: method + type: string + description: > + The HTTP method used + - name: response + type: group + description: > + HTTP response information + fields: + - name: body + type: string + description: > + The HTTP payload returned diff --git a/metricbeat/module/http/json/json.go b/metricbeat/module/http/json/json.go index 9893064ad6e..86b1644032b 100644 --- a/metricbeat/module/http/json/json.go +++ b/metricbeat/module/http/json/json.go @@ -2,6 +2,7 @@ package json import ( "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" ) @@ -19,7 +20,9 @@ func init() { // multiple fetch calls. type MetricSet struct { mb.BaseMetricSet - counter int + http *helper.HTTP + body string + headers map[string]string } // New create a new instance of the MetricSet @@ -27,15 +30,28 @@ type MetricSet struct { // configuration entries if needed. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - config := struct{}{} + config := struct { + Method string `config:"method"` + Body string `config:"body"` + Headers map[string]string `config:"headers"` + }{} if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } + http := helper.NewHTTP(base) + http.SetMethod(config.Method) + http.SetBody([]byte(config.Body)) + for key, value := range config.Headers { + http.SetHeader(key,value) + } + return &MetricSet{ BaseMetricSet: base, - counter: 1, + http: http, + body: config.Body, + headers: config.Headers, }, nil } @@ -44,10 +60,28 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // descriptive error must be returned. func (m *MetricSet) Fetch() (common.MapStr, error) { + response, err := m.http.FetchResponse() + if err != nil { + return nil, err + } + defer response.Body.Close() + event := common.MapStr{ - "counter": m.counter, + "response.status_code": response.StatusCode, + } + + event["request"] = common.MapStr{ + "header": response.Request.Header, + "method": response.Request.Method, + "body": response.Request.Body, + + } + + event["response"] = common.MapStr{ + "status_code": response.StatusCode, + "header": response.Header, + "body": response.Body, } - m.counter++ return event, nil } From 22428835dd9beec94b5f50fc61dc84cace604c61 Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Sun, 23 Apr 2017 13:29:12 +0200 Subject: [PATCH 05/40] Add first fields information Added first request/event --- metricbeat/module/http/json/_meta/fields.yml | 22 ++++++-- metricbeat/module/http/json/json.go | 55 ++++++++++++++------ 2 files changed, 58 insertions(+), 19 deletions(-) diff --git a/metricbeat/module/http/json/_meta/fields.yml b/metricbeat/module/http/json/_meta/fields.yml index ef432c3dbda..4d0148e3df4 100644 --- a/metricbeat/module/http/json/_meta/fields.yml +++ b/metricbeat/module/http/json/_meta/fields.yml @@ -8,16 +8,32 @@ description: > HTTP request information fields: + - name: header + type: nested + description: > + The HTTP headers sent - name: method - type: string + type: keyword description: > The HTTP method used + - name: body + type: keyword + description: > + The HTTP payload sent - name: response type: group description: > HTTP response information fields: + - name: header + type: nested + description: > + The HTTP headers received + - name: status_code + type: keyword + description: > + The HTTP status code - name: body - type: string + type: keyword description: > - The HTTP payload returned + The HTTP payload received diff --git a/metricbeat/module/http/json/json.go b/metricbeat/module/http/json/json.go index 86b1644032b..3bb9629851e 100644 --- a/metricbeat/module/http/json/json.go +++ b/metricbeat/module/http/json/json.go @@ -4,6 +4,9 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" + "net/http" + "strings" + "io/ioutil" ) // init registers the MetricSet with the central registry. @@ -20,9 +23,11 @@ func init() { // multiple fetch calls. type MetricSet struct { mb.BaseMetricSet - http *helper.HTTP - body string - headers map[string]string + http *helper.HTTP + headers map[string]string + method string + body string + } // New create a new instance of the MetricSet @@ -31,9 +36,9 @@ type MetricSet struct { func New(base mb.BaseMetricSet) (mb.MetricSet, error) { config := struct { - Method string `config:"method"` - Body string `config:"body"` - Headers map[string]string `config:"headers"` + Method string `config:"method"` + Body string `config:"body"` + Headers map[string]string `config:"headers"` }{} if err := base.Module().UnpackConfig(&config); err != nil { @@ -44,14 +49,16 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { http.SetMethod(config.Method) http.SetBody([]byte(config.Body)) for key, value := range config.Headers { - http.SetHeader(key,value) + http.SetHeader(key, value) } return &MetricSet{ BaseMetricSet: base, http: http, - body: config.Body, headers: config.Headers, + method: config.Method, + body: config.Body, + }, nil } @@ -66,22 +73,38 @@ func (m *MetricSet) Fetch() (common.MapStr, error) { } defer response.Body.Close() - event := common.MapStr{ - "response.status_code": response.StatusCode, + responseBody, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, err } - event["request"] = common.MapStr{ - "header": response.Request.Header, - "method": response.Request.Method, - "body": response.Request.Body, + event := common.MapStr{} + event["request"] = common.MapStr{ + "headers": m.headers, + "method": m.method, + "body": m.body, } event["response"] = common.MapStr{ "status_code": response.StatusCode, - "header": response.Header, - "body": response.Body, + "headers": m.getHeaders(response.Header), + "body": responseBody, } return event, nil } + +func (m *MetricSet) getHeaders(header http.Header) map[string]string { + + headers := make(map[string]string) + for k, v := range header { + value := "" + for _, h := range v { + value += h + " ," + } + value = strings.TrimRight(value, " ,") + headers[k] = value + } + return headers +} From c4426022655993e800cd36de9cfbe568a9060d8b Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Sun, 23 Apr 2017 22:04:51 +0200 Subject: [PATCH 06/40] Updated config and docu files. --- metricbeat/docs/fields.asciidoc | 83 ++++++++++++++++++++ metricbeat/metricbeat.full.yml | 8 ++ metricbeat/module/http/json/_meta/fields.yml | 4 +- 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index d773d07267a..9d96cad1632 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -23,6 +23,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -3443,6 +3444,88 @@ type: integer The average queue time in ms over the last 1024 requests. +[[exported-fields-http]] +== http Fields + +http Module + + + +[float] +== http Fields + + + + +[float] +== json Fields + +json metricset + + + +[float] +== request Fields + +HTTP request information + + + +[float] +=== http.json.request.header + +type: nested + +The HTTP headers sent + + +[float] +=== http.json.request.method + +type: keyword + +The HTTP method used + + +[float] +=== http.json.request.body + +type: keyword + +The HTTP payload sent + + +[float] +== response Fields + +HTTP response information + + + +[float] +=== http.json.response.header + +type: nested + +The HTTP headers received + + +[float] +=== http.json.response.status_code + +type: keyword + +The HTTP status code + + +[float] +=== http.json.response.body + +type: keyword + +The HTTP payload received + + [[exported-fields-jolokia]] == Jolokia Fields diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index cecf656b791..c152da4e7e1 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -160,6 +160,14 @@ metricbeat.modules: period: 10s hosts: ["tcp://127.0.0.1:14567"] +#-------------------------------- http Module -------------------------------- +- module: http + metricsets: ["json"] + enabled: true + period: 10s + hosts: ["localhost"] + + #------------------------------- Jolokia Module ------------------------------ - module: jolokia metricsets: ["jmx"] diff --git a/metricbeat/module/http/json/_meta/fields.yml b/metricbeat/module/http/json/_meta/fields.yml index 4d0148e3df4..be59c8ef14b 100644 --- a/metricbeat/module/http/json/_meta/fields.yml +++ b/metricbeat/module/http/json/_meta/fields.yml @@ -11,7 +11,7 @@ - name: header type: nested description: > - The HTTP headers sent + The HTTP headers sent - name: method type: keyword description: > @@ -28,7 +28,7 @@ - name: header type: nested description: > - The HTTP headers received + The HTTP headers received - name: status_code type: keyword description: > From 621f763b2c6095314a4e0a66d0c33384bc9d0210 Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Sun, 23 Apr 2017 22:12:03 +0200 Subject: [PATCH 07/40] Fix fmt findings --- metricbeat/module/http/json/json.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/http/json/json.go b/metricbeat/module/http/json/json.go index 3bb9629851e..e7c7070a3ed 100644 --- a/metricbeat/module/http/json/json.go +++ b/metricbeat/module/http/json/json.go @@ -4,9 +4,9 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" + "io/ioutil" "net/http" "strings" - "io/ioutil" ) // init registers the MetricSet with the central registry. @@ -25,9 +25,8 @@ type MetricSet struct { mb.BaseMetricSet http *helper.HTTP headers map[string]string - method string + method string body string - } // New create a new instance of the MetricSet @@ -58,7 +57,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { headers: config.Headers, method: config.Method, body: config.Body, - }, nil } From 282d81d8603c10a5d80b134def299c8ac801d10a Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 24 Apr 2017 09:54:10 +0200 Subject: [PATCH 08/40] Update CONTRIBUTING.md to Golang 1.8.1 (#4094) Update of the Contributing guide was missed in https://github.com/elastic/beats/pull/4033. --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 44257d94f4a..8425cad4866 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,7 +51,7 @@ Beats](https://github.com/elastic/beats/blob/master/libbeat/docs/communitybeats. The Beats are Go programs, so install the latest version of [golang](http://golang.org/) if you don't have it already. The current Go version -used for development is Golang 1.7.4. +used for development is Golang 1.8.1. The location where you clone is important. Please clone under the source directory of your `GOPATH`. If you don't have `GOPATH` already set, you can From 3efb9dbf10e0a86d97632b1d084b5811b99db29d Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 24 Apr 2017 11:11:23 +0200 Subject: [PATCH 09/40] Allow to overwrite uri in http helper (#4084) This allows if a second request is needed with a different path in a metricset to reuse the http helper and hostparser and only overwrite the path temporarly. --- metricbeat/helper/http.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/metricbeat/helper/http.go b/metricbeat/helper/http.go index cc397c5b73f..7d50dcf8fbc 100644 --- a/metricbeat/helper/http.go +++ b/metricbeat/helper/http.go @@ -19,6 +19,7 @@ type HTTP struct { base mb.BaseMetricSet client *http.Client // HTTP client that is reused across requests. headers map[string]string + uri string method string body []byte } @@ -62,6 +63,7 @@ func NewHTTP(base mb.BaseMetricSet) *HTTP { }, headers: config.Headers, method: "GET", + uri: base.HostData().SanitizedURI, body: nil, } } @@ -77,7 +79,7 @@ func (h *HTTP) FetchResponse() (*http.Response, error) { reader = bytes.NewReader(h.body) } - req, err := http.NewRequest(h.method, h.base.HostData().SanitizedURI, reader) + req, err := http.NewRequest(h.method, h.uri, reader) if h.base.HostData().User != "" || h.base.HostData().Password != "" { req.SetBasicAuth(h.base.HostData().User, h.base.HostData().Password) } @@ -102,6 +104,10 @@ func (h *HTTP) SetMethod(method string) { h.method = method } +func (h *HTTP) SetURI(uri string) { + h.uri = uri +} + func (h *HTTP) SetBody(body []byte) { h.body = body } From 856e151c7a0a200a06306f97480b85a8c928d4ad Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 24 Apr 2017 14:25:58 +0200 Subject: [PATCH 10/40] Skip empty events without error (#4087) In some cases, a metricset does not return an event but also not an error. This can be the case if a metricset for a distributed system should only send events for the master. All metricbeat instances are running on all nodes, but only one node should send the events and the other skip the event. This changes makes this behaviour possible. --- metricbeat/mb/module/wrapper.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metricbeat/mb/module/wrapper.go b/metricbeat/mb/module/wrapper.go index ab71232f0c5..b7483d27b4c 100644 --- a/metricbeat/mb/module/wrapper.go +++ b/metricbeat/mb/module/wrapper.go @@ -311,6 +311,10 @@ func (r *eventReporter) Error(err error) bool { } func (r *eventReporter) ErrorWith(err error, meta common.MapStr) bool { + // Skip nil events without error + if err == nil && meta == nil { + return true + } timestamp := r.start elapsed := time.Duration(0) From 38e04b461dd5b947110d86e4c6421475d5e659e6 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Mon, 24 Apr 2017 16:36:20 +0200 Subject: [PATCH 11/40] Fix race in go-metrics adapater (#4098) Fix race in go-metrics adapter, if registry is updated concurrently from multiple go routines. --- CHANGELOG.asciidoc | 1 + libbeat/monitoring/adapter/go-metrics.go | 25 ++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 7a9d9babe5c..14993bf24bb 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -54,6 +54,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Fix potential elasticsearch output URL parsing error if protocol scheme is missing. {pull}3671[3671] - Improve error message when downloading the dashboards fails. {pull}3805[3805] - Fix panic when testing regex-AST to match against date patterns. {issue}3889[3889] +- Fix panic due to race condition in kafka output. {pull}4098[4098] *Filebeat* - Always use absolute path for event and registry. {pull}3328[3328] diff --git a/libbeat/monitoring/adapter/go-metrics.go b/libbeat/monitoring/adapter/go-metrics.go index 2d0b94188ac..143bb0bf6ea 100644 --- a/libbeat/monitoring/adapter/go-metrics.go +++ b/libbeat/monitoring/adapter/go-metrics.go @@ -3,6 +3,7 @@ package adapter import ( "fmt" "reflect" + "sync" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/monitoring" @@ -21,6 +22,8 @@ import ( // It's recommended to not mix go-metrics with other metrics types // in the same namespace. type GoMetricsRegistry struct { + mutex sync.Mutex + reg *monitoring.Registry filters *metricFilters @@ -80,6 +83,12 @@ func (r *GoMetricsRegistry) find(name string) interface{} { // It's recommended to not mix go-metrics with other metrics types in one // namespace. func (r *GoMetricsRegistry) Get(name string) interface{} { + r.mutex.Lock() + defer r.mutex.Unlock() + return r.get(name) +} + +func (r *GoMetricsRegistry) get(name string) interface{} { m := r.find(name) if m == nil { return r.shadow.Get(name) @@ -95,7 +104,10 @@ func (r *GoMetricsRegistry) Get(name string) interface{} { // GetOrRegister retries an existing metric via `Get` or registers a new one // if the metric is unknown. For lazy instantiation metric can be a function. func (r *GoMetricsRegistry) GetOrRegister(name string, metric interface{}) interface{} { - v := r.Get(name) + r.mutex.Lock() + defer r.mutex.Unlock() + + v := r.get(name) if v != nil { return v } @@ -106,7 +118,10 @@ func (r *GoMetricsRegistry) GetOrRegister(name string, metric interface{}) inter // Register adds a new metric. // An error is returned if the metric is already known. func (r *GoMetricsRegistry) Register(name string, metric interface{}) error { - if r.Get(name) != nil { + r.mutex.Lock() + defer r.mutex.Unlock() + + if r.get(name) != nil { return fmt.Errorf("metric '%v' already registered", name) } @@ -139,6 +154,9 @@ func (r *GoMetricsRegistry) RunHealthchecks() {} // Unregister removes a metric. func (r *GoMetricsRegistry) Unregister(name string) { + r.mutex.Lock() + defer r.mutex.Unlock() + st := r.rmState(name) r.reg.Remove(st.name) r.shadow.Unregister(name) @@ -146,6 +164,9 @@ func (r *GoMetricsRegistry) Unregister(name string) { // UnregisterAll calls `Clear` on the underlying monitoring.Registry func (r *GoMetricsRegistry) UnregisterAll() { + r.mutex.Lock() + defer r.mutex.Unlock() + r.shadow.UnregisterAll() err := r.reg.Clear() if err != nil { From fafe0884f2f6733922d9be79a7139409f5cdb639 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 24 Apr 2017 22:08:13 +0200 Subject: [PATCH 12/40] Set 10000 field limit also for ES v5 (#4079) This was accidentially removed in https://github.com/elastic/beats/pull/4055 --- libbeat/template/template.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libbeat/template/template.go b/libbeat/template/template.go index bd09ae29fe8..f2ad6b7f04a 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -117,7 +117,9 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. if t.esVersion.IsMajor(2) || t.esVersion.IsMajor(5) { basicStructure.Put("mappings._default_._all.norms.enabled", false) - } else { + } + + if t.esVersion.major >= 5 { // Metricbeat exceeds the default of 1000 fields basicStructure.Put("settings.index.mapping.total_fields.limit", defaultTotalFieldsLimit) } From 2bc0a2068450f87e9091a7c15b93c08c016a8a9b Mon Sep 17 00:00:00 2001 From: yeer Date: Tue, 25 Apr 2017 14:39:43 +0800 Subject: [PATCH 13/40] File restructure by providers for add_cloud_metadata (#4088) * Rename package name, packages in add_could_metadata and add_locale should follow the name of their current folder --- libbeat/docs/processors-config.asciidoc | 2 +- .../add_cloud_metadata/add_cloud_metadata.go | 109 +----- .../add_cloud_metadata_test.go | 368 ------------------ .../add_cloud_metadata/provider_aws_ec2.go | 24 ++ .../provider_aws_ec2_test.go | 78 ++++ .../provider_digital_ocean.go | 22 ++ .../provider_digital_ocean_test.go | 107 +++++ .../add_cloud_metadata/provider_google_gce.go | 35 ++ .../provider_google_gce_test.go | 147 +++++++ .../provider_tencent_cloud.go | 40 ++ .../provider_tencent_cloud_test.go | 69 ++++ libbeat/processors/add_locale/add_locale.go | 2 +- .../processors/add_locale/add_locale_test.go | 2 +- 13 files changed, 526 insertions(+), 479 deletions(-) delete mode 100644 libbeat/processors/add_cloud_metadata/add_cloud_metadata_test.go create mode 100644 libbeat/processors/add_cloud_metadata/provider_aws_ec2.go create mode 100644 libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go create mode 100644 libbeat/processors/add_cloud_metadata/provider_digital_ocean.go create mode 100644 libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go create mode 100644 libbeat/processors/add_cloud_metadata/provider_google_gce.go create mode 100644 libbeat/processors/add_cloud_metadata/provider_google_gce_test.go create mode 100644 libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go create mode 100644 libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go diff --git a/libbeat/docs/processors-config.asciidoc b/libbeat/docs/processors-config.asciidoc index e70b97f3ff9..b7d3cc79e9a 100644 --- a/libbeat/docs/processors-config.asciidoc +++ b/libbeat/docs/processors-config.asciidoc @@ -343,7 +343,7 @@ _GCE_ } ------------------------------------------------------------------------------- -_Tencent Clound_ +_Tencent Cloud_ [source,json] ------------------------------------------------------------------------------- diff --git a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go index 08b8645276c..d1b86e150fe 100644 --- a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go +++ b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go @@ -1,4 +1,4 @@ -package actions +package add_cloud_metadata import ( "bytes" @@ -13,8 +13,6 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" - s "github.com/elastic/beats/metricbeat/schema" - c "github.com/elastic/beats/metricbeat/schema/mapstriface" "github.com/pkg/errors" ) @@ -23,73 +21,12 @@ const ( // use for their metadata service. metadataHost = "169.254.169.254" - // AWS EC2 Metadata Service - ec2InstanceIdentityURI = "/2014-02-25/dynamic/instance-identity/document" - - // DigitalOcean Metadata Service - doMetadataURI = "/metadata/v1.json" - - // Google GCE Metadata Service - gceMetadataURI = "/computeMetadata/v1/?recursive=true&alt=json" - - // Tencent Clound Metadata Service - qcloudMetadataHost = "metadata.tencentyun.com" - qcloudMetadataInstanceIDURI = "/meta-data/instance-id" - qcloudMetadataRegionURI = "/meta-data/placement/region" - qcloudMetadataZoneURI = "/meta-data/placement/zone" - // Default config defaultTimeOut = 3 * time.Second ) var debugf = logp.MakeDebug("filters") -// metadata schemas for all prividers. -var ( - ec2Schema = func(m map[string]interface{}) common.MapStr { - out, _ := s.Schema{ - "instance_id": c.Str("instanceId"), - "machine_type": c.Str("instanceType"), - "region": c.Str("region"), - "availability_zone": c.Str("availabilityZone"), - }.Apply(m) - return out - } - - doSchema = func(m map[string]interface{}) common.MapStr { - out, _ := s.Schema{ - "instance_id": c.StrFromNum("droplet_id"), - "region": c.Str("region"), - }.Apply(m) - return out - } - - gceHeaders = map[string]string{"Metadata-Flavor": "Google"} - gceSchema = func(m map[string]interface{}) common.MapStr { - out := common.MapStr{} - - if instance, ok := m["instance"].(map[string]interface{}); ok { - s.Schema{ - "instance_id": c.StrFromNum("id"), - "machine_type": c.Str("machineType"), - "availability_zone": c.Str("zone"), - }.ApplyTo(out, instance) - } - - if project, ok := m["project"].(map[string]interface{}); ok { - s.Schema{ - "project_id": c.Str("projectId"), - }.ApplyTo(out, project) - } - - return out - } - - qcloudSchema = func(m map[string]interface{}) common.MapStr { - return common.MapStr(m) - } -) - // init registers the add_cloud_metadata processor. func init() { processors.RegisterPlugin("add_cloud_metadata", newCloudMetadata) @@ -299,50 +236,6 @@ func newMetadataFetcher( return fetcher, nil } -func newDoMetadataFetcher(c common.Config) (*metadataFetcher, error) { - fetcher, err := newMetadataFetcher(c, "digitalocean", nil, metadataHost, doSchema, doMetadataURI) - return fetcher, err -} - -func newEc2MetadataFetcher(c common.Config) (*metadataFetcher, error) { - fetcher, err := newMetadataFetcher(c, "ec2", nil, metadataHost, ec2Schema, ec2InstanceIdentityURI) - return fetcher, err -} - -func newGceMetadataFetcher(c common.Config) (*metadataFetcher, error) { - fetcher, err := newMetadataFetcher(c, "gce", gceHeaders, metadataHost, gceSchema, gceMetadataURI) - return fetcher, err -} - -// newQcloudMetadataFetcher return the concrete metadata fetcher for qcloud provider -// which requires more than one way to assemble the metadata. -func newQcloudMetadataFetcher(c common.Config) (*metadataFetcher, error) { - urls, err := getMetadataURLs(c, qcloudMetadataHost, []string{ - qcloudMetadataInstanceIDURI, - qcloudMetadataRegionURI, - qcloudMetadataZoneURI, - }) - if err != nil { - return nil, err - } - responseHandlers := map[string]responseHandler{ - urls[0]: func(all []byte, result *result) error { - result.metadata["instance_id"] = string(all) - return nil - }, - urls[1]: func(all []byte, result *result) error { - result.metadata["region"] = string(all) - return nil - }, - urls[2]: func(all []byte, result *result) error { - result.metadata["availability_zone"] = string(all) - return nil - }, - } - fetcher := &metadataFetcher{"qcloud", nil, responseHandlers, qcloudSchema} - return fetcher, nil -} - func setupFetchers(c common.Config) ([]*metadataFetcher, error) { var fetchers []*metadataFetcher doFetcher, err := newDoMetadataFetcher(c) diff --git a/libbeat/processors/add_cloud_metadata/add_cloud_metadata_test.go b/libbeat/processors/add_cloud_metadata/add_cloud_metadata_test.go deleted file mode 100644 index b79e879abdb..00000000000 --- a/libbeat/processors/add_cloud_metadata/add_cloud_metadata_test.go +++ /dev/null @@ -1,368 +0,0 @@ -package actions - -import ( - "net/http" - "net/http/httptest" - "testing" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/logp" - "github.com/stretchr/testify/assert" -) - -const ec2InstanceIdentityDocument = `{ - "devpayProductCodes" : null, - "privateIp" : "10.0.0.1", - "availabilityZone" : "us-east-1c", - "accountId" : "111111111111111", - "version" : "2010-08-31", - "instanceId" : "i-11111111", - "billingProducts" : null, - "instanceType" : "t2.medium", - "imageId" : "ami-6869aa05", - "pendingTime" : "2016-09-20T15:43:02Z", - "architecture" : "x86_64", - "kernelId" : null, - "ramdiskId" : null, - "region" : "us-east-1" -}` - -const digitalOceanMetadataV1 = `{ - "droplet_id":1111111, - "hostname":"sample-droplet", - "vendor_data":"#cloud-config\ndisable_root: false\nmanage_etc_hosts: true\n\ncloud_config_modules:\n - ssh\n - set_hostname\n - [ update_etc_hosts, once-per-instance ]\n\ncloud_final_modules:\n - scripts-vendor\n - scripts-per-once\n - scripts-per-boot\n - scripts-per-instance\n - scripts-user\n", - "public_keys":["ssh-rsa 111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111 sammy@digitalocean.com"], - "region":"nyc3", - "interfaces":{ - "private":[ - { - "ipv4":{ - "ip_address":"10.0.0.2", - "netmask":"255.255.0.0", - "gateway":"10.10.0.1" - }, - "mac":"54:11:00:00:00:00", - "type":"private" - } - ], - "public":[ - { - "ipv4":{ - "ip_address":"192.168.20.105", - "netmask":"255.255.192.0", - "gateway":"192.168.20.1" - }, - "ipv6":{ - "ip_address":"1111:1111:0000:0000:0000:0000:0000:0000", - "cidr":64, - "gateway":"0000:0000:0800:0010:0000:0000:0000:0001" - }, - "mac":"34:00:00:ff:00:00", - "type":"public"} - ] - }, - "floating_ip": { - "ipv4": { - "active": false - } - }, - "dns":{ - "nameservers":[ - "2001:4860:4860::8844", - "2001:4860:4860::8888", - "8.8.8.8" - ] - } -}` - -const gceMetadataV1 = `{ - "instance": { - "attributes": {}, - "cpuPlatform": "Intel Haswell", - "description": "", - "disks": [ - { - "deviceName": "test-gce-dev", - "index": 0, - "mode": "READ_WRITE", - "type": "PERSISTENT" - } - ], - "hostname": "test-gce-dev.c.test-dev.internal", - "id": 3910564293633576924, - "image": "", - "licenses": [ - { - "id": "1000000" - } - ], - "machineType": "projects/111111111111/machineTypes/f1-micro", - "maintenanceEvent": "NONE", - "networkInterfaces": [ - { - "accessConfigs": [ - { - "externalIp": "10.10.10.10", - "type": "ONE_TO_ONE_NAT" - } - ], - "forwardedIps": [], - "ip": "10.10.0.2", - "ipAliases": [], - "mac": "44:00:00:00:00:01", - "network": "projects/111111111111/networks/default" - } - ], - "scheduling": { - "automaticRestart": "TRUE", - "onHostMaintenance": "MIGRATE", - "preemptible": "FALSE" - }, - "serviceAccounts": { - "111111111111-compute@developer.gserviceaccount.com": { - "aliases": [ - "default" - ], - "email": "111111111111-compute@developer.gserviceaccount.com", - "scopes": [ - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write", - "https://www.googleapis.com/auth/servicecontrol", - "https://www.googleapis.com/auth/service.management.readonly" - ] - }, - "default": { - "aliases": [ - "default" - ], - "email": "111111111111-compute@developer.gserviceaccount.com", - "scopes": [ - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write", - "https://www.googleapis.com/auth/servicecontrol", - "https://www.googleapis.com/auth/service.management.readonly" - ] - } - }, - "tags": [], - "virtualClock": { - "driftToken": "0" - }, - "zone": "projects/111111111111/zones/us-east1-b" - }, - "project": { - "attributes": { - "sshKeys": "developer:ssh-rsa 222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222 google-ssh {\"userName\":\"foo@bar.com\",\"expireOn\":\"2016-10-06T20:20:41+0000\"}\ndev:ecdsa-sha2-nistp256 4444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444= google-ssh {\"userName\":\"foo@bar.com\",\"expireOn\":\"2016-10-06T20:20:40+0000\"}\ndev:ssh-rsa 444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444 dev" - }, - "numericProjectId": 111111111111, - "projectId": "test-dev" - } -}` - -func initEC2TestServer() *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.RequestURI == "/2014-02-25/dynamic/instance-identity/document" { - w.Write([]byte(ec2InstanceIdentityDocument)) - return - } - - http.Error(w, "not found", http.StatusNotFound) - })) -} - -func initDigitalOceanTestServer() *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.RequestURI == "/metadata/v1.json" { - w.Write([]byte(digitalOceanMetadataV1)) - return - } - - http.Error(w, "not found", http.StatusNotFound) - })) -} - -func initGCETestServer() *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.RequestURI == "/computeMetadata/v1/?recursive=true&alt=json" { - w.Write([]byte(gceMetadataV1)) - return - } - - http.Error(w, "not found", http.StatusNotFound) - })) -} - -func initQCloundTestServer() *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.RequestURI == "/meta-data/instance-id" { - w.Write([]byte("ins-qcloudv5")) - return - } - if r.RequestURI == "/meta-data/placement/region" { - w.Write([]byte("china-south-gz")) - return - } - if r.RequestURI == "/meta-data/placement/zone" { - w.Write([]byte("gz-azone2")) - return - } - - http.Error(w, "not found", http.StatusNotFound) - })) -} - -func TestRetrieveAWSMetadata(t *testing.T) { - if testing.Verbose() { - logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"}) - } - - server := initEC2TestServer() - defer server.Close() - - config, err := common.NewConfigFrom(map[string]interface{}{ - "host": server.Listener.Addr().String(), - }) - if err != nil { - t.Fatal(err) - } - - p, err := newCloudMetadata(*config) - if err != nil { - t.Fatal(err) - } - - actual, err := p.Run(common.MapStr{}) - if err != nil { - t.Fatal(err) - } - - expected := common.MapStr{ - "meta": common.MapStr{ - "cloud": common.MapStr{ - "provider": "ec2", - "instance_id": "i-11111111", - "machine_type": "t2.medium", - "region": "us-east-1", - "availability_zone": "us-east-1c", - }, - }, - } - assert.Equal(t, expected, actual) -} - -func TestRetrieveDigitalOceanMetadata(t *testing.T) { - if testing.Verbose() { - logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"}) - } - - server := initDigitalOceanTestServer() - defer server.Close() - - config, err := common.NewConfigFrom(map[string]interface{}{ - "host": server.Listener.Addr().String(), - }) - if err != nil { - t.Fatal(err) - } - - p, err := newCloudMetadata(*config) - if err != nil { - t.Fatal(err) - } - - actual, err := p.Run(common.MapStr{}) - if err != nil { - t.Fatal(err) - } - - expected := common.MapStr{ - "meta": common.MapStr{ - "cloud": common.MapStr{ - "provider": "digitalocean", - "instance_id": "1111111", - "region": "nyc3", - }, - }, - } - assert.Equal(t, expected, actual) -} - -func TestRetrieveGCEMetadata(t *testing.T) { - if testing.Verbose() { - logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"}) - } - - server := initGCETestServer() - defer server.Close() - - config, err := common.NewConfigFrom(map[string]interface{}{ - "host": server.Listener.Addr().String(), - }) - if err != nil { - t.Fatal(err) - } - - p, err := newCloudMetadata(*config) - if err != nil { - t.Fatal(err) - } - - actual, err := p.Run(common.MapStr{}) - if err != nil { - t.Fatal(err) - } - - expected := common.MapStr{ - "meta": common.MapStr{ - "cloud": common.MapStr{ - "provider": "gce", - "instance_id": "3910564293633576924", - "machine_type": "projects/111111111111/machineTypes/f1-micro", - "availability_zone": "projects/111111111111/zones/us-east1-b", - "project_id": "test-dev", - }, - }, - } - assert.Equal(t, expected, actual) -} - -func TestRetrieveQCloudMetadata(t *testing.T) { - if testing.Verbose() { - logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"}) - } - - server := initQCloundTestServer() - defer server.Close() - - config, err := common.NewConfigFrom(map[string]interface{}{ - "host": server.Listener.Addr().String(), - }) - - if err != nil { - t.Fatal(err) - } - - p, err := newCloudMetadata(*config) - if err != nil { - t.Fatal(err) - } - - actual, err := p.Run(common.MapStr{}) - if err != nil { - t.Fatal(err) - } - - expected := common.MapStr{ - "meta": common.MapStr{ - "cloud": common.MapStr{ - "provider": "qcloud", - "instance_id": "ins-qcloudv5", - "region": "china-south-gz", - "availability_zone": "gz-azone2", - }, - }, - } - assert.Equal(t, expected, actual) -} diff --git a/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go b/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go new file mode 100644 index 00000000000..a8c987b93fc --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go @@ -0,0 +1,24 @@ +package add_cloud_metadata + +import ( + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/metricbeat/schema" + c "github.com/elastic/beats/metricbeat/schema/mapstriface" +) + +// AWS EC2 Metadata Service +func newEc2MetadataFetcher(config common.Config) (*metadataFetcher, error) { + ec2InstanceIdentityURI := "/2014-02-25/dynamic/instance-identity/document" + ec2Schema := func(m map[string]interface{}) common.MapStr { + out, _ := s.Schema{ + "instance_id": c.Str("instanceId"), + "machine_type": c.Str("instanceType"), + "region": c.Str("region"), + "availability_zone": c.Str("availabilityZone"), + }.Apply(m) + return out + } + + fetcher, err := newMetadataFetcher(config, "ec2", nil, metadataHost, ec2Schema, ec2InstanceIdentityURI) + return fetcher, err +} diff --git a/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go b/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go new file mode 100644 index 00000000000..cf1709f1f26 --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go @@ -0,0 +1,78 @@ +package add_cloud_metadata + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/stretchr/testify/assert" +) + +const ec2InstanceIdentityDocument = `{ + "devpayProductCodes" : null, + "privateIp" : "10.0.0.1", + "availabilityZone" : "us-east-1c", + "accountId" : "111111111111111", + "version" : "2010-08-31", + "instanceId" : "i-11111111", + "billingProducts" : null, + "instanceType" : "t2.medium", + "imageId" : "ami-6869aa05", + "pendingTime" : "2016-09-20T15:43:02Z", + "architecture" : "x86_64", + "kernelId" : null, + "ramdiskId" : null, + "region" : "us-east-1" +}` + +func initEC2TestServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/2014-02-25/dynamic/instance-identity/document" { + w.Write([]byte(ec2InstanceIdentityDocument)) + return + } + + http.Error(w, "not found", http.StatusNotFound) + })) +} + +func TestRetrieveAWSMetadata(t *testing.T) { + if testing.Verbose() { + logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"}) + } + + server := initEC2TestServer() + defer server.Close() + + config, err := common.NewConfigFrom(map[string]interface{}{ + "host": server.Listener.Addr().String(), + }) + if err != nil { + t.Fatal(err) + } + + p, err := newCloudMetadata(*config) + if err != nil { + t.Fatal(err) + } + + actual, err := p.Run(common.MapStr{}) + if err != nil { + t.Fatal(err) + } + + expected := common.MapStr{ + "meta": common.MapStr{ + "cloud": common.MapStr{ + "provider": "ec2", + "instance_id": "i-11111111", + "machine_type": "t2.medium", + "region": "us-east-1", + "availability_zone": "us-east-1c", + }, + }, + } + assert.Equal(t, expected, actual) +} diff --git a/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go b/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go new file mode 100644 index 00000000000..4a7b7428d64 --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go @@ -0,0 +1,22 @@ +package add_cloud_metadata + +import ( + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/metricbeat/schema" + c "github.com/elastic/beats/metricbeat/schema/mapstriface" +) + +// DigitalOcean Metadata Service +func newDoMetadataFetcher(config common.Config) (*metadataFetcher, error) { + doSchema := func(m map[string]interface{}) common.MapStr { + out, _ := s.Schema{ + "instance_id": c.StrFromNum("droplet_id"), + "region": c.Str("region"), + }.Apply(m) + return out + } + doMetadataURI := "/metadata/v1.json" + + fetcher, err := newMetadataFetcher(config, "digitalocean", nil, metadataHost, doSchema, doMetadataURI) + return fetcher, err +} diff --git a/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go b/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go new file mode 100644 index 00000000000..9a1f007e50f --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go @@ -0,0 +1,107 @@ +package add_cloud_metadata + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/stretchr/testify/assert" +) + +const digitalOceanMetadataV1 = `{ + "droplet_id":1111111, + "hostname":"sample-droplet", + "vendor_data":"#cloud-config\ndisable_root: false\nmanage_etc_hosts: true\n\ncloud_config_modules:\n - ssh\n - set_hostname\n - [ update_etc_hosts, once-per-instance ]\n\ncloud_final_modules:\n - scripts-vendor\n - scripts-per-once\n - scripts-per-boot\n - scripts-per-instance\n - scripts-user\n", + "public_keys":["ssh-rsa 111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111 sammy@digitalocean.com"], + "region":"nyc3", + "interfaces":{ + "private":[ + { + "ipv4":{ + "ip_address":"10.0.0.2", + "netmask":"255.255.0.0", + "gateway":"10.10.0.1" + }, + "mac":"54:11:00:00:00:00", + "type":"private" + } + ], + "public":[ + { + "ipv4":{ + "ip_address":"192.168.20.105", + "netmask":"255.255.192.0", + "gateway":"192.168.20.1" + }, + "ipv6":{ + "ip_address":"1111:1111:0000:0000:0000:0000:0000:0000", + "cidr":64, + "gateway":"0000:0000:0800:0010:0000:0000:0000:0001" + }, + "mac":"34:00:00:ff:00:00", + "type":"public"} + ] + }, + "floating_ip": { + "ipv4": { + "active": false + } + }, + "dns":{ + "nameservers":[ + "2001:4860:4860::8844", + "2001:4860:4860::8888", + "8.8.8.8" + ] + } +}` + +func initDigitalOceanTestServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/metadata/v1.json" { + w.Write([]byte(digitalOceanMetadataV1)) + return + } + + http.Error(w, "not found", http.StatusNotFound) + })) +} + +func TestRetrieveDigitalOceanMetadata(t *testing.T) { + if testing.Verbose() { + logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"}) + } + + server := initDigitalOceanTestServer() + defer server.Close() + + config, err := common.NewConfigFrom(map[string]interface{}{ + "host": server.Listener.Addr().String(), + }) + if err != nil { + t.Fatal(err) + } + + p, err := newCloudMetadata(*config) + if err != nil { + t.Fatal(err) + } + + actual, err := p.Run(common.MapStr{}) + if err != nil { + t.Fatal(err) + } + + expected := common.MapStr{ + "meta": common.MapStr{ + "cloud": common.MapStr{ + "provider": "digitalocean", + "instance_id": "1111111", + "region": "nyc3", + }, + }, + } + assert.Equal(t, expected, actual) +} diff --git a/libbeat/processors/add_cloud_metadata/provider_google_gce.go b/libbeat/processors/add_cloud_metadata/provider_google_gce.go new file mode 100644 index 00000000000..f6a84ccde08 --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_google_gce.go @@ -0,0 +1,35 @@ +package add_cloud_metadata + +import ( + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/metricbeat/schema" + c "github.com/elastic/beats/metricbeat/schema/mapstriface" +) + +// Google GCE Metadata Service +func newGceMetadataFetcher(config common.Config) (*metadataFetcher, error) { + gceMetadataURI := "/computeMetadata/v1/?recursive=true&alt=json" + gceHeaders := map[string]string{"Metadata-Flavor": "Google"} + gceSchema := func(m map[string]interface{}) common.MapStr { + out := common.MapStr{} + + if instance, ok := m["instance"].(map[string]interface{}); ok { + s.Schema{ + "instance_id": c.StrFromNum("id"), + "machine_type": c.Str("machineType"), + "availability_zone": c.Str("zone"), + }.ApplyTo(out, instance) + } + + if project, ok := m["project"].(map[string]interface{}); ok { + s.Schema{ + "project_id": c.Str("projectId"), + }.ApplyTo(out, project) + } + + return out + } + + fetcher, err := newMetadataFetcher(config, "gce", gceHeaders, metadataHost, gceSchema, gceMetadataURI) + return fetcher, err +} diff --git a/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go b/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go new file mode 100644 index 00000000000..87f8447d789 --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go @@ -0,0 +1,147 @@ +package add_cloud_metadata + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/stretchr/testify/assert" +) + +const gceMetadataV1 = `{ + "instance": { + "attributes": {}, + "cpuPlatform": "Intel Haswell", + "description": "", + "disks": [ + { + "deviceName": "test-gce-dev", + "index": 0, + "mode": "READ_WRITE", + "type": "PERSISTENT" + } + ], + "hostname": "test-gce-dev.c.test-dev.internal", + "id": 3910564293633576924, + "image": "", + "licenses": [ + { + "id": "1000000" + } + ], + "machineType": "projects/111111111111/machineTypes/f1-micro", + "maintenanceEvent": "NONE", + "networkInterfaces": [ + { + "accessConfigs": [ + { + "externalIp": "10.10.10.10", + "type": "ONE_TO_ONE_NAT" + } + ], + "forwardedIps": [], + "ip": "10.10.0.2", + "ipAliases": [], + "mac": "44:00:00:00:00:01", + "network": "projects/111111111111/networks/default" + } + ], + "scheduling": { + "automaticRestart": "TRUE", + "onHostMaintenance": "MIGRATE", + "preemptible": "FALSE" + }, + "serviceAccounts": { + "111111111111-compute@developer.gserviceaccount.com": { + "aliases": [ + "default" + ], + "email": "111111111111-compute@developer.gserviceaccount.com", + "scopes": [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/service.management.readonly" + ] + }, + "default": { + "aliases": [ + "default" + ], + "email": "111111111111-compute@developer.gserviceaccount.com", + "scopes": [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/service.management.readonly" + ] + } + }, + "tags": [], + "virtualClock": { + "driftToken": "0" + }, + "zone": "projects/111111111111/zones/us-east1-b" + }, + "project": { + "attributes": { + "sshKeys": "developer:ssh-rsa 222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222 google-ssh {\"userName\":\"foo@bar.com\",\"expireOn\":\"2016-10-06T20:20:41+0000\"}\ndev:ecdsa-sha2-nistp256 4444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444= google-ssh {\"userName\":\"foo@bar.com\",\"expireOn\":\"2016-10-06T20:20:40+0000\"}\ndev:ssh-rsa 444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444 dev" + }, + "numericProjectId": 111111111111, + "projectId": "test-dev" + } +}` + +func initGCETestServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/computeMetadata/v1/?recursive=true&alt=json" { + w.Write([]byte(gceMetadataV1)) + return + } + + http.Error(w, "not found", http.StatusNotFound) + })) +} + +func TestRetrieveGCEMetadata(t *testing.T) { + if testing.Verbose() { + logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"}) + } + + server := initGCETestServer() + defer server.Close() + + config, err := common.NewConfigFrom(map[string]interface{}{ + "host": server.Listener.Addr().String(), + }) + if err != nil { + t.Fatal(err) + } + + p, err := newCloudMetadata(*config) + if err != nil { + t.Fatal(err) + } + + actual, err := p.Run(common.MapStr{}) + if err != nil { + t.Fatal(err) + } + + expected := common.MapStr{ + "meta": common.MapStr{ + "cloud": common.MapStr{ + "provider": "gce", + "instance_id": "3910564293633576924", + "machine_type": "projects/111111111111/machineTypes/f1-micro", + "availability_zone": "projects/111111111111/zones/us-east1-b", + "project_id": "test-dev", + }, + }, + } + assert.Equal(t, expected, actual) +} diff --git a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go new file mode 100644 index 00000000000..aec4d9022e8 --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go @@ -0,0 +1,40 @@ +package add_cloud_metadata + +import "github.com/elastic/beats/libbeat/common" + +// Tenccent Cloud Metadata Service +func newQcloudMetadataFetcher(c common.Config) (*metadataFetcher, error) { + qcloudMetadataHost := "metadata.tencentyun.com" + qcloudMetadataInstanceIDURI := "/meta-data/instance-id" + qcloudMetadataRegionURI := "/meta-data/placement/region" + qcloudMetadataZoneURI := "/meta-data/placement/zone" + + qcloudSchema := func(m map[string]interface{}) common.MapStr { + return common.MapStr(m) + } + + urls, err := getMetadataURLs(c, qcloudMetadataHost, []string{ + qcloudMetadataInstanceIDURI, + qcloudMetadataRegionURI, + qcloudMetadataZoneURI, + }) + if err != nil { + return nil, err + } + responseHandlers := map[string]responseHandler{ + urls[0]: func(all []byte, result *result) error { + result.metadata["instance_id"] = string(all) + return nil + }, + urls[1]: func(all []byte, result *result) error { + result.metadata["region"] = string(all) + return nil + }, + urls[2]: func(all []byte, result *result) error { + result.metadata["availability_zone"] = string(all) + return nil + }, + } + fetcher := &metadataFetcher{"qcloud", nil, responseHandlers, qcloudSchema} + return fetcher, nil +} diff --git a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go new file mode 100644 index 00000000000..b2bb3560bed --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go @@ -0,0 +1,69 @@ +package add_cloud_metadata + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/stretchr/testify/assert" +) + +func initQCloudTestServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/meta-data/instance-id" { + w.Write([]byte("ins-qcloudv5")) + return + } + if r.RequestURI == "/meta-data/placement/region" { + w.Write([]byte("china-south-gz")) + return + } + if r.RequestURI == "/meta-data/placement/zone" { + w.Write([]byte("gz-azone2")) + return + } + + http.Error(w, "not found", http.StatusNotFound) + })) +} + +func TestRetrieveQCloudMetadata(t *testing.T) { + if testing.Verbose() { + logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"}) + } + + server := initQCloudTestServer() + defer server.Close() + + config, err := common.NewConfigFrom(map[string]interface{}{ + "host": server.Listener.Addr().String(), + }) + + if err != nil { + t.Fatal(err) + } + + p, err := newCloudMetadata(*config) + if err != nil { + t.Fatal(err) + } + + actual, err := p.Run(common.MapStr{}) + if err != nil { + t.Fatal(err) + } + + expected := common.MapStr{ + "meta": common.MapStr{ + "cloud": common.MapStr{ + "provider": "qcloud", + "instance_id": "ins-qcloudv5", + "region": "china-south-gz", + "availability_zone": "gz-azone2", + }, + }, + } + assert.Equal(t, expected, actual) +} diff --git a/libbeat/processors/add_locale/add_locale.go b/libbeat/processors/add_locale/add_locale.go index 9c7ecef5407..dfd631dd348 100644 --- a/libbeat/processors/add_locale/add_locale.go +++ b/libbeat/processors/add_locale/add_locale.go @@ -1,4 +1,4 @@ -package actions +package add_locale import ( "time" diff --git a/libbeat/processors/add_locale/add_locale_test.go b/libbeat/processors/add_locale/add_locale_test.go index 3d2b31a9396..10bbaa6c782 100644 --- a/libbeat/processors/add_locale/add_locale_test.go +++ b/libbeat/processors/add_locale/add_locale_test.go @@ -1,4 +1,4 @@ -package actions +package add_locale import ( "testing" From de11679f7eb1bc527d79bd2774c9b87da2c41c5e Mon Sep 17 00:00:00 2001 From: Martin Scholz Date: Tue, 25 Apr 2017 09:47:19 +0200 Subject: [PATCH 14/40] Add perfmon raw counter values calculations (#3972) Add perfmon raw counter values calculations This PR adds functions to calculate performance counters from raw values. These functions should serve more as internal apis. --- .../windows/perfmon/defs_pdh_windows.go | 5 ++ .../windows/perfmon/defs_pdh_windows_386.go | 15 +++++ .../windows/perfmon/defs_pdh_windows_amd64.go | 15 +++++ metricbeat/module/windows/perfmon/doc.go | 2 +- .../module/windows/perfmon/mkpdh_defs.go | 5 ++ .../perfmon/pdh_integration_windows_test.go | 56 ++++++++++++++++++- .../module/windows/perfmon/pdh_windows.go | 33 +++++++++++ .../module/windows/perfmon/zpdh_windows.go | 37 ++++++++++-- 8 files changed, 161 insertions(+), 7 deletions(-) diff --git a/metricbeat/module/windows/perfmon/defs_pdh_windows.go b/metricbeat/module/windows/perfmon/defs_pdh_windows.go index 28f016b46e5..60e54d14f50 100644 --- a/metricbeat/module/windows/perfmon/defs_pdh_windows.go +++ b/metricbeat/module/windows/perfmon/defs_pdh_windows.go @@ -218,3 +218,8 @@ const ( // PdhCounterValue is the structure that receives the counter value. type PdhCounterValue C.PDH_FMT_COUNTERVALUE + +// PdhRawCounter is the structure that receives the raw counter. +type PdhRawCounter C.PDH_RAW_COUNTER + +type PdhFileTime C.FILETIME diff --git a/metricbeat/module/windows/perfmon/defs_pdh_windows_386.go b/metricbeat/module/windows/perfmon/defs_pdh_windows_386.go index d9f3a2a48bb..c5b032cfdbc 100644 --- a/metricbeat/module/windows/perfmon/defs_pdh_windows_386.go +++ b/metricbeat/module/windows/perfmon/defs_pdh_windows_386.go @@ -205,3 +205,18 @@ type PdhCounterValue struct { LongValue int32 Pad_cgo_1 [4]byte } + +type PdhRawCounter struct { + CStatus uint32 + TimeStamp PdhFileTime + Pad_cgo_0 [4]byte + FirstValue int64 + SecondValue int64 + MultiCount uint32 + Pad_cgo_1 [4]byte +} + +type PdhFileTime struct { + DwLowDateTime uint32 + DwHighDateTime uint32 +} diff --git a/metricbeat/module/windows/perfmon/defs_pdh_windows_amd64.go b/metricbeat/module/windows/perfmon/defs_pdh_windows_amd64.go index d9f3a2a48bb..c5b032cfdbc 100644 --- a/metricbeat/module/windows/perfmon/defs_pdh_windows_amd64.go +++ b/metricbeat/module/windows/perfmon/defs_pdh_windows_amd64.go @@ -205,3 +205,18 @@ type PdhCounterValue struct { LongValue int32 Pad_cgo_1 [4]byte } + +type PdhRawCounter struct { + CStatus uint32 + TimeStamp PdhFileTime + Pad_cgo_0 [4]byte + FirstValue int64 + SecondValue int64 + MultiCount uint32 + Pad_cgo_1 [4]byte +} + +type PdhFileTime struct { + DwLowDateTime uint32 + DwHighDateTime uint32 +} diff --git a/metricbeat/module/windows/perfmon/doc.go b/metricbeat/module/windows/perfmon/doc.go index b6daada8e57..e04267b29fd 100644 --- a/metricbeat/module/windows/perfmon/doc.go +++ b/metricbeat/module/windows/perfmon/doc.go @@ -4,6 +4,6 @@ package perfmon //go:generate go run mkpdh_defs.go //go:generate go run run.go -cmd "go tool cgo -godefs defs_pdh_windows.go" -goarch amd64 -output defs_pdh_windows_amd64.go -//go:generate go run run.go -cmd "go tool cgo -godefs defs_pdh_windows.go" -goarch 386 -output defs_pdh_windows_386.go +//go:generate go run run.go -cmd "go tool cgo -godefs defs_pdh_windows.go" -goarch 386 -output defs_pdh_windows_386.go //go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zpdh_windows.go pdh_windows.go //go:generate gofmt -w defs_pdh_windows_amd64.go defs_pdh_windows_386.go zpdh_windows.go diff --git a/metricbeat/module/windows/perfmon/mkpdh_defs.go b/metricbeat/module/windows/perfmon/mkpdh_defs.go index c0c1f1092f5..de6f9ae734f 100644 --- a/metricbeat/module/windows/perfmon/mkpdh_defs.go +++ b/metricbeat/module/windows/perfmon/mkpdh_defs.go @@ -80,6 +80,11 @@ const ( // PdhCounterValue is the structure that receives the counter value. type PdhCounterValue C.PDH_FMT_COUNTERVALUE + +// PdhRawCounter is the structure that receives the raw counter. +type PdhRawCounter C.PDH_RAW_COUNTER + +type PdhFileTime C.FILETIME ` var ( diff --git a/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go b/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go index 97f747fa079..1524df41d7d 100644 --- a/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go +++ b/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go @@ -4,6 +4,8 @@ package perfmon import ( "testing" + "time" + "unsafe" "github.com/pkg/errors" "github.com/stretchr/testify/assert" @@ -81,7 +83,7 @@ func TestNonExistingCounter(t *testing.T) { func TestNonExistingObject(t *testing.T) { config := make([]CounterConfig, 1) config[0].Alias = "processor.time.total.pct" - config[0].Query = "\\non existing object\\% Processor Performance" + config[0].Query = "\\non existing object\\% Processor Time" handle, err := NewPerfmonReader(config) if assert.Error(t, err) { assert.EqualValues(t, PDH_CSTATUS_NO_OBJECT, errors.Cause(err)) @@ -92,3 +94,55 @@ func TestNonExistingObject(t *testing.T) { assert.NoError(t, err) } } + +func TestRawValues(t *testing.T) { + query, err := NewQuery("") + if err != nil { + t.Fatal(err) + } + defer query.Close() + + err = query.AddCounter(processorTimeCounter) + if err != nil && err != PDH_NO_MORE_DATA { + t.Fatal(err) + } + + var values []float64 + + for i := 0; i < 2; i++ { + + if err = query.Execute(); err != nil { + t.Fatal(err) + } + + _, rawvalue1, err := PdhGetRawCounterValue(query.counters[processorTimeCounter]) + + if err != nil { + t.Fatal(err) + } + + time.Sleep(time.Millisecond * 1000) + + if err = query.Execute(); err != nil { + t.Fatal(err) + } + + _, rawvalue2, err := PdhGetRawCounterValue(query.counters[processorTimeCounter]) + + if err != nil { + t.Fatal(err) + } + + value, err := PdhCalculateCounterFromRawValue(query.counters[processorTimeCounter], PdhFmtDouble|PdhFmtNoCap100, rawvalue2, rawvalue1) + + if err != nil { + t.Fatal(err) + } + + values = append(values, *(*float64)(unsafe.Pointer(&value.LongValue))) + + } + + t.Log(values) + +} diff --git a/metricbeat/module/windows/perfmon/pdh_windows.go b/metricbeat/module/windows/perfmon/pdh_windows.go index 641eed68ec8..76d915d33c0 100644 --- a/metricbeat/module/windows/perfmon/pdh_windows.go +++ b/metricbeat/module/windows/perfmon/pdh_windows.go @@ -19,6 +19,9 @@ import ( //sys _PdhAddCounter(query PdhQueryHandle, counterPath string, userData uintptr, counter *PdhCounterHandle) (errcode error) [failretval!=0] = pdh.PdhAddEnglishCounterW //sys _PdhCollectQueryData(query PdhQueryHandle) (errcode error) [failretval!=0] = pdh.PdhCollectQueryData //sys _PdhGetFormattedCounterValue(counter PdhCounterHandle, format PdhCounterFormat, counterType *uint32, value *PdhCounterValue) (errcode error) [failretval!=0] = pdh.PdhGetFormattedCounterValue +//sys _PdhGetRawCounterValue(counter PdhCounterHandle, counterType *uint32, value *PdhRawCounter) (errcode error) [failretval!=0] = pdh.PdhGetRawCounterValue +//sys _PdhCalculateCounterFromRawValue(counter PdhCounterHandle, format PdhCounterFormat, rawValue1 *PdhRawCounter, rawValue2 *PdhRawCounter, value *PdhCounterValue) (errcode error) [failretval!=0] = pdh.PdhCalculateCounterFromRawValue +//sys _PdhFormatFromRawValue(counterType uint32, format PdhCounterFormat, timeBase *uint64, rawValue1 *PdhRawCounter, rawValue2 *PdhRawCounter, value *PdhCounterValue) (errcode error) [failretval!=0] = pdh.PdhFormatFromRawValue //sys _PdhCloseQuery(query PdhQueryHandle) (errcode error) [failretval!=0] = pdh.PdhCloseQuery type PdhQueryHandle uintptr @@ -74,6 +77,36 @@ func PdhGetFormattedCounterValue(counter PdhCounterHandle, format PdhCounterForm return counterType, &value, nil } +func PdhGetRawCounterValue(counter PdhCounterHandle) (uint32, *PdhRawCounter, error) { + var counterType uint32 + var value PdhRawCounter + if err := _PdhGetRawCounterValue(counter, &counterType, &value); err != nil { + return 0, nil, PdhErrno(err.(syscall.Errno)) + } + + return counterType, &value, nil +} + +func PdhCalculateCounterFromRawValue(counter PdhCounterHandle, format PdhCounterFormat, rawValue1 *PdhRawCounter, rawValue2 *PdhRawCounter) (*PdhCounterValue, error) { + var value PdhCounterValue + if err := _PdhCalculateCounterFromRawValue(counter, format, rawValue1, rawValue2, &value); err != nil { + return nil, PdhErrno(err.(syscall.Errno)) + } + + return &value, nil +} + +func PdhFormatFromRawValue(format PdhCounterFormat, rawValue1 *PdhRawCounter, rawValue2 *PdhRawCounter) (*PdhCounterValue, error) { + var counterType uint32 + var value PdhCounterValue + var timeBase uint64 + if err := _PdhFormatFromRawValue(counterType, format, &timeBase, rawValue1, rawValue2, &value); err != nil { + return nil, PdhErrno(err.(syscall.Errno)) + } + + return &value, nil +} + func PdhCloseQuery(query PdhQueryHandle) error { if err := _PdhCloseQuery(query); err != nil { return PdhErrno(err.(syscall.Errno)) diff --git a/metricbeat/module/windows/perfmon/zpdh_windows.go b/metricbeat/module/windows/perfmon/zpdh_windows.go index 987cbf385c6..2296a4d35ea 100644 --- a/metricbeat/module/windows/perfmon/zpdh_windows.go +++ b/metricbeat/module/windows/perfmon/zpdh_windows.go @@ -14,11 +14,14 @@ var _ unsafe.Pointer var ( modpdh = windows.NewLazySystemDLL("pdh.dll") - procPdhOpenQueryW = modpdh.NewProc("PdhOpenQueryW") - procPdhAddEnglishCounterW = modpdh.NewProc("PdhAddEnglishCounterW") - procPdhCollectQueryData = modpdh.NewProc("PdhCollectQueryData") - procPdhGetFormattedCounterValue = modpdh.NewProc("PdhGetFormattedCounterValue") - procPdhCloseQuery = modpdh.NewProc("PdhCloseQuery") + procPdhOpenQueryW = modpdh.NewProc("PdhOpenQueryW") + procPdhAddEnglishCounterW = modpdh.NewProc("PdhAddEnglishCounterW") + procPdhCollectQueryData = modpdh.NewProc("PdhCollectQueryData") + procPdhGetFormattedCounterValue = modpdh.NewProc("PdhGetFormattedCounterValue") + procPdhGetRawCounterValue = modpdh.NewProc("PdhGetRawCounterValue") + procPdhCalculateCounterFromRawValue = modpdh.NewProc("PdhCalculateCounterFromRawValue") + procPdhFormatFromRawValue = modpdh.NewProc("PdhFormatFromRawValue") + procPdhCloseQuery = modpdh.NewProc("PdhCloseQuery") ) func _PdhOpenQuery(dataSource *uint16, userData uintptr, query *PdhQueryHandle) (errcode error) { @@ -62,6 +65,30 @@ func _PdhGetFormattedCounterValue(counter PdhCounterHandle, format PdhCounterFor return } +func _PdhGetRawCounterValue(counter PdhCounterHandle, counterType *uint32, value *PdhRawCounter) (errcode error) { + r0, _, _ := syscall.Syscall(procPdhGetRawCounterValue.Addr(), 3, uintptr(counter), uintptr(unsafe.Pointer(counterType)), uintptr(unsafe.Pointer(value))) + if r0 != 0 { + errcode = syscall.Errno(r0) + } + return +} + +func _PdhCalculateCounterFromRawValue(counter PdhCounterHandle, format PdhCounterFormat, rawValue1 *PdhRawCounter, rawValue2 *PdhRawCounter, value *PdhCounterValue) (errcode error) { + r0, _, _ := syscall.Syscall6(procPdhCalculateCounterFromRawValue.Addr(), 5, uintptr(counter), uintptr(format), uintptr(unsafe.Pointer(rawValue1)), uintptr(unsafe.Pointer(rawValue2)), uintptr(unsafe.Pointer(value)), 0) + if r0 != 0 { + errcode = syscall.Errno(r0) + } + return +} + +func _PdhFormatFromRawValue(counterType uint32, format PdhCounterFormat, timeBase *uint64, rawValue1 *PdhRawCounter, rawValue2 *PdhRawCounter, value *PdhCounterValue) (errcode error) { + r0, _, _ := syscall.Syscall6(procPdhFormatFromRawValue.Addr(), 6, uintptr(counterType), uintptr(format), uintptr(unsafe.Pointer(timeBase)), uintptr(unsafe.Pointer(rawValue1)), uintptr(unsafe.Pointer(rawValue2)), uintptr(unsafe.Pointer(value))) + if r0 != 0 { + errcode = syscall.Errno(r0) + } + return +} + func _PdhCloseQuery(query PdhQueryHandle) (errcode error) { r0, _, _ := syscall.Syscall(procPdhCloseQuery.Addr(), 1, uintptr(query), 0, 0) if r0 != 0 { From af609585a1cdf15a587eb622031b1347566010c9 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Tue, 25 Apr 2017 09:55:13 +0200 Subject: [PATCH 15/40] Add x-pack monitoring note to kibana/es docs (#4012) * Remove kibana from the short config and comment it out by default --- metricbeat/docs/index.asciidoc | 1 + metricbeat/docs/modules/elasticsearch.asciidoc | 4 ++-- metricbeat/docs/modules/kibana.asciidoc | 6 ++---- metricbeat/metricbeat.full.yml | 1 - metricbeat/module/elasticsearch/_meta/docs.asciidoc | 4 ++-- metricbeat/module/kibana/_meta/config.yml | 1 - metricbeat/module/kibana/_meta/docs.asciidoc | 5 ++--- metricbeat/module/kibana/_meta/fields.yml | 1 + 8 files changed, 10 insertions(+), 13 deletions(-) diff --git a/metricbeat/docs/index.asciidoc b/metricbeat/docs/index.asciidoc index 85c043bafda..cecd6a094a0 100644 --- a/metricbeat/docs/index.asciidoc +++ b/metricbeat/docs/index.asciidoc @@ -12,6 +12,7 @@ include::../../libbeat/docs/version.asciidoc[] :beatname_lc: metricbeat :beatname_uc: Metricbeat :security: X-Pack Security +:monitoringdoc: https://www.elastic.co/guide/en/x-pack/current include::./overview.asciidoc[] diff --git a/metricbeat/docs/modules/elasticsearch.asciidoc b/metricbeat/docs/modules/elasticsearch.asciidoc index c1c58e5022a..3de0f509783 100644 --- a/metricbeat/docs/modules/elasticsearch.asciidoc +++ b/metricbeat/docs/modules/elasticsearch.asciidoc @@ -3,9 +3,9 @@ This file is generated! See scripts/docs_collector.py //// [[metricbeat-module-elasticsearch]] -== elasticsearch Module +== Elasticsearch Module -This is the elasticsearch Module. The elasticsearch module contains a minimal set of metrics to enable monitoring of elasticsearch across multiple versions. +This is the Elasticsearch Module. The Elasticsearch module contains a minimal set of metrics to enable monitoring of Elasticsearch across multiple versions. To monitor more Elasticsearch metrics, use our {monitoringdoc}/xpack-monitoring.html[X-Pack monitoring] which is available under a free basic license. [float] diff --git a/metricbeat/docs/modules/kibana.asciidoc b/metricbeat/docs/modules/kibana.asciidoc index 2c418f18488..ff90a453ce2 100644 --- a/metricbeat/docs/modules/kibana.asciidoc +++ b/metricbeat/docs/modules/kibana.asciidoc @@ -3,10 +3,9 @@ This file is generated! See scripts/docs_collector.py //// [[metricbeat-module-kibana]] -== kibana Module - -This is the kibana Module. +== Kibana Module +This is the Kibana Module and only tracks the high level metrics. To monitor more Kibana metrics, use our {monitoringdoc}/xpack-monitoring.html[X-Pack monitoring] which is available under a free basic license. [float] @@ -23,7 +22,6 @@ metricbeat.modules: enabled: false period: 10s hosts: ["localhost:5601"] - ---- [float] diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index cecf656b791..121eed7b6ca 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -207,7 +207,6 @@ metricbeat.modules: period: 10s hosts: ["localhost:5601"] - #------------------------------- kubelet Module ------------------------------ - module: kubelet metricsets: ["node","container","volume","pod","system"] diff --git a/metricbeat/module/elasticsearch/_meta/docs.asciidoc b/metricbeat/module/elasticsearch/_meta/docs.asciidoc index 1afc2224b3f..e912f0c86eb 100644 --- a/metricbeat/module/elasticsearch/_meta/docs.asciidoc +++ b/metricbeat/module/elasticsearch/_meta/docs.asciidoc @@ -1,3 +1,3 @@ -== elasticsearch Module +== Elasticsearch Module -This is the elasticsearch Module. The elasticsearch module contains a minimal set of metrics to enable monitoring of elasticsearch across multiple versions. +This is the Elasticsearch Module. The Elasticsearch module contains a minimal set of metrics to enable monitoring of Elasticsearch across multiple versions. To monitor more Elasticsearch metrics, use our {monitoringdoc}/xpack-monitoring.html[X-Pack monitoring] which is available under a free basic license. diff --git a/metricbeat/module/kibana/_meta/config.yml b/metricbeat/module/kibana/_meta/config.yml index 52a838ec4b9..bd673be5658 100644 --- a/metricbeat/module/kibana/_meta/config.yml +++ b/metricbeat/module/kibana/_meta/config.yml @@ -3,4 +3,3 @@ enabled: false period: 10s hosts: ["localhost:5601"] - diff --git a/metricbeat/module/kibana/_meta/docs.asciidoc b/metricbeat/module/kibana/_meta/docs.asciidoc index b88abe0ee65..829b0ab1beb 100644 --- a/metricbeat/module/kibana/_meta/docs.asciidoc +++ b/metricbeat/module/kibana/_meta/docs.asciidoc @@ -1,4 +1,3 @@ -== kibana Module - -This is the kibana Module. +== Kibana Module +This is the Kibana Module and only tracks the high level metrics. To monitor more Kibana metrics, use our {monitoringdoc}/xpack-monitoring.html[X-Pack monitoring] which is available under a free basic license. diff --git a/metricbeat/module/kibana/_meta/fields.yml b/metricbeat/module/kibana/_meta/fields.yml index 499a9f03431..b2dbd6f9ee3 100644 --- a/metricbeat/module/kibana/_meta/fields.yml +++ b/metricbeat/module/kibana/_meta/fields.yml @@ -4,6 +4,7 @@ []experimental kibana Module + short_config: false fields: - name: kibana type: group From 5afda3d9f5559f57c6c2bc7ab8c55c9a9219a385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 25 Apr 2017 15:13:56 +0200 Subject: [PATCH 16/40] Minor enhancements to kubernetes processor (#4068) * Move kubernetes processor to root processors folder * Restart watch call on error * Flag kubernetes proccesor as beta * Fix config validation for `in_cluster` cases * Add kubernetes metadata fields * Put kubernetes processor fields under a common namespace --- filebeat/beater/filebeat.go | 4 +- filebeat/docs/fields.asciidoc | 48 +++++++++++++++++++ .../kubernetes/indexing.go | 2 +- .../kubernetes/indexing_test.go | 0 heartbeat/docs/fields.asciidoc | 48 +++++++++++++++++++ libbeat/beat/beat.go | 2 +- .../processors/kubernetes/_meta/fields.yml | 29 +++++++++++ .../{annotate => }/kubernetes/config.go | 0 .../{annotate => }/kubernetes/indexing.go | 8 +++- .../kubernetes/indexing_test.go | 15 ++++-- .../{annotate => }/kubernetes/kubernetes.go | 13 ++--- .../{annotate => }/kubernetes/podwatcher.go | 6 +-- metricbeat/beater/metricbeat.go | 4 +- metricbeat/docs/fields.asciidoc | 48 +++++++++++++++++++ .../kubernetes/indexing.go | 6 ++- .../kubernetes/indexing_test.go | 11 +++-- packetbeat/docs/fields.asciidoc | 48 +++++++++++++++++++ winlogbeat/docs/fields.asciidoc | 48 +++++++++++++++++++ 18 files changed, 313 insertions(+), 27 deletions(-) rename filebeat/{processor/annotate => processors}/kubernetes/indexing.go (95%) rename filebeat/{processor/annotate => processors}/kubernetes/indexing_test.go (100%) create mode 100644 libbeat/processors/kubernetes/_meta/fields.yml rename libbeat/processors/{annotate => }/kubernetes/config.go (100%) rename libbeat/processors/{annotate => }/kubernetes/indexing.go (97%) rename libbeat/processors/{annotate => }/kubernetes/indexing_test.go (96%) rename libbeat/processors/{annotate => }/kubernetes/kubernetes.go (95%) rename libbeat/processors/{annotate => }/kubernetes/podwatcher.go (97%) rename metricbeat/{processor/annotate => processors}/kubernetes/indexing.go (94%) rename metricbeat/{processor/annotate => processors}/kubernetes/indexing_test.go (90%) diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index 91d40a26ee6..3d766d316d6 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -18,8 +18,8 @@ import ( "github.com/elastic/beats/filebeat/registrar" "github.com/elastic/beats/filebeat/spooler" - //Add filebeat level processors - _ "github.com/elastic/beats/filebeat/processor/annotate/kubernetes" + // Add filebeat level processors + _ "github.com/elastic/beats/filebeat/processors/kubernetes" ) var ( diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index b5b2aa9a6c8..6c53acf11be 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -17,6 +17,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -682,6 +683,53 @@ type: text The logged message. +[[exported-fields-kubernetes]] +== Kubernetes info Fields + +Kubernetes metadata added by the kubernetes processor + + + +[float] +=== kubernetes.pod.name + +type: keyword + +Kubernetes pod name + + +[float] +=== kubernetes.namespace + +type: keyword + +Kubernetes namespace + + +[float] +=== kubernetes.labels + +type: object + +Kubernetes labels map + + +[float] +=== kubernetes.annotations + +type: object + +Kubernetes annotations map + + +[float] +=== kubernetes.container.name + +type: keyword + +Kubernetes container name + + [[exported-fields-log]] == Log File Content Fields diff --git a/filebeat/processor/annotate/kubernetes/indexing.go b/filebeat/processors/kubernetes/indexing.go similarity index 95% rename from filebeat/processor/annotate/kubernetes/indexing.go rename to filebeat/processors/kubernetes/indexing.go index 981c5643564..9531e36fa7e 100644 --- a/filebeat/processor/annotate/kubernetes/indexing.go +++ b/filebeat/processors/kubernetes/indexing.go @@ -6,7 +6,7 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/libbeat/processors/annotate/kubernetes" + "github.com/elastic/beats/libbeat/processors/kubernetes" ) func init() { diff --git a/filebeat/processor/annotate/kubernetes/indexing_test.go b/filebeat/processors/kubernetes/indexing_test.go similarity index 100% rename from filebeat/processor/annotate/kubernetes/indexing_test.go rename to filebeat/processors/kubernetes/indexing_test.go diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc index e6f5e37819a..b64cdbd166a 100644 --- a/heartbeat/docs/fields.asciidoc +++ b/heartbeat/docs/fields.asciidoc @@ -15,6 +15,7 @@ grouped in the following categories: * <> * <> * <> +* <> -- [[exported-fields-beat]] @@ -356,3 +357,50 @@ required: True Boolean indicator if monitor could validate the service to be available. +[[exported-fields-kubernetes]] +== Kubernetes info Fields + +Kubernetes metadata added by the kubernetes processor + + + +[float] +=== kubernetes.pod.name + +type: keyword + +Kubernetes pod name + + +[float] +=== kubernetes.namespace + +type: keyword + +Kubernetes namespace + + +[float] +=== kubernetes.labels + +type: object + +Kubernetes labels map + + +[float] +=== kubernetes.annotations + +type: object + +Kubernetes annotations map + + +[float] +=== kubernetes.container.name + +type: keyword + +Kubernetes container name + + diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index f0438b16e8f..e9151c97ed7 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -64,7 +64,7 @@ import ( _ "github.com/elastic/beats/libbeat/processors/actions" _ "github.com/elastic/beats/libbeat/processors/add_cloud_metadata" _ "github.com/elastic/beats/libbeat/processors/add_locale" - _ "github.com/elastic/beats/libbeat/processors/annotate/kubernetes" + _ "github.com/elastic/beats/libbeat/processors/kubernetes" // Register default monitoring reporting _ "github.com/elastic/beats/libbeat/monitoring/report/elasticsearch" diff --git a/libbeat/processors/kubernetes/_meta/fields.yml b/libbeat/processors/kubernetes/_meta/fields.yml new file mode 100644 index 00000000000..4228291be0c --- /dev/null +++ b/libbeat/processors/kubernetes/_meta/fields.yml @@ -0,0 +1,29 @@ +- key: kubernetes + title: Kubernetes info + description: > + Kubernetes metadata added by the kubernetes processor + fields: + - name: kubernetes.pod.name + type: keyword + description: > + Kubernetes pod name + + - name: kubernetes.namespace + type: keyword + description: > + Kubernetes namespace + + - name: kubernetes.labels + type: object + description: > + Kubernetes labels map + + - name: kubernetes.annotations + type: object + description: > + Kubernetes annotations map + + - name: kubernetes.container.name + type: keyword + description: > + Kubernetes container name diff --git a/libbeat/processors/annotate/kubernetes/config.go b/libbeat/processors/kubernetes/config.go similarity index 100% rename from libbeat/processors/annotate/kubernetes/config.go rename to libbeat/processors/kubernetes/config.go diff --git a/libbeat/processors/annotate/kubernetes/indexing.go b/libbeat/processors/kubernetes/indexing.go similarity index 97% rename from libbeat/processors/annotate/kubernetes/indexing.go rename to libbeat/processors/kubernetes/indexing.go index 63b05b781eb..b296146911e 100644 --- a/libbeat/processors/annotate/kubernetes/indexing.go +++ b/libbeat/processors/kubernetes/indexing.go @@ -194,7 +194,9 @@ func (g *GenDefaultMeta) GenerateMetaData(pod *corev1.Pod) common.MapStr { annotationsMap = generateMapSubset(pod.Metadata.Annotations, g.annotations) meta := common.MapStr{ - "pod": pod.Metadata.GetName(), + "pod": common.MapStr{ + "name": pod.Metadata.GetName(), + }, "namespace": pod.Metadata.GetNamespace(), } @@ -263,7 +265,9 @@ func (c *ContainerIndexer) GetMetadata(pod *corev1.Pod) []MetadataIndex { var metadata []MetadataIndex for i := 0; i < len(containers); i++ { containerMeta := commonMeta.Clone() - containerMeta["container"] = pod.Status.ContainerStatuses[i].Name + containerMeta["container"] = common.MapStr{ + "name": pod.Status.ContainerStatuses[i].Name, + } metadata = append(metadata, MetadataIndex{ Index: containers[i], Data: containerMeta, diff --git a/libbeat/processors/annotate/kubernetes/indexing_test.go b/libbeat/processors/kubernetes/indexing_test.go similarity index 96% rename from libbeat/processors/annotate/kubernetes/indexing_test.go rename to libbeat/processors/kubernetes/indexing_test.go index da9830cfac1..e9a6f731f99 100644 --- a/libbeat/processors/annotate/kubernetes/indexing_test.go +++ b/libbeat/processors/kubernetes/indexing_test.go @@ -1,11 +1,12 @@ package kubernetes import ( + "testing" + "github.com/elastic/beats/libbeat/common" corev1 "github.com/ericchiang/k8s/api/v1" metav1 "github.com/ericchiang/k8s/apis/meta/v1" "github.com/stretchr/testify/assert" - "testing" ) var metagen = &GenDefaultMeta{} @@ -34,7 +35,9 @@ func TestPodIndexer(t *testing.T) { assert.Equal(t, indexers[0].Index, podName) expected := common.MapStr{ - "pod": "testpod", + "pod": common.MapStr{ + "name": "testpod", + }, "namespace": "testns", "labels": common.MapStr{ "labelkey": "labelvalue", @@ -76,7 +79,9 @@ func TestContainerIndexer(t *testing.T) { assert.Equal(t, len(indexers), 0) assert.Equal(t, len(indices), 0) expected := common.MapStr{ - "pod": "testpod", + "pod": common.MapStr{ + "name": "testpod", + }, "namespace": "testns", "labels": common.MapStr{ "labelkey": "labelvalue", @@ -91,7 +96,9 @@ func TestContainerIndexer(t *testing.T) { ContainerID: &cid, }, } - expected["container"] = container + expected["container"] = common.MapStr{ + "name": container, + } indexers = conIndexer.GetMetadata(&pod) assert.Equal(t, len(indexers), 1) diff --git a/libbeat/processors/annotate/kubernetes/kubernetes.go b/libbeat/processors/kubernetes/kubernetes.go similarity index 95% rename from libbeat/processors/annotate/kubernetes/kubernetes.go rename to libbeat/processors/kubernetes/kubernetes.go index cdeafe12946..76892adf08f 100644 --- a/libbeat/processors/annotate/kubernetes/kubernetes.go +++ b/libbeat/processors/kubernetes/kubernetes.go @@ -40,6 +40,8 @@ func init() { } func newKubernetesAnnotator(cfg common.Config) (processors.Processor, error) { + logp.Beta("The kubernetes processor is beta") + config := defaultKuberentesAnnotatorConfig() err := cfg.Unpack(&config) @@ -182,7 +184,6 @@ func newKubernetesAnnotator(cfg common.Config) (processors.Processor, error) { } func (k kubernetesAnnotator) Run(event common.MapStr) (common.MapStr, error) { - index := k.matchers.MetadataIndex(event) if index == "" { return event, nil @@ -194,15 +195,15 @@ func (k kubernetesAnnotator) Run(event common.MapStr) (common.MapStr, error) { } meta := common.MapStr{} - metaIface, ok := event["metadata"] + metaIface, ok := event["kubernetes"] if !ok { - event["metadata"] = common.MapStr{} + event["kubernetes"] = common.MapStr{} } else { meta = metaIface.(common.MapStr) } - meta["kubernetes"] = metadata - event["metadata"] = meta + meta.Update(metadata) + event["kubernetes"] = meta return event, nil } @@ -210,7 +211,7 @@ func (k kubernetesAnnotator) Run(event common.MapStr) (common.MapStr, error) { func (k kubernetesAnnotator) String() string { return "kubernetes" } func validate(config kubeAnnotatorConfig) error { - if config.KubeConfig == "" { + if !config.InCluster && config.KubeConfig == "" { return errors.New("`kube_config` path can't be empty when in_cluster is set to false") } return nil diff --git a/libbeat/processors/annotate/kubernetes/podwatcher.go b/libbeat/processors/kubernetes/podwatcher.go similarity index 97% rename from libbeat/processors/annotate/kubernetes/podwatcher.go rename to libbeat/processors/kubernetes/podwatcher.go index 688f987bc3f..0064e7e3dac 100644 --- a/libbeat/processors/annotate/kubernetes/podwatcher.go +++ b/libbeat/processors/kubernetes/podwatcher.go @@ -87,12 +87,12 @@ func (p *PodWatcher) watchPods() { time.Sleep(time.Second) continue } + for { _, pod, err := watcher.Next() if err != nil { - logp.Err("kubernetes: Watching API eror %v", err) - time.Sleep(time.Second) - continue + logp.Err("kubernetes: Watching API error %v", err) + break } p.podQueue <- pod diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index c1792166297..0165755d26b 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -13,8 +13,8 @@ import ( "github.com/elastic/beats/libbeat/cfgfile" "github.com/pkg/errors" - //Add metricbeat specific processors - _ "github.com/elastic/beats/metricbeat/processor/annotate/kubernetes" + // Add metricbeat specific processors + _ "github.com/elastic/beats/metricbeat/processors/kubernetes" ) // Metricbeat implements the Beater interface for metricbeat. diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index d773d07267a..ced1e37c9f6 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -27,6 +27,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -4447,6 +4448,53 @@ type: long Total inodes +[[exported-fields-kubernetes]] +== Kubernetes info Fields + +Kubernetes metadata added by the kubernetes processor + + + +[float] +=== kubernetes.pod.name + +type: keyword + +Kubernetes pod name + + +[float] +=== kubernetes.namespace + +type: keyword + +Kubernetes namespace + + +[float] +=== kubernetes.labels + +type: object + +Kubernetes labels map + + +[float] +=== kubernetes.annotations + +type: object + +Kubernetes annotations map + + +[float] +=== kubernetes.container.name + +type: keyword + +Kubernetes container name + + [[exported-fields-memcached]] == memcached Fields diff --git a/metricbeat/processor/annotate/kubernetes/indexing.go b/metricbeat/processors/kubernetes/indexing.go similarity index 94% rename from metricbeat/processor/annotate/kubernetes/indexing.go rename to metricbeat/processors/kubernetes/indexing.go index a1e9087444c..701a0148776 100644 --- a/metricbeat/processor/annotate/kubernetes/indexing.go +++ b/metricbeat/processors/kubernetes/indexing.go @@ -5,7 +5,7 @@ import ( "strings" "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/processors/annotate/kubernetes" + "github.com/elastic/beats/libbeat/processors/kubernetes" corev1 "github.com/ericchiang/k8s/api/v1" ) @@ -59,7 +59,9 @@ func (h *IpPortIndexer) GetMetadata(pod *corev1.Pod) []kubernetes.MetadataIndex continue } if strings.Index(hostPorts[i], fmt.Sprintf("%s:%d", *pod.Status.PodIP, *port.ContainerPort)) != -1 { - containerMeta["container"] = container.Name + containerMeta["container"] = common.MapStr{ + "name": container.Name, + } dobreak = true break } diff --git a/metricbeat/processor/annotate/kubernetes/indexing_test.go b/metricbeat/processors/kubernetes/indexing_test.go similarity index 90% rename from metricbeat/processor/annotate/kubernetes/indexing_test.go rename to metricbeat/processors/kubernetes/indexing_test.go index 7777bbd93c2..dd985c55d92 100644 --- a/metricbeat/processor/annotate/kubernetes/indexing_test.go +++ b/metricbeat/processors/kubernetes/indexing_test.go @@ -2,12 +2,13 @@ package kubernetes import ( "fmt" + "testing" + "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/processors/annotate/kubernetes" + "github.com/elastic/beats/libbeat/processors/kubernetes" corev1 "github.com/ericchiang/k8s/api/v1" metav1 "github.com/ericchiang/k8s/apis/meta/v1" "github.com/stretchr/testify/assert" - "testing" ) var metagen = &kubernetes.GenDefaultMeta{} @@ -45,7 +46,9 @@ func TestIpPortIndexer(t *testing.T) { assert.Equal(t, len(indexers), 0) assert.Equal(t, len(indices), 0) expected := common.MapStr{ - "pod": "testpod", + "pod": common.MapStr{ + "name": "testpod", + }, "namespace": "testns", "labels": common.MapStr{ "labelkey": "labelvalue", @@ -63,7 +66,7 @@ func TestIpPortIndexer(t *testing.T) { }, }, } - expected["container"] = container + expected["container"] = common.MapStr{"name": container} indexers = ipIndexer.GetMetadata(&pod) assert.Equal(t, len(indexers), 1) diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index 5e3191b73f8..ff4b99dab57 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -21,6 +21,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -1901,6 +1902,53 @@ type: long The response code. +[[exported-fields-kubernetes]] +== Kubernetes info Fields + +Kubernetes metadata added by the kubernetes processor + + + +[float] +=== kubernetes.pod.name + +type: keyword + +Kubernetes pod name + + +[float] +=== kubernetes.namespace + +type: keyword + +Kubernetes namespace + + +[float] +=== kubernetes.labels + +type: object + +Kubernetes labels map + + +[float] +=== kubernetes.annotations + +type: object + +Kubernetes annotations map + + +[float] +=== kubernetes.container.name + +type: keyword + +Kubernetes container name + + [[exported-fields-memcache]] == Memcache Fields diff --git a/winlogbeat/docs/fields.asciidoc b/winlogbeat/docs/fields.asciidoc index 842ea03e08a..ce747921ab0 100644 --- a/winlogbeat/docs/fields.asciidoc +++ b/winlogbeat/docs/fields.asciidoc @@ -16,6 +16,7 @@ grouped in the following categories: * <> * <> * <> +* <> -- [[exported-fields-beat]] @@ -427,3 +428,50 @@ The raw XML representation of the event obtained from Windows. This field is onl The XML representation of the event is useful for troubleshooting purposes. The data in the fields reported by Winlogbeat can be compared to the data in the XML to diagnose problems. +[[exported-fields-kubernetes]] +== Kubernetes info Fields + +Kubernetes metadata added by the kubernetes processor + + + +[float] +=== kubernetes.pod.name + +type: keyword + +Kubernetes pod name + + +[float] +=== kubernetes.namespace + +type: keyword + +Kubernetes namespace + + +[float] +=== kubernetes.labels + +type: object + +Kubernetes labels map + + +[float] +=== kubernetes.annotations + +type: object + +Kubernetes annotations map + + +[float] +=== kubernetes.container.name + +type: keyword + +Kubernetes container name + + From 354fdd6792b77ed5f099e017caab7c8ec4fa2ead Mon Sep 17 00:00:00 2001 From: Vijay Samuel Date: Tue, 25 Apr 2017 13:04:24 -0700 Subject: [PATCH 17/40] Adding query APIs for metricsets and modules from metricbeat registry (#4102) * Adding query APIs for metricsets and modules from metricbeat registry * Added locking to make `mb.Register` thread-safe. --- CHANGELOG.asciidoc | 1 + metricbeat/mb/registry.go | 53 ++++++++++++++++++++++++++++++++-- metricbeat/mb/registry_test.go | 24 +++++++++++++++ 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 14993bf24bb..8fe093b551c 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -144,6 +144,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Add new MetricSet interfaces for developers (`Closer`, `ReportingFetcher`, and `PushMetricSet`). {pull}3908[3908] - Add kubelet module {pull}3916[3916] - Add dropwizard module {pull}4022[4022] +- Adding query APIs for metricsets and modules from metricbeat registry {pull}4102[4102] *Packetbeat* - Add `fields` and `fields_under_root` to packetbeat protocols configurations. {pull}3518[3518] diff --git a/metricbeat/mb/registry.go b/metricbeat/mb/registry.go index 3dca6636b0c..fb8473ba792 100644 --- a/metricbeat/mb/registry.go +++ b/metricbeat/mb/registry.go @@ -4,6 +4,7 @@ import ( "fmt" "sort" "strings" + "sync" "github.com/elastic/beats/libbeat/logp" ) @@ -44,8 +45,10 @@ type metricSetFactoryInfo struct { } // Register contains the factory functions for creating new Modules and new -// MetricSets. +// MetricSets. Registers are thread safe for concurrent usage. type Register struct { + // Lock to control concurrent read/writes + lock sync.RWMutex // A map of module name to ModuleFactory. modules map[string]ModuleFactory // A map of module name to nested map of MetricSet name to metricSetFactoryInfo. @@ -64,6 +67,9 @@ func NewRegister() *Register { // name is empty, factory is nil, or if a factory has already been registered // under the name. func (r *Register) AddModule(name string, factory ModuleFactory) error { + r.lock.Lock() + defer r.lock.Unlock() + if name == "" { return fmt.Errorf("module name is required") } @@ -89,6 +95,9 @@ func (r *Register) AddModule(name string, factory ModuleFactory) error { // returned if any parameter is empty or nil or if a factory has already been // registered under the name. func (r *Register) AddMetricSet(module string, name string, factory MetricSetFactory, hostParser ...HostParser) error { + r.lock.Lock() + defer r.lock.Unlock() + if module == "" { return fmt.Errorf("module name is required") } @@ -122,12 +131,18 @@ func (r *Register) AddMetricSet(module string, name string, factory MetricSetFac // moduleFactory returns the registered ModuleFactory associated with the // given name. It returns nil if no ModuleFactory is registered. func (r *Register) moduleFactory(name string) ModuleFactory { + r.lock.RLock() + defer r.lock.RUnlock() + return r.modules[strings.ToLower(name)] } // metricSetFactory returns the registered MetricSetFactory associated with the // given name. It returns an error if no MetricSetFactory is registered. func (r *Register) metricSetFactory(module, name string) (MetricSetFactory, HostParser, error) { + r.lock.RLock() + defer r.lock.RUnlock() + module = strings.ToLower(module) name = strings.ToLower(name) @@ -144,9 +159,43 @@ func (r *Register) metricSetFactory(module, name string) (MetricSetFactory, Host return info.factory, info.hostParser, nil } +//Modules returns the list of module names that are registered +func (r *Register) Modules() []string { + r.lock.RLock() + defer r.lock.RUnlock() + + modules := make([]string, 0, len(r.modules)) + for module := range r.modules { + modules = append(modules, module) + } + + return modules +} + +//MetricSets returns the list of metricsets registered for a given module +func (r *Register) MetricSets(module string) []string { + r.lock.RLock() + defer r.lock.RUnlock() + + var metricsets []string + + sets, ok := r.metricSets[module] + if ok { + metricsets = make([]string, 0, len(sets)) + for name := range sets { + metricsets = append(metricsets, name) + } + } + + return metricsets +} + // String return a string representation of the registered ModuleFactory's and // MetricSetFactory's. -func (r Register) String() string { +func (r *Register) String() string { + r.lock.RLock() + defer r.lock.RUnlock() + var modules []string for module := range r.modules { modules = append(modules, module) diff --git a/metricbeat/mb/registry_test.go b/metricbeat/mb/registry_test.go index 7766df7b1db..4ae8b12d464 100644 --- a/metricbeat/mb/registry_test.go +++ b/metricbeat/mb/registry_test.go @@ -150,3 +150,27 @@ func TestMetricSetFactory(t *testing.T) { assert.NotNil(t, hp) // Can't compare functions in Go so just check for non-nil. }) } + +func TestMetricSetQuery(t *testing.T) { + registry := NewRegister() + err := registry.AddMetricSet(moduleName, metricSetName, fakeMetricSetFactory) + if err != nil { + t.Fatal(err) + } + + metricsets := registry.MetricSets(moduleName) + assert.Equal(t, len(metricsets), 1) + assert.Equal(t, metricsets[0], metricSetName) + + metricsets = registry.MetricSets("foo") + assert.Equal(t, len(metricsets), 0) +} + +func TestModuleQuery(t *testing.T) { + registry := NewRegister() + registry.modules[moduleName] = fakeModuleFactory + + modules := registry.Modules() + assert.Equal(t, len(modules), 1) + assert.Equal(t, modules[0], moduleName) +} From ca2e449353625a5f35cae3d92fffa3f02c2444ff Mon Sep 17 00:00:00 2001 From: Giuseppe Valente <7AC@users.noreply.github.com> Date: Tue, 25 Apr 2017 13:22:03 -0700 Subject: [PATCH 18/40] Suggest GOPATH in ~/go (#4110) --- CONTRIBUTING.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8425cad4866..961f1a7140f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,7 +55,7 @@ used for development is Golang 1.8.1. The location where you clone is important. Please clone under the source directory of your `GOPATH`. If you don't have `GOPATH` already set, you can -simply set it to your home directory (`export GOPATH=$HOME`). +simply set it to the `go` directory in your home (`export GOPATH=$HOME/go`). $ mkdir -p ${GOPATH}/src/github.com/elastic $ cd ${GOPATH}/src/github.com/elastic @@ -116,4 +116,3 @@ To manage the `vendor/` folder we use the govendor documentation on how to add or update vendored dependencies. In most cases `govendor fetch your/dependency@version +out` will get the job done. - From aaee997204159b9cfdf10a4425ae331bd92a2b58 Mon Sep 17 00:00:00 2001 From: Giuseppe Valente <7AC@users.noreply.github.com> Date: Tue, 25 Apr 2017 23:21:32 -0700 Subject: [PATCH 19/40] filebeat: expand double wildcards in prospector (#3980) Expand double wildcards into standard glob patterns, up to a maximum depth of 8 levels after the wildcard. Resolves https://github.com/elastic/beats/issues/2084 --- CHANGELOG.asciidoc | 1 + filebeat/_meta/common.full.p2.yml | 5 + .../configuration/filebeat-options.asciidoc | 15 +- filebeat/filebeat.full.yml | 5 + filebeat/input/file/glob.go | 71 +++++++++ filebeat/input/file/glob_other_test.go | 140 ++++++++++++++++++ filebeat/input/file/glob_test.go | 82 ++++++++++ filebeat/input/file/glob_windows_test.go | 92 ++++++++++++ filebeat/prospector/config.go | 1 + filebeat/prospector/prospector_log.go | 15 +- 10 files changed, 420 insertions(+), 7 deletions(-) create mode 100644 filebeat/input/file/glob.go create mode 100644 filebeat/input/file/glob_other_test.go create mode 100644 filebeat/input/file/glob_test.go create mode 100644 filebeat/input/file/glob_windows_test.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 8fe093b551c..f9d7f2e4505 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -121,6 +121,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Add base for supporting prospector level processors {pull}3853[3853] - Add auditd module for reading audit logs on Linux. {pull}3750[3750] {pull}3941[3941] - Add filebeat.config.path as replacement for config_dir. {pull}4051[4051] +- Add a `recursive_glob.enabled` setting to expand "**" in patterns. {{pull}}3980[3980] *Heartbeat* diff --git a/filebeat/_meta/common.full.p2.yml b/filebeat/_meta/common.full.p2.yml index bc949f75aef..a70f75f00a8 100644 --- a/filebeat/_meta/common.full.p2.yml +++ b/filebeat/_meta/common.full.p2.yml @@ -82,6 +82,11 @@ filebeat.prospectors: # This is especially useful for multiline log messages which can get large. #max_bytes: 10485760 + ### Recursive glob configuration + + # Expand "**" patterns into regular glob patterns. + #recursive_glob.enabled: true + ### JSON configuration # Decode JSON options. Enable this if your logs are structured in JSON. diff --git a/filebeat/docs/reference/configuration/filebeat-options.asciidoc b/filebeat/docs/reference/configuration/filebeat-options.asciidoc index dd072016d7e..9af1dde0e2a 100644 --- a/filebeat/docs/reference/configuration/filebeat-options.asciidoc +++ b/filebeat/docs/reference/configuration/filebeat-options.asciidoc @@ -41,12 +41,22 @@ supported by https://golang.org/pkg/path/filepath/#Glob[Golang Glob] are also supported here. For example, to fetch all files from a predefined level of subdirectories, the following pattern can be used: `/var/log/*/*.log`. This fetches all `.log` files from the subfolders of `/var/log`. It does not -fetch log files from the `/var/log` folder itself. Currently it is not possible -to recursively fetch all files in all subdirectories of a directory. +fetch log files from the `/var/log` folder itself. +It is possible to recursively fetch all files in all subdirectories of a directory +using the optional <> settings. Filebeat starts a harvester for each file that it finds under the specified paths. You can specify one path per line. Each line begins with a dash (-). +[[recursive_glob]] +===== recursive_glob + +*`enabled`*:: Enable expanding `**` into recursive glob patterns. With this feature enabled, +the rightmost `**` in each path is expanded into a fixed +number of glob patterns. For example: `/foo/**` expands to `/foo`, `/foo/*`, +`/foo/*/*`, and so on. The feature is disabled by default, and if enabled it expands a single `**` +into a 8-level deep `*` pattern. + ===== encoding The file encoding to use for reading files that contain international characters. @@ -591,4 +601,3 @@ include::../../../../libbeat/docs/dashboardsconfig.asciidoc[] include::../../../../libbeat/docs/loggingconfig.asciidoc[] include::../../../../libbeat/docs/processors-config.asciidoc[] - diff --git a/filebeat/filebeat.full.yml b/filebeat/filebeat.full.yml index 52c2f3583fc..18ef8a7a576 100644 --- a/filebeat/filebeat.full.yml +++ b/filebeat/filebeat.full.yml @@ -249,6 +249,11 @@ filebeat.prospectors: # This is especially useful for multiline log messages which can get large. #max_bytes: 10485760 + ### Recursive glob configuration + + # Expand "**" patterns into regular glob patterns. + #recursive_glob.enabled: true + ### JSON configuration # Decode JSON options. Enable this if your logs are structured in JSON. diff --git a/filebeat/input/file/glob.go b/filebeat/input/file/glob.go new file mode 100644 index 00000000000..71917fcbaa4 --- /dev/null +++ b/filebeat/input/file/glob.go @@ -0,0 +1,71 @@ +package file + +import ( + "fmt" + "path/filepath" +) + +func wildcards(doubleStarPatternDepth uint8, dir string, suffix string) []string { + wildcardList := []string{} + w := "" + i := uint8(0) + if dir == "" && suffix == "" { + // Don't expand to "" on relative paths + w = "*" + i = 1 + } + for ; i <= doubleStarPatternDepth; i++ { + wildcardList = append(wildcardList, w) + w = filepath.Join(w, "*") + } + return wildcardList +} + +// globPattern detects the use of "**" and expands it to standard glob patterns up to a max depth +func globPatterns(pattern string, doubleStarPatternDepth uint8) ([]string, error) { + if doubleStarPatternDepth == 0 { + return []string{pattern}, nil + } + var wildcardList []string + var prefix string + var suffix string + dir, file := filepath.Split(filepath.Clean(pattern)) + for file != "" && file != "." { + if file == "**" { + if len(wildcardList) > 0 { + return nil, fmt.Errorf("multiple ** in %q", pattern) + } + wildcardList = wildcards(doubleStarPatternDepth, dir, suffix) + prefix = dir + } else if len(wildcardList) == 0 { + suffix = filepath.Join(file, suffix) + } + dir, file = filepath.Split(filepath.Clean(dir)) + } + if len(wildcardList) == 0 { + return []string{pattern}, nil + } + var patterns []string + for _, w := range wildcardList { + patterns = append(patterns, filepath.Join(prefix, w, suffix)) + } + return patterns, nil +} + +// Glob expands '**' patterns into multiple patterns to satisfy https://golang.org/pkg/path/filepath/#Match +func Glob(pattern string, doubleStarPatternDepth uint8) ([]string, error) { + patterns, err := globPatterns(pattern, doubleStarPatternDepth) + if err != nil { + return nil, err + } + var matches []string + for _, p := range patterns { + // Evaluate the path as a wildcards/shell glob + match, err := filepath.Glob(p) + if err != nil { + return nil, err + } + matches = append(matches, match...) + } + return matches, nil +} diff --git a/filebeat/input/file/glob_other_test.go b/filebeat/input/file/glob_other_test.go new file mode 100644 index 00000000000..23373bf5a49 --- /dev/null +++ b/filebeat/input/file/glob_other_test.go @@ -0,0 +1,140 @@ +// +build !windows + +package file + +var globTests = []globTest{ + { + "*", + []string{ + "foo", + }, + }, + { + "foo/*", + []string{ + "foo/bar", + }, + }, + { + "*/*", + []string{ + "foo/bar", + }, + }, + { + "**", + []string{ + "", + "foo", + "foo/bar", + "foo/bar/baz", + "foo/bar/baz/qux", + }, + }, + { + "foo**", + []string{ + "foo", + }, + }, + { + "foo/**", + []string{ + "foo", + "foo/bar", + "foo/bar/baz", + "foo/bar/baz/qux", + "foo/bar/baz/qux/quux", + }, + }, + { + "foo/**/baz", + []string{ + "foo/bar/baz", + }, + }, + { + "foo/**/bazz", + []string{}, + }, + { + "foo/**/bar", + []string{ + "foo/bar", + }, + }, + { + "foo//bar", + []string{ + "foo/bar", + }, + }, +} + +var globPatternsTests = []globPatternsTest{ + { + "**", + []string{"*", "*/*"}, + false, + }, + { + "/**", + []string{"/", "/*", "/*/*"}, + false, + }, + { + "**/", + []string{"*", "*/*"}, + false, + }, + { + "/foo/**", + []string{"/foo", "/foo/*", "/foo/*/*"}, + false, + }, + { + "/foo/**/bar", + []string{"/foo/bar", "/foo/*/bar", "/foo/*/*/bar"}, + false, + }, + { + "**/bar", + []string{"bar", "*/bar", "*/*/bar"}, + false, + }, + { + "/**/bar", + []string{"/bar", "/*/bar", "/*/*/bar"}, + false, + }, + { + "**/**", + []string{"*", "*/*"}, + true, + }, + { + "/**/**", + []string{"*", "*/*"}, + true, + }, + { + "foo**/bar", + []string{"foo**/bar"}, + false, + }, + { + "**foo/bar", + []string{"**foo/bar"}, + false, + }, + { + "foo/**bar", + []string{"foo/**bar"}, + false, + }, + { + "foo/bar**", + []string{"foo/bar**"}, + false, + }, +} diff --git a/filebeat/input/file/glob_test.go b/filebeat/input/file/glob_test.go new file mode 100644 index 00000000000..aa9d2c58e63 --- /dev/null +++ b/filebeat/input/file/glob_test.go @@ -0,0 +1,82 @@ +package file + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +type globTest struct { + pattern string + expectedMatches []string +} + +func TestGlob(t *testing.T) { + root, err := ioutil.TempDir("", "testglob") + if err != nil { + t.Fatal(err) + } + os.MkdirAll(filepath.Join(root, "foo/bar/baz/qux/quux"), 0755) + for _, test := range globTests { + pattern := filepath.Join(root, test.pattern) + matches, err := Glob(pattern, 4) + if err != nil { + t.Fatal(err) + continue + } + var normalizedMatches []string + for _, m := range matches { + if len(m) < len(root) { + t.Fatalf("Matches for %q are expected to be under %s and %q is not", test.pattern, root, m) + } + var normalizedMatch string + if len(m) > len(root) { + normalizedMatch = m[len(root)+1:] + } else { + normalizedMatch = m[len(root):] + } + normalizedMatches = append(normalizedMatches, normalizedMatch) + } + matchError := func() { + t.Fatalf("Pattern %q matched %q instead of %q", test.pattern, normalizedMatches, test.expectedMatches) + } + if len(normalizedMatches) != len(test.expectedMatches) { + matchError() + continue + } + for i, expectedMatch := range test.expectedMatches { + if normalizedMatches[i] != expectedMatch { + matchError() + } + } + } +} + +type globPatternsTest struct { + pattern string + expectedPatterns []string + expectedError bool +} + +func TestGlobPatterns(t *testing.T) { + for _, test := range globPatternsTests { + patterns, err := globPatterns(test.pattern, 2) + if err != nil { + if test.expectedError { + continue + } + t.Fatal(err) + } + if len(patterns) != len(test.expectedPatterns) { + t.Fatalf("%q expanded to %q (%d) instead of %q (%d)", test.pattern, patterns, len(patterns), + test.expectedPatterns, len(test.expectedPatterns)) + } + for i, p := range patterns { + if p != test.expectedPatterns[i] { + t.Fatalf("%q expanded to %q instead of %q", test.pattern, patterns, test.expectedPatterns) + break + } + } + } +} diff --git a/filebeat/input/file/glob_windows_test.go b/filebeat/input/file/glob_windows_test.go new file mode 100644 index 00000000000..aadbe8979fd --- /dev/null +++ b/filebeat/input/file/glob_windows_test.go @@ -0,0 +1,92 @@ +// +build windows + +package file + +var globTests = []globTest{ + { + "*", + []string{ + "foo", + }, + }, + { + "foo\\*", + []string{ + "foo\\bar", + }, + }, + { + "foo/*", + []string{ + "foo\\bar", + }, + }, + { + "*\\*", + []string{ + "foo\\bar", + }, + }, + { + "**", + []string{ + "", + "foo", + "foo\\bar", + "foo\\bar\\baz", + "foo\\bar\\baz\\qux", + }, + }, + { + "foo**", + []string{ + "foo", + }, + }, + { + "foo\\**", + []string{ + "foo", + "foo\\bar", + "foo\\bar\\baz", + "foo\\bar\\baz\\qux", + "foo\\bar\\baz\\qux\\quux", + }, + }, + { + "foo\\**\\baz", + []string{ + "foo\\bar\\baz", + }, + }, + { + "foo/**\\baz", + []string{ + "foo\\bar\\baz", + }, + }, + { + "foo\\**\\bazz", + []string{}, + }, + { + "foo\\**\\bar", + []string{ + "foo\\bar", + }, + }, + { + "foo\\\\bar", + []string{ + "foo\\bar", + }, + }, +} + +var globPatternsTests = []globPatternsTest{ + { + "C:\\foo\\**\\bar", + []string{"C:\\foo\\bar", "C:\\foo\\*\\bar", "C:\\foo\\*\\*\\bar"}, + false, + }, +} diff --git a/filebeat/prospector/config.go b/filebeat/prospector/config.go index 839fa2e7932..1cef49f67b4 100644 --- a/filebeat/prospector/config.go +++ b/filebeat/prospector/config.go @@ -45,6 +45,7 @@ type prospectorConfig struct { Module string `config:"_module_name"` // hidden option to set the module name Fileset string `config:"_fileset_name"` // hidden option to set the fileset name Processors processors.PluginConfig `config:"processors"` + recursiveGlob bool `config:"recursive_glob.enabled"` } func (config *prospectorConfig) Validate() error { diff --git a/filebeat/prospector/prospector_log.go b/filebeat/prospector/prospector_log.go index 9ddfb4c5af9..01ad464263d 100644 --- a/filebeat/prospector/prospector_log.go +++ b/filebeat/prospector/prospector_log.go @@ -13,6 +13,10 @@ import ( "github.com/elastic/beats/libbeat/monitoring" ) +const ( + recursiveGlobDepth = 8 +) + var ( filesRenamed = monitoring.NewInt(nil, "filebeat.prospector.log.files.renamed") filesTruncated = monitoring.NewInt(nil, "filebeat.prospector.log.files.truncated") @@ -140,11 +144,14 @@ func (l *Log) getFiles() map[string]os.FileInfo { paths := map[string]os.FileInfo{} - for _, glob := range l.config.Paths { - // Evaluate the path as a wildcards/shell glob - matches, err := filepath.Glob(glob) + for _, path := range l.config.Paths { + depth := uint8(0) + if l.config.recursiveGlob { + depth = recursiveGlobDepth + } + matches, err := file.Glob(path, depth) if err != nil { - logp.Err("glob(%s) failed: %v", glob, err) + logp.Err("glob(%s) failed: %v", path, err) continue } From dce68871f8d0e127a8511bf70862d02ebd4089df Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Wed, 26 Apr 2017 09:20:09 +0200 Subject: [PATCH 20/40] Disable default prospector and adjust short configs (#4105) This does two changes: * Adds `enabled: false` in the default prospector in the short and long configs. (which fixes #3442) * Updates the short config files of the modules to include the path definitions. I think this is better for a "module first" experience, where it gives a bit of context on how they work. This means that the default configuration starts and gives no errors, but also doesn't publish anything. IMO, this behaviour is fine considering configuration reloading. * Print error in case all prospectors are disabled and config reloading is off Also, add the `auth` fileset to the config samples, it was missing. --- filebeat/_meta/common.full.p2.yml | 3 + filebeat/_meta/common.p2.yml | 3 + filebeat/beater/filebeat.go | 12 ++- filebeat/filebeat.full.yml | 25 +++--- filebeat/filebeat.yml | 77 ++++++++++++++++++- filebeat/module/apache2/_meta/config.full.yml | 5 -- filebeat/module/apache2/_meta/config.yml | 16 ++++ filebeat/module/auditd/_meta/config.yml | 7 ++ filebeat/module/icinga/_meta/config.yml | 23 ++++++ filebeat/module/mysql/_meta/config.yml | 15 ++++ filebeat/module/nginx/_meta/config.full.yml | 5 -- filebeat/module/nginx/_meta/config.yml | 19 ++++- filebeat/module/system/_meta/config.full.yml | 12 +++ filebeat/module/system/_meta/config.yml | 15 ++++ filebeat/tests/system/test_prospector.py | 7 +- 15 files changed, 208 insertions(+), 36 deletions(-) create mode 100644 filebeat/module/apache2/_meta/config.yml diff --git a/filebeat/_meta/common.full.p2.yml b/filebeat/_meta/common.full.p2.yml index a70f75f00a8..6fdaa8cd4a8 100644 --- a/filebeat/_meta/common.full.p2.yml +++ b/filebeat/_meta/common.full.p2.yml @@ -16,6 +16,9 @@ filebeat.prospectors: #------------------------------ Log prospector -------------------------------- - input_type: log + # Change to true to enable this prospector configuration. + enabled: false + # Paths that should be crawled and fetched. Glob based paths. # To fetch all ".log" files from a specific level of subdirectories # /var/log/*/*.log can be used. diff --git a/filebeat/_meta/common.p2.yml b/filebeat/_meta/common.p2.yml index c7889efe882..80d7d03cc69 100644 --- a/filebeat/_meta/common.p2.yml +++ b/filebeat/_meta/common.p2.yml @@ -11,6 +11,9 @@ filebeat.prospectors: - input_type: log + # Change to true to enable this prospector configuration. + enabled: false + # Paths that should be crawled and fetched. Glob based paths. paths: - /var/log/*.log diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index 3d766d316d6..48d422c2dc7 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -57,8 +57,16 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) { // Add prospectors created by the modules config.Prospectors = append(config.Prospectors, moduleProspectors...) - if !config.ConfigProspector.Enabled() && len(config.Prospectors) == 0 { - return nil, errors.New("No prospectors defined. What files do you want me to watch?") + haveEnabledProspectors := false + for _, prospector := range config.Prospectors { + if prospector.Enabled() { + haveEnabledProspectors = true + break + } + } + + if !config.ConfigProspector.Enabled() && !haveEnabledProspectors { + return nil, errors.New("No modules or prospectors enabled and configuration reloading disabled. What files do you want me to watch?") } if *once && config.ConfigProspector.Enabled() { diff --git a/filebeat/filebeat.full.yml b/filebeat/filebeat.full.yml index 18ef8a7a576..6027f6c7568 100644 --- a/filebeat/filebeat.full.yml +++ b/filebeat/filebeat.full.yml @@ -25,17 +25,24 @@ filebeat.modules: # can be added under this section. #prospector: + # Authorization logs + #auth: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Prospector configuration (advanced). Any prospector configuration option + # can be added under this section. + #prospector: + #------------------------------- Apache2 Module ------------------------------ #- module: apache2 # Access logs #access: #enabled: true - # Ingest Node pipeline to use. Options are `with_plugins` (default) - # and `no_plugins`. Use `no_plugins` if you don't have the geoip or - # the user agent Node ingest plugins installed. - #var.pipeline: with_plugins - # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: @@ -139,11 +146,6 @@ filebeat.modules: #access: #enabled: true - # Ingest Node pipeline to use. Options are `with_plugins` (default) - # and `no_plugins`. Use `no_plugins` if you don't have the geoip or - # the user agent Node ingest plugins installed. - #var.pipeline: with_plugins - # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: @@ -183,6 +185,9 @@ filebeat.prospectors: #------------------------------ Log prospector -------------------------------- - input_type: log + # Change to true to enable this prospector configuration. + enabled: false + # Paths that should be crawled and fetched. Glob based paths. # To fetch all ".log" files from a specific level of subdirectories # /var/log/*/*.log can be used. diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index 545eac7b54a..790eee22a01 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -13,19 +13,85 @@ filebeat.modules: #------------------------------- System Module ------------------------------- #- module: system + # Syslog + #syslog: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Authorization logs + #auth: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + +#------------------------------- Apache2 Module ------------------------------ +#- module: apache2 + # Access logs + #access: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Error logs + #error: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: #------------------------------- Auditd Module ------------------------------- #- module: auditd + #log: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + #-------------------------------- MySQL Module ------------------------------- #- module: mysql + # Error logs + #error: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Slow logs + #slowlog: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: #-------------------------------- Nginx Module ------------------------------- #- module: nginx - # Ingest Node pipeline to use. Options are `with_plugins` (default) - # and `no_plugins`. Use `no_plugins` if you don't have the geoip or - # the user agent Node ingest plugins installed. - #access.var.pipeline: with_plugins + # Access logs + #access: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Error logs + #error: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: # For more available modules and options, please see the filebeat.full.yml sample @@ -41,6 +107,9 @@ filebeat.prospectors: - input_type: log + # Change to true to enable this prospector configuration. + enabled: false + # Paths that should be crawled and fetched. Glob based paths. paths: - /var/log/*.log diff --git a/filebeat/module/apache2/_meta/config.full.yml b/filebeat/module/apache2/_meta/config.full.yml index 44481fee9a8..91117963687 100644 --- a/filebeat/module/apache2/_meta/config.full.yml +++ b/filebeat/module/apache2/_meta/config.full.yml @@ -3,11 +3,6 @@ #access: #enabled: true - # Ingest Node pipeline to use. Options are `with_plugins` (default) - # and `no_plugins`. Use `no_plugins` if you don't have the geoip or - # the user agent Node ingest plugins installed. - #var.pipeline: with_plugins - # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: diff --git a/filebeat/module/apache2/_meta/config.yml b/filebeat/module/apache2/_meta/config.yml new file mode 100644 index 00000000000..6ccb548ad4f --- /dev/null +++ b/filebeat/module/apache2/_meta/config.yml @@ -0,0 +1,16 @@ +#- module: apache2 + # Access logs + #access: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Error logs + #error: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/filebeat/module/auditd/_meta/config.yml b/filebeat/module/auditd/_meta/config.yml index 844a9b2ee1a..5669924ba0b 100644 --- a/filebeat/module/auditd/_meta/config.yml +++ b/filebeat/module/auditd/_meta/config.yml @@ -1 +1,8 @@ #- module: auditd + #log: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + diff --git a/filebeat/module/icinga/_meta/config.yml b/filebeat/module/icinga/_meta/config.yml index 54e3e9fa68a..7657f4467ee 100644 --- a/filebeat/module/icinga/_meta/config.yml +++ b/filebeat/module/icinga/_meta/config.yml @@ -1 +1,24 @@ #- module: icinga + # Main logs + #main: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Debug logs + #debug: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Startup logs + #startup: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/filebeat/module/mysql/_meta/config.yml b/filebeat/module/mysql/_meta/config.yml index 22942e984f0..aadb5631092 100644 --- a/filebeat/module/mysql/_meta/config.yml +++ b/filebeat/module/mysql/_meta/config.yml @@ -1 +1,16 @@ #- module: mysql + # Error logs + #error: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Slow logs + #slowlog: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/filebeat/module/nginx/_meta/config.full.yml b/filebeat/module/nginx/_meta/config.full.yml index f80f92dbc3b..249d1bdb22f 100644 --- a/filebeat/module/nginx/_meta/config.full.yml +++ b/filebeat/module/nginx/_meta/config.full.yml @@ -3,11 +3,6 @@ #access: #enabled: true - # Ingest Node pipeline to use. Options are `with_plugins` (default) - # and `no_plugins`. Use `no_plugins` if you don't have the geoip or - # the user agent Node ingest plugins installed. - #var.pipeline: with_plugins - # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: diff --git a/filebeat/module/nginx/_meta/config.yml b/filebeat/module/nginx/_meta/config.yml index ae1f0dd933d..e1b45939726 100644 --- a/filebeat/module/nginx/_meta/config.yml +++ b/filebeat/module/nginx/_meta/config.yml @@ -1,5 +1,16 @@ #- module: nginx - # Ingest Node pipeline to use. Options are `with_plugins` (default) - # and `no_plugins`. Use `no_plugins` if you don't have the geoip or - # the user agent Node ingest plugins installed. - #access.var.pipeline: with_plugins + # Access logs + #access: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Error logs + #error: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/filebeat/module/system/_meta/config.full.yml b/filebeat/module/system/_meta/config.full.yml index eef3fa9aaf2..266677dc118 100644 --- a/filebeat/module/system/_meta/config.full.yml +++ b/filebeat/module/system/_meta/config.full.yml @@ -10,3 +10,15 @@ # Prospector configuration (advanced). Any prospector configuration option # can be added under this section. #prospector: + + # Authorization logs + #auth: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Prospector configuration (advanced). Any prospector configuration option + # can be added under this section. + #prospector: diff --git a/filebeat/module/system/_meta/config.yml b/filebeat/module/system/_meta/config.yml index 010586ba9b4..d31ca8bc6ea 100644 --- a/filebeat/module/system/_meta/config.yml +++ b/filebeat/module/system/_meta/config.yml @@ -1 +1,16 @@ #- module: system + # Syslog + #syslog: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Authorization logs + #auth: + #enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/filebeat/tests/system/test_prospector.py b/filebeat/tests/system/test_prospector.py index c704ae34193..1b746af0a6c 100644 --- a/filebeat/tests/system/test_prospector.py +++ b/filebeat/tests/system/test_prospector.py @@ -276,14 +276,9 @@ def test_shutdown_no_prospectors(self): filebeat = self.start_beat() - # wait for first "Start next scan" log message self.wait_until( lambda: self.log_contains( - "No prospectors defined"), - max_timeout=10) - - self.wait_until( - lambda: self.log_contains("No prospectors defined"), + "No modules or prospectors enabled"), max_timeout=10) filebeat.check_wait(exit_code=1) From a279b5d88480a8d5189be0e90fa4b2fbd58fa575 Mon Sep 17 00:00:00 2001 From: Vijay Samuel Date: Wed, 26 Apr 2017 02:20:14 -0700 Subject: [PATCH 21/40] Adding goimports support to make check and fmt (#4114) --- CHANGELOG.asciidoc | 1 + filebeat/processors/kubernetes/indexing_test.go | 3 ++- libbeat/processors/kubernetes/config.go | 3 ++- libbeat/scripts/Makefile | 9 ++++++--- .../ceph/cluster_disk/cluster_disk_integration_test.go | 3 ++- .../module/ceph/pool_disk/pool_disk_integration_test.go | 3 ++- metricbeat/module/docker/image/data.go | 5 +++-- metricbeat/module/dropwizard/collector/collector.go | 3 ++- metricbeat/module/dropwizard/collector/data.go | 2 +- metricbeat/module/golang/heap/heap.go | 3 ++- metricbeat/module/golang/util.go | 3 ++- packetbeat/protos/cassandra/internal/gocql/frame.go | 5 +++-- packetbeat/protos/nfs/config.go | 3 ++- 13 files changed, 30 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index f9d7f2e4505..5f5b51bc0c4 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -109,6 +109,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Add kubernetes processor {pull}3888[3888] - Add support for include_labels and include_annotations in kubernetes processor {pull}4043[4043] - Support new `index_patterns` field when loading templates for Elasticsearch >= 6.0 {pull}4056[4056] +- Adding goimports support to make check and fmt {pull}4114[4114] *Filebeat* diff --git a/filebeat/processors/kubernetes/indexing_test.go b/filebeat/processors/kubernetes/indexing_test.go index be5f08b6e26..5ba98056d59 100644 --- a/filebeat/processors/kubernetes/indexing_test.go +++ b/filebeat/processors/kubernetes/indexing_test.go @@ -2,9 +2,10 @@ package kubernetes import ( "fmt" + "testing" + "github.com/elastic/beats/libbeat/common" "github.com/stretchr/testify/assert" - "testing" ) func TestLogsPathMatcher(t *testing.T) { diff --git a/libbeat/processors/kubernetes/config.go b/libbeat/processors/kubernetes/config.go index 99f5a17ed2c..01fbc5a1bb7 100644 --- a/libbeat/processors/kubernetes/config.go +++ b/libbeat/processors/kubernetes/config.go @@ -1,8 +1,9 @@ package kubernetes import ( - "github.com/elastic/beats/libbeat/common" "time" + + "github.com/elastic/beats/libbeat/common" ) type kubeAnnotatorConfig struct { diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 03f122e14a7..3421aca2a70 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -30,6 +30,8 @@ COVERAGE_DIR=${BUILD_DIR}/coverage COVERAGE_TOOL=${BEAT_GOPATH}/bin/gotestcover COVERAGE_TOOL_REPO=github.com/elastic/beats/vendor/github.com/pierrre/gotestcover TESTIFY_TOOL_REPO=github.com/elastic/beats/vendor/github.com/stretchr/testify +GOIMPORTS=goimports +GOIMPORTS_REPO=golang.org/x/tools/cmd/goimports GOLINT=golint GOLINT_REPO=github.com/golang/lint/golint REVIEWDOG=reviewdog -conf ${ES_BEATS}/reviewdog.yml @@ -93,13 +95,14 @@ crosscompile: $(GOFILES) .PHONY: check check: python-env ## @build Checks project and source code if everything is according to standard - @gofmt -l ${GOFILES_NOVENDOR} | (! grep . -q) || (echo "Code differs from gofmt's style" && false) go vet ${GOPACKAGES} + @go get $(GOIMPORTS_REPO) + @goimports -l ${GOFILES_NOVENDOR} | (! grep . -q) || (echo "Code differs from goimports' style" && false) ${FIND} -name *.py -exec autopep8 -d --max-line-length 120 {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false) .PHONY: fmt -fmt: python-env ## @build Runs `gofmt -s -l -w` and `autopep8`on the project's source code, modifying any files that do not match its style. - gofmt -s -l -w ${GOFILES_NOVENDOR} +fmt: python-env ## @build Runs `goimports -l -w` and `autopep8`on the project's source code, modifying any files that do not match its style. + goimports -l -w ${GOFILES_NOVENDOR} ${FIND} -name *.py -exec autopep8 --in-place --max-line-length 120 {} \; .PHONY: lint diff --git a/metricbeat/module/ceph/cluster_disk/cluster_disk_integration_test.go b/metricbeat/module/ceph/cluster_disk/cluster_disk_integration_test.go index 8cfd9ccf4e0..3937c7e9b98 100644 --- a/metricbeat/module/ceph/cluster_disk/cluster_disk_integration_test.go +++ b/metricbeat/module/ceph/cluster_disk/cluster_disk_integration_test.go @@ -2,9 +2,10 @@ package cluster_disk import ( "fmt" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" "os" "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) func TestData(t *testing.T) { diff --git a/metricbeat/module/ceph/pool_disk/pool_disk_integration_test.go b/metricbeat/module/ceph/pool_disk/pool_disk_integration_test.go index b7fead7b893..3202bf3a994 100644 --- a/metricbeat/module/ceph/pool_disk/pool_disk_integration_test.go +++ b/metricbeat/module/ceph/pool_disk/pool_disk_integration_test.go @@ -2,9 +2,10 @@ package pool_disk import ( "fmt" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" "os" "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) func TestData(t *testing.T) { diff --git a/metricbeat/module/docker/image/data.go b/metricbeat/module/docker/image/data.go index 877a3d7db63..0dc24770148 100644 --- a/metricbeat/module/docker/image/data.go +++ b/metricbeat/module/docker/image/data.go @@ -1,11 +1,12 @@ package image import ( - "github.com/elastic/beats/libbeat/common" + "time" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/module/docker" + dc "github.com/fsouza/go-dockerclient" - "time" ) func eventsMapping(imagesList []dc.APIImages) []common.MapStr { diff --git a/metricbeat/module/dropwizard/collector/collector.go b/metricbeat/module/dropwizard/collector/collector.go index a4b35e03bde..e98bf67a8d7 100644 --- a/metricbeat/module/dropwizard/collector/collector.go +++ b/metricbeat/module/dropwizard/collector/collector.go @@ -2,12 +2,13 @@ package collector import ( "encoding/json" + "strings" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" - "strings" ) const ( diff --git a/metricbeat/module/dropwizard/collector/data.go b/metricbeat/module/dropwizard/collector/data.go index 9c795d453d5..bf54592dc69 100644 --- a/metricbeat/module/dropwizard/collector/data.go +++ b/metricbeat/module/dropwizard/collector/data.go @@ -1,9 +1,9 @@ package collector import ( + "encoding/json" "strings" - "encoding/json" "github.com/elastic/beats/libbeat/common" ) diff --git a/metricbeat/module/golang/heap/heap.go b/metricbeat/module/golang/heap/heap.go index 573b9f28077..140fa4f4b71 100644 --- a/metricbeat/module/golang/heap/heap.go +++ b/metricbeat/module/golang/heap/heap.go @@ -2,13 +2,14 @@ package heap import ( "encoding/json" + "runtime" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" "github.com/elastic/beats/metricbeat/module/golang" - "runtime" ) const ( diff --git a/metricbeat/module/golang/util.go b/metricbeat/module/golang/util.go index 0518b77d0af..0cf09d5e439 100644 --- a/metricbeat/module/golang/util.go +++ b/metricbeat/module/golang/util.go @@ -2,8 +2,9 @@ package golang import ( "bytes" - "github.com/elastic/beats/libbeat/logp" "strings" + + "github.com/elastic/beats/libbeat/logp" ) /** diff --git a/packetbeat/protos/cassandra/internal/gocql/frame.go b/packetbeat/protos/cassandra/internal/gocql/frame.go index 3115d78fa25..542106c2799 100644 --- a/packetbeat/protos/cassandra/internal/gocql/frame.go +++ b/packetbeat/protos/cassandra/internal/gocql/frame.go @@ -7,10 +7,11 @@ package cassandra import ( "errors" "fmt" - "github.com/elastic/beats/libbeat/common/streambuf" - "github.com/elastic/beats/libbeat/logp" "runtime" "sync" + + "github.com/elastic/beats/libbeat/common/streambuf" + "github.com/elastic/beats/libbeat/logp" ) var ( diff --git a/packetbeat/protos/nfs/config.go b/packetbeat/protos/nfs/config.go index eabd31e4c90..b09b1b35616 100644 --- a/packetbeat/protos/nfs/config.go +++ b/packetbeat/protos/nfs/config.go @@ -1,8 +1,9 @@ package nfs import ( - "github.com/elastic/beats/packetbeat/config" "time" + + "github.com/elastic/beats/packetbeat/config" ) type rpcConfig struct { From 7a8bc7040681a6512de56fd69bc159f4fe36ea4f Mon Sep 17 00:00:00 2001 From: Rockybean Date: Wed, 26 Apr 2017 19:49:44 +0800 Subject: [PATCH 22/40] Add rsbeat to communitybeats doc (#4108) --- libbeat/docs/communitybeats.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/docs/communitybeats.asciidoc b/libbeat/docs/communitybeats.asciidoc index 2e7d50264ee..749dc3ea3bd 100644 --- a/libbeat/docs/communitybeats.asciidoc +++ b/libbeat/docs/communitybeats.asciidoc @@ -52,6 +52,7 @@ https://github.com/carlpett/prombeat[prombeat]:: Indexes https://prometheus.io[P https://github.com/voigt/redditbeat[redditbeat]:: Collects new Reddit Submissions of one or multiple Subreddits. https://github.com/chrsblck/redisbeat[redisbeat]:: Used for Redis monitoring. https://github.com/consulthys/retsbeat[retsbeat]:: Collects counts of http://www.reso.org[RETS] resource/class records from https://en.wikipedia.org/wiki/Multiple_listing_service[Multiple Listing Service] (MLS) servers. +https://github.com/yourdream/rsbeat[rsbeat]:: Ships redis slow logs to elasticsearch and anlyze by Kibana. https://github.com/martinhoefling/saltbeat[saltbeat]:: Reads events from salt master event bus. https://github.com/consulthys/springbeat[springbeat]:: Collects health and metrics data from Spring Boot applications running with the actuator module. https://github.com/buehler/go-elastic-twitterbeat[twitterbeat]:: Reads tweets for specified screen names. @@ -62,7 +63,6 @@ network intrusion detection software and indexes the records in Elasticsearch. https://github.com/mrkschan/uwsgibeat[uwsgibeat]:: Reads stats from uWSGI. https://github.com/eskibars/wmibeat[wmibeat]:: Uses WMI to grab your favorite, configurable Windows metrics. - Have you created a Beat that's not listed? If so, add the name and description of your Beat to the source document for https://github.com/elastic/beats/blob/master/libbeat/docs/communitybeats.asciidoc[Community Beats] and https://help.github.com/articles/using-pull-requests[open a pull request] in the https://github.com/elastic/beats[Beats GitHub repository] to get your change merged. When you're ready, go ahead and https://discuss.elastic.co/c/annoucements[announce] your new Beat in the Elastic discussion forum. From 7d15bf3599af63b87878ffe4070c5c30c519c498 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Thu, 27 Apr 2017 00:26:36 +0200 Subject: [PATCH 23/40] Fix link to the MacOSX SDK tarball (#4120) The original download was temporarily down and then it came back up with a different sha1. Switching to what seems to be a link closer to the source. This will require backporting in all branches that need to be built. --- dev-tools/packer/docker/xgo-image/base/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dev-tools/packer/docker/xgo-image/base/Dockerfile b/dev-tools/packer/docker/xgo-image/base/Dockerfile index d827126add3..ceadbb30027 100644 --- a/dev-tools/packer/docker/xgo-image/base/Dockerfile +++ b/dev-tools/packer/docker/xgo-image/base/Dockerfile @@ -29,14 +29,13 @@ RUN \ binutils-multiarch rsync \ --no-install-recommends -# Configure the container for OSX cross compilation # Configure the container for OSX cross compilation ENV OSX_SDK MacOSX10.11.sdk ENV OSX_NDK_X86 /usr/local/osx-ndk-x86 RUN \ - OSX_SDK_PATH=https://s3.dockerproject.org/darwin/v2/$OSX_SDK.tar.xz && \ - $FETCH $OSX_SDK_PATH dd228a335194e3392f1904ce49aff1b1da26ca62 && \ + OSX_SDK_PATH=https://github.com/phracker/MacOSX-SDKs/releases/download/MacOSX10.11.sdk/MacOSX10.11.sdk.tar.xz && \ + $FETCH $OSX_SDK_PATH f3430e3d923644e66c0c13f7a48754e7b6aa2e3f && \ \ git clone https://github.com/tpoechtrager/osxcross.git && \ mv `basename $OSX_SDK_PATH` /osxcross/tarballs/ && \ From 074add2c8a39973c4af3d521432733a5960b2ff7 Mon Sep 17 00:00:00 2001 From: Vijay Samuel Date: Wed, 26 Apr 2017 23:47:58 -0700 Subject: [PATCH 24/40] Fixing nil pointer on prometheus collector when http response is nil (#4119) --- CHANGELOG.asciidoc | 1 + metricbeat/module/prometheus/collector/collector.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 5f5b51bc0c4..1edfbb55bf9 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -147,6 +147,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Add kubelet module {pull}3916[3916] - Add dropwizard module {pull}4022[4022] - Adding query APIs for metricsets and modules from metricbeat registry {pull}4102[4102] +- Fixing nil pointer on prometheus collector when http response is nil {pull}4119[4119] *Packetbeat* - Add `fields` and `fields_under_root` to packetbeat protocols configurations. {pull}3518[3518] diff --git a/metricbeat/module/prometheus/collector/collector.go b/metricbeat/module/prometheus/collector/collector.go index 5cb83d8ebed..77915b3caae 100644 --- a/metricbeat/module/prometheus/collector/collector.go +++ b/metricbeat/module/prometheus/collector/collector.go @@ -57,10 +57,10 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch() ([]common.MapStr, error) { resp, err := m.http.FetchResponse() - defer resp.Body.Close() if err != nil { return nil, err } + defer resp.Body.Close() families, err := prometheus.GetMetricFamiliesFromResponse(resp) if err != nil { From bee77eecb971f4bba7db5a4cc955b9a3ff343a14 Mon Sep 17 00:00:00 2001 From: yeer Date: Thu, 27 Apr 2017 16:21:13 +0800 Subject: [PATCH 25/40] Remove duplicate code in glob_watcher_test.go (#4117) --- libbeat/cfgfile/glob_watcher_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/libbeat/cfgfile/glob_watcher_test.go b/libbeat/cfgfile/glob_watcher_test.go index 65da24287f4..43e6c74431d 100644 --- a/libbeat/cfgfile/glob_watcher_test.go +++ b/libbeat/cfgfile/glob_watcher_test.go @@ -36,7 +36,6 @@ func TestGlobWatcher(t *testing.T) { assert.NoError(t, err) assert.True(t, changed) - files, changed, err = gcd.Scan() files, changed, err = gcd.Scan() assert.Equal(t, 2, len(files)) assert.NoError(t, err) From 24aed8cc5aed185071a0aeb61ab3b12b2882b508 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Thu, 27 Apr 2017 15:22:02 +0200 Subject: [PATCH 26/40] Refactor harvester to send events directly to the spooler (#4070) * Refactor harvester to send events directly to the spooler Previously all events were sent through the prospector to update the send and process the data. The state update in the prospector is now done through a mutex and processing happens directly in the harvester. This is a major change in the architecture on how filebeat works. It should simplify the shutting down of harvester and prospectors as less channels are involved in the communication. Some of the configs were moved around between the prospector and harvester package, but in the long run the idea is to merge the two into one. Further changes: * Add read/write lock to state handling to make it more efficent * Did some work on the communication channels. It's still a bit messy and needs cleanup in a follow up PR. * Processing happens now in the harvester directly. This should be part of the publisher in the future. * Prospector now only communicates with the spooler to update state, never events. * Move initial state update logic to harvester to simplify code This PR should not change any behavior in filebeat. * add stdin hack * Introduce HasState for source. This is a hack to circumvent https://github.com/elastic/beats/pull/3376#issuecomment-297674262 --- filebeat/beater/filebeat.go | 3 +- filebeat/channel/outlet.go | 13 ++- filebeat/harvester/config.go | 43 +++++---- filebeat/harvester/harvester.go | 62 ++++++++++++- filebeat/harvester/log.go | 28 ++++-- filebeat/harvester/source/file.go | 1 + filebeat/harvester/source/pipe.go | 1 + filebeat/harvester/source/source.go | 1 + filebeat/input/file/state.go | 15 ++-- filebeat/prospector/config.go | 35 +++----- filebeat/prospector/prospector.go | 89 ++++--------------- filebeat/prospector/prospector_log.go | 9 +- .../prospector/prospector_log_other_test.go | 5 +- filebeat/prospector/registry.go | 6 ++ 14 files changed, 168 insertions(+), 143 deletions(-) diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index 48d422c2dc7..9f82276f55c 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -11,6 +11,7 @@ import ( "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/outputs/elasticsearch" + "github.com/elastic/beats/filebeat/channel" cfg "github.com/elastic/beats/filebeat/config" "github.com/elastic/beats/filebeat/crawler" "github.com/elastic/beats/filebeat/fileset" @@ -148,7 +149,7 @@ func (fb *Filebeat) Run(b *beat.Beat) error { return err } - crawler, err := crawler.New(newSpoolerOutlet(fb.done, spooler, wgEvents), config.Prospectors, fb.done, *once) + crawler, err := crawler.New(channel.NewOutlet(fb.done, spooler.Channel, wgEvents), config.Prospectors, fb.done, *once) if err != nil { logp.Err("Could not init crawler: %v", err) return err diff --git a/filebeat/channel/outlet.go b/filebeat/channel/outlet.go index 4695b302899..aee6524671f 100644 --- a/filebeat/channel/outlet.go +++ b/filebeat/channel/outlet.go @@ -5,6 +5,7 @@ import ( "sync/atomic" "github.com/elastic/beats/filebeat/input" + "github.com/elastic/beats/filebeat/prospector" ) // Outlet struct is used to be passed to an object which needs an outlet @@ -19,13 +20,13 @@ type Outlet struct { wg *sync.WaitGroup // Use for counting active events done <-chan struct{} signal <-chan struct{} - channel chan *input.Event + channel chan *input.Data isOpen int32 // atomic indicator } func NewOutlet( done <-chan struct{}, - c chan *input.Event, + c chan *input.Data, wg *sync.WaitGroup, ) *Outlet { return &Outlet{ @@ -42,7 +43,7 @@ func (o *Outlet) SetSignal(signal <-chan struct{}) { o.signal = signal } -func (o *Outlet) OnEvent(event *input.Event) bool { +func (o *Outlet) OnEvent(event *input.Data) bool { open := atomic.LoadInt32(&o.isOpen) == 1 if !open { return false @@ -67,7 +68,7 @@ func (o *Outlet) OnEvent(event *input.Event) bool { // OnEventSignal can be stopped by the signal that is set with SetSignal // This does not close the outlet. Only OnEvent does close the outlet. // If OnEventSignal is used, it must be ensured that only one producer is used. -func (o *Outlet) OnEventSignal(event *input.Event) bool { +func (o *Outlet) OnEventSignal(event *input.Data) bool { open := atomic.LoadInt32(&o.isOpen) == 1 if !open { return false @@ -88,3 +89,7 @@ func (o *Outlet) OnEventSignal(event *input.Event) bool { return true } } + +func (o *Outlet) Copy() prospector.Outlet { + return NewOutlet(o.done, o.channel, o.wg) +} diff --git a/filebeat/harvester/config.go b/filebeat/harvester/config.go index 60b4b5de2f7..05b653f403a 100644 --- a/filebeat/harvester/config.go +++ b/filebeat/harvester/config.go @@ -8,7 +8,9 @@ import ( "github.com/elastic/beats/filebeat/harvester/reader" "github.com/dustin/go-humanize" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/match" + "github.com/elastic/beats/libbeat/processors" ) var ( @@ -24,26 +26,35 @@ var ( CloseRenamed: false, CloseEOF: false, CloseTimeout: 0, + DocumentType: "log", + CleanInactive: 0, } ) type harvesterConfig struct { - BufferSize int `config:"harvester_buffer_size"` - Encoding string `config:"encoding"` - InputType string `config:"input_type"` - Backoff time.Duration `config:"backoff" validate:"min=0,nonzero"` - BackoffFactor int `config:"backoff_factor" validate:"min=1"` - MaxBackoff time.Duration `config:"max_backoff" validate:"min=0,nonzero"` - CloseInactive time.Duration `config:"close_inactive"` - CloseRemoved bool `config:"close_removed"` - CloseRenamed bool `config:"close_renamed"` - CloseEOF bool `config:"close_eof"` - CloseTimeout time.Duration `config:"close_timeout" validate:"min=0"` - ExcludeLines []match.Matcher `config:"exclude_lines"` - IncludeLines []match.Matcher `config:"include_lines"` - MaxBytes int `config:"max_bytes" validate:"min=0,nonzero"` - Multiline *reader.MultilineConfig `config:"multiline"` - JSON *reader.JSONConfig `config:"json"` + common.EventMetadata `config:",inline"` // Fields and tags to add to events. + BufferSize int `config:"harvester_buffer_size"` + Encoding string `config:"encoding"` + InputType string `config:"input_type"` + Backoff time.Duration `config:"backoff" validate:"min=0,nonzero"` + BackoffFactor int `config:"backoff_factor" validate:"min=1"` + MaxBackoff time.Duration `config:"max_backoff" validate:"min=0,nonzero"` + CloseInactive time.Duration `config:"close_inactive"` + CloseRemoved bool `config:"close_removed"` + CloseRenamed bool `config:"close_renamed"` + CloseEOF bool `config:"close_eof"` + CloseTimeout time.Duration `config:"close_timeout" validate:"min=0"` + ExcludeLines []match.Matcher `config:"exclude_lines"` + IncludeLines []match.Matcher `config:"include_lines"` + MaxBytes int `config:"max_bytes" validate:"min=0,nonzero"` + Multiline *reader.MultilineConfig `config:"multiline"` + JSON *reader.JSONConfig `config:"json"` + DocumentType string `config:"document_type"` + CleanInactive time.Duration `config:"clean_inactive" validate:"min=0"` + Pipeline string `config:"pipeline"` + Module string `config:"_module_name"` // hidden option to set the module name + Fileset string `config:"_fileset_name"` // hidden option to set the fileset name + Processors processors.PluginConfig `config:"processors"` } func (config *harvesterConfig) Validate() error { diff --git a/filebeat/harvester/harvester.go b/filebeat/harvester/harvester.go index 731eb17ace0..fcc64ec1538 100644 --- a/filebeat/harvester/harvester.go +++ b/filebeat/harvester/harvester.go @@ -18,13 +18,14 @@ import ( "github.com/satori/go.uuid" - "github.com/elastic/beats/filebeat/channel" "github.com/elastic/beats/filebeat/config" "github.com/elastic/beats/filebeat/harvester/encoding" "github.com/elastic/beats/filebeat/harvester/source" "github.com/elastic/beats/filebeat/input" "github.com/elastic/beats/filebeat/input/file" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/processors" ) var ( @@ -35,9 +36,16 @@ var ( ErrClosed = errors.New("reader closed") ) +type Outlet interface { + SetSignal(signal <-chan struct{}) + OnEventSignal(event *input.Data) bool + OnEvent(event *input.Data) bool +} + type Harvester struct { config harvesterConfig state file.State + states *file.States prospectorChan chan *input.Event file source.FileSource /* the file being watched */ fileReader *LogFile @@ -46,19 +54,22 @@ type Harvester struct { done chan struct{} stopOnce sync.Once stopWg *sync.WaitGroup - outlet *channel.Outlet + outlet Outlet ID uuid.UUID + processors *processors.Processors } func NewHarvester( cfg *common.Config, state file.State, - outlet *channel.Outlet, + states *file.States, + outlet Outlet, ) (*Harvester, error) { h := &Harvester{ config: defaultConfig, state: state, + states: states, done: make(chan struct{}), stopWg: &sync.WaitGroup{}, outlet: outlet, @@ -75,6 +86,18 @@ func NewHarvester( } h.encodingFactory = encodingFactory + f, err := processors.New(h.config.Processors) + if err != nil { + return nil, err + } + + h.processors = f + + // Add ttl if clean_inactive is set + if h.config.CleanInactive > 0 { + h.state.TTL = h.config.CleanInactive + } + // Add outlet signal so harvester can also stop itself h.outlet.SetSignal(h.done) @@ -93,3 +116,36 @@ func (h *Harvester) open() error { return fmt.Errorf("Invalid input type") } } + +// updateState updates the prospector state and forwards the event to the spooler +// All state updates done by the prospector itself are synchronous to make sure not states are overwritten +func (h *Harvester) forwardEvent(event *input.Event) error { + + // Add additional prospector meta data to the event + event.EventMetadata = h.config.EventMetadata + event.InputType = h.config.InputType + event.DocumentType = h.config.DocumentType + event.JSONConfig = h.config.JSON + event.Pipeline = h.config.Pipeline + event.Module = h.config.Module + event.Fileset = h.config.Fileset + + eventHolder := event.GetData() + //run the filters before sending to spooler + if event.Bytes > 0 { + eventHolder.Event = h.processors.Run(eventHolder.Event) + } + + if eventHolder.Event == nil { + eventHolder.Metadata.Bytes = 0 + } + + ok := h.outlet.OnEventSignal(&eventHolder) + + if !ok { + logp.Info("Prospector outlet closed") + return errors.New("prospector outlet closed") + } + + return nil +} diff --git a/filebeat/harvester/log.go b/filebeat/harvester/log.go index f9e92584276..d4bfe145498 100644 --- a/filebeat/harvester/log.go +++ b/filebeat/harvester/log.go @@ -3,14 +3,11 @@ package harvester import ( "bytes" "errors" + "fmt" "io" "os" "time" - "golang.org/x/text/transform" - - "fmt" - "github.com/elastic/beats/filebeat/config" "github.com/elastic/beats/filebeat/harvester/reader" "github.com/elastic/beats/filebeat/harvester/source" @@ -18,6 +15,8 @@ import ( "github.com/elastic/beats/filebeat/input/file" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/monitoring" + + "golang.org/x/text/transform" ) var ( @@ -171,7 +170,11 @@ func (h *Harvester) Stop() { // sendEvent sends event to the spooler channel // Return false if event was not sent func (h *Harvester) sendEvent(event *input.Event) bool { - return h.outlet.OnEventSignal(event) + if h.file.HasState() { + h.states.Update(event.State) + } + err := h.forwardEvent(event) + return err == nil } // sendStateUpdate send an empty event with the current state to update the registry @@ -179,10 +182,19 @@ func (h *Harvester) sendEvent(event *input.Event) bool { // case the output is blocked the harvester will stay open to make sure no new harvester // is started. As soon as the output becomes available again, the finished state is written // and processing can continue. -func (h *Harvester) sendStateUpdate() { +func (h *Harvester) SendStateUpdate() { + + if !h.file.HasState() { + return + } + logp.Debug("harvester", "Update state: %s, offset: %v", h.state.Source, h.state.Offset) + event := input.NewEvent(h.state) - h.outlet.OnEvent(event) + h.states.Update(event.State) + + data := event.GetData() + h.outlet.OnEvent(&data) } // shouldExportLine decides if the line is exported or not based on @@ -317,7 +329,7 @@ func (h *Harvester) close() { // On completion, push offset so we can continue where we left off if we relaunch on the same file // Only send offset if file object was created successfully - h.sendStateUpdate() + h.SendStateUpdate() } else { logp.Warn("Stopping harvester, NOT closing file as file info not available: %s", h.state.Source) } diff --git a/filebeat/harvester/source/file.go b/filebeat/harvester/source/file.go index d55fe2cfb72..752324fc7d0 100644 --- a/filebeat/harvester/source/file.go +++ b/filebeat/harvester/source/file.go @@ -7,3 +7,4 @@ type File struct { } func (File) Continuable() bool { return true } +func (File) HasState() bool { return true } diff --git a/filebeat/harvester/source/pipe.go b/filebeat/harvester/source/pipe.go index 3b81b5a54ea..413233e081c 100644 --- a/filebeat/harvester/source/pipe.go +++ b/filebeat/harvester/source/pipe.go @@ -13,3 +13,4 @@ func (p Pipe) Close() error { return p.File.Close() } func (p Pipe) Name() string { return p.File.Name() } func (p Pipe) Stat() (os.FileInfo, error) { return p.File.Stat() } func (p Pipe) Continuable() bool { return false } +func (p Pipe) HasState() bool { return false } diff --git a/filebeat/harvester/source/source.go b/filebeat/harvester/source/source.go index c913908d868..733f113f187 100644 --- a/filebeat/harvester/source/source.go +++ b/filebeat/harvester/source/source.go @@ -14,4 +14,5 @@ type FileSource interface { LogSource Stat() (os.FileInfo, error) Continuable() bool // can we continue processing after EOF? + HasState() bool // does this source have a state? } diff --git a/filebeat/input/file/state.go b/filebeat/input/file/state.go index d227b25175d..7626678eb93 100644 --- a/filebeat/input/file/state.go +++ b/filebeat/input/file/state.go @@ -39,7 +39,7 @@ func (s *State) IsEmpty() bool { // States handles list of FileState type States struct { states []State - sync.Mutex + sync.RWMutex } func NewStates() *States { @@ -66,9 +66,8 @@ func (s *States) Update(newState State) { } func (s *States) FindPrevious(newState State) State { - // TODO: This currently blocks writing updates every time state is fetched. Should be improved for performance - s.Lock() - defer s.Unlock() + s.RLock() + defer s.RUnlock() _, state := s.findPrevious(newState) return state } @@ -122,16 +121,16 @@ func (s *States) Cleanup() int { // Count returns number of states func (s *States) Count() int { - s.Lock() - defer s.Unlock() + s.RLock() + defer s.RUnlock() return len(s.states) } // Returns a copy of the file states func (s *States) GetStates() []State { - s.Lock() - defer s.Unlock() + s.RLock() + defer s.RUnlock() newStates := make([]State, len(s.states)) copy(newStates, s.states) diff --git a/filebeat/prospector/config.go b/filebeat/prospector/config.go index 1cef49f67b4..190a5a56951 100644 --- a/filebeat/prospector/config.go +++ b/filebeat/prospector/config.go @@ -5,16 +5,12 @@ import ( "time" cfg "github.com/elastic/beats/filebeat/config" - "github.com/elastic/beats/filebeat/harvester/reader" - "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/match" - "github.com/elastic/beats/libbeat/processors" ) var ( defaultConfig = prospectorConfig{ Enabled: true, - DocumentType: "log", IgnoreOlder: 0, ScanFrequency: 10 * time.Second, InputType: cfg.DefaultInputType, @@ -27,25 +23,18 @@ var ( ) type prospectorConfig struct { - common.EventMetadata `config:",inline"` // Fields and tags to add to events. - Enabled bool `config:"enabled"` - DocumentType string `config:"document_type"` - ExcludeFiles []match.Matcher `config:"exclude_files"` - IgnoreOlder time.Duration `config:"ignore_older"` - Paths []string `config:"paths"` - ScanFrequency time.Duration `config:"scan_frequency" validate:"min=0,nonzero"` - InputType string `config:"input_type"` - CleanInactive time.Duration `config:"clean_inactive" validate:"min=0"` - CleanRemoved bool `config:"clean_removed"` - HarvesterLimit uint64 `config:"harvester_limit" validate:"min=0"` - Symlinks bool `config:"symlinks"` - TailFiles bool `config:"tail_files"` - JSON *reader.JSONConfig `config:"json"` - Pipeline string `config:"pipeline"` - Module string `config:"_module_name"` // hidden option to set the module name - Fileset string `config:"_fileset_name"` // hidden option to set the fileset name - Processors processors.PluginConfig `config:"processors"` - recursiveGlob bool `config:"recursive_glob.enabled"` + Enabled bool `config:"enabled"` + ExcludeFiles []match.Matcher `config:"exclude_files"` + IgnoreOlder time.Duration `config:"ignore_older"` + Paths []string `config:"paths"` + ScanFrequency time.Duration `config:"scan_frequency" validate:"min=0,nonzero"` + InputType string `config:"input_type"` + CleanInactive time.Duration `config:"clean_inactive" validate:"min=0"` + CleanRemoved bool `config:"clean_removed"` + HarvesterLimit uint64 `config:"harvester_limit" validate:"min=0"` + Symlinks bool `config:"symlinks"` + TailFiles bool `config:"tail_files"` + recursiveGlob bool `config:"recursive_glob.enabled"` } func (config *prospectorConfig) Validate() error { diff --git a/filebeat/prospector/prospector.go b/filebeat/prospector/prospector.go index 604c5cdddd1..3a9c27273af 100644 --- a/filebeat/prospector/prospector.go +++ b/filebeat/prospector/prospector.go @@ -8,7 +8,6 @@ import ( "github.com/mitchellh/hashstructure" - "github.com/elastic/beats/filebeat/channel" cfg "github.com/elastic/beats/filebeat/config" "github.com/elastic/beats/filebeat/harvester" "github.com/elastic/beats/filebeat/input" @@ -16,7 +15,6 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/monitoring" - "github.com/elastic/beats/libbeat/processors" ) var ( @@ -35,13 +33,11 @@ type Prospector struct { runWg *sync.WaitGroup states *file.States wg *sync.WaitGroup - channelWg *sync.WaitGroup // Separate waitgroup for channels as not stopped on completion id uint64 Once bool registry *harvesterRegistry beatDone chan struct{} eventCounter *sync.WaitGroup - processors *processors.Processors } // Prospectorer is the interface common to all prospectors @@ -52,7 +48,10 @@ type Prospectorer interface { // Outlet is the outlet for a prospector type Outlet interface { + SetSignal(signal <-chan struct{}) + OnEventSignal(event *input.Data) bool OnEvent(event *input.Data) bool + Copy() Outlet } // NewProspector instantiates a new prospector @@ -67,7 +66,6 @@ func NewProspector(cfg *common.Config, outlet Outlet, beatDone chan struct{}) (* runDone: make(chan struct{}), runWg: &sync.WaitGroup{}, states: &file.States{}, - channelWg: &sync.WaitGroup{}, Once: false, registry: newHarvesterRegistry(), beatDone: beatDone, @@ -86,13 +84,6 @@ func NewProspector(cfg *common.Config, outlet Outlet, beatDone chan struct{}) (* return nil, err } - f, err := processors.New(prospector.config.Processors) - if err != nil { - return nil, err - } - - prospector.processors = f - logp.Debug("prospector", "File Configs: %v", prospector.config.Paths) return prospector, nil @@ -137,28 +128,6 @@ func (p *Prospector) Start() { p.wg.Add(1) logp.Info("Starting prospector of type: %v; id: %v ", p.config.InputType, p.ID()) - // Open channel to receive events from harvester and forward them to spooler - // Here potential filtering can happen - p.channelWg.Add(1) - go func() { - defer p.channelWg.Done() - for { - select { - case <-p.channelDone: - logp.Info("Prospector channel stopped") - return - case <-p.beatDone: - logp.Info("Prospector channel stopped because beat is stopping.") - return - case event := <-p.harvesterChan: - // No stopping on error, because on error it is expected that beatDone is closed - // in the next run. If not, this will further drain the channel. - p.updateState(event) - p.eventCounter.Done() - } - } - }() - if p.Once { // Makes sure prospectors can complete first scan before stopped defer p.runWg.Wait() @@ -207,31 +176,19 @@ func (p *Prospector) ID() uint64 { // updateState updates the prospector state and forwards the event to the spooler // All state updates done by the prospector itself are synchronous to make sure not states are overwritten -func (p *Prospector) updateState(event *input.Event) error { +func (p *Prospector) updateState(state file.State) error { // Add ttl if cleanOlder is enabled and TTL is not already 0 - if p.config.CleanInactive > 0 && event.State.TTL != 0 { - event.State.TTL = p.config.CleanInactive + if p.config.CleanInactive > 0 && state.TTL != 0 { + state.TTL = p.config.CleanInactive } - // Add additional prospector meta data to the event - event.EventMetadata = p.config.EventMetadata - event.InputType = p.config.InputType - event.DocumentType = p.config.DocumentType - event.JSONConfig = p.config.JSON - event.Pipeline = p.config.Pipeline - event.Module = p.config.Module - event.Fileset = p.config.Fileset - - eventHolder := event.GetData() - //run the filters before sending to spooler - if event.Bytes > 0 { - eventHolder.Event = p.processors.Run(eventHolder.Event) - } + // Update first internal state + p.states.Update(state) - if eventHolder.Event == nil { - eventHolder.Metadata.Bytes = 0 - } + eventHolder := input.NewEvent(state).GetData() + // Set to 0 as these are state updates only + eventHolder.Metadata.Bytes = 0 ok := p.outlet.OnEvent(&eventHolder) @@ -240,7 +197,6 @@ func (p *Prospector) updateState(event *input.Event) error { return errors.New("prospector outlet closed") } - p.states.Update(event.State) return nil } @@ -297,17 +253,17 @@ func (p *Prospector) waitEvents() { close(p.channelDone) case <-p.beatDone: } - // Waits until channel go-routine properly stopped - p.channelWg.Wait() } // createHarvester creates a new harvester instance from the given state func (p *Prospector) createHarvester(state file.State) (*harvester.Harvester, error) { - outlet := channel.NewOutlet(p.beatDone, p.harvesterChan, p.eventCounter) + // Each harvester gets its own copy of the outlet + outlet := p.outlet.Copy() h, err := harvester.NewHarvester( p.cfg, state, + p.states, outlet, ) @@ -323,9 +279,9 @@ func (p *Prospector) startHarvester(state file.State, offset int64) error { return fmt.Errorf("Harvester limit reached") } - state.Offset = offset // Set state to "not" finished to indicate that a harvester is running state.Finished = false + state.Offset = offset // Create harvester with state h, err := p.createHarvester(state) @@ -333,23 +289,8 @@ func (p *Prospector) startHarvester(state file.State, offset int64) error { return err } - // State is directly updated and not through channel to make state update synchronous - err = p.updateState(input.NewEvent(state)) - if err != nil { - return err - } - reader, err := h.Setup() if err != nil { - // Set state to finished True again in case of setup failure to make sure - // file can be picked up again by a future harvester - state.Finished = true - - updateErr := p.updateState(input.NewEvent(state)) - // This should only happen in the case that filebeat is stopped - if updateErr != nil { - logp.Err("Error updating state: %v", updateErr) - } return fmt.Errorf("Error setting up harvester: %s", err) } diff --git a/filebeat/prospector/prospector_log.go b/filebeat/prospector/prospector_log.go index 01ad464263d..b471fb4ea1a 100644 --- a/filebeat/prospector/prospector_log.go +++ b/filebeat/prospector/prospector_log.go @@ -7,7 +7,6 @@ import ( "time" "github.com/elastic/beats/filebeat/harvester" - "github.com/elastic/beats/filebeat/input" "github.com/elastic/beats/filebeat/input/file" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/monitoring" @@ -60,7 +59,7 @@ func (l *Log) LoadStates(states []file.State) error { } // Update prospector states and send new states to registry - err := l.Prospector.updateState(input.NewEvent(state)) + err := l.Prospector.updateState(state) if err != nil { logp.Err("Problem putting initial state: %+v", err) return err @@ -131,7 +130,7 @@ func (l *Log) removeState(state file.State) { } state.TTL = 0 - err := l.Prospector.updateState(input.NewEvent(state)) + err := l.Prospector.updateState(state) if err != nil { logp.Err("File cleanup state update error: %s", err) } @@ -323,7 +322,7 @@ func (l *Log) harvestExistingFile(newState file.State, oldState file.State) { logp.Debug("prospector", "Updating state for renamed file: %s -> %s, Current offset: %v", oldState.Source, newState.Source, oldState.Offset) // Update state because of file rotation oldState.Source = newState.Source - err := l.Prospector.updateState(input.NewEvent(oldState)) + err := l.Prospector.updateState(oldState) if err != nil { logp.Err("File rotation state update error: %s", err) } @@ -367,7 +366,7 @@ func (l *Log) handleIgnoreOlder(lastState, newState file.State) error { // Write state for ignore_older file as none exists yet newState.Finished = true - err := l.Prospector.updateState(input.NewEvent(newState)) + err := l.Prospector.updateState(newState) if err != nil { return err } diff --git a/filebeat/prospector/prospector_log_other_test.go b/filebeat/prospector/prospector_log_other_test.go index da0afe50a5b..543399c561c 100644 --- a/filebeat/prospector/prospector_log_other_test.go +++ b/filebeat/prospector/prospector_log_other_test.go @@ -155,4 +155,7 @@ func TestInit(t *testing.T) { // TestOutlet is an empty outlet for testing type TestOutlet struct{} -func (o TestOutlet) OnEvent(event *input.Data) bool { return true } +func (o TestOutlet) OnEvent(event *input.Data) bool { return true } +func (o TestOutlet) OnEventSignal(event *input.Data) bool { return true } +func (o TestOutlet) SetSignal(signal <-chan struct{}) {} +func (o TestOutlet) Copy() Outlet { return o } diff --git a/filebeat/prospector/registry.go b/filebeat/prospector/registry.go index 86c8aa1161d..0f600f5d9f8 100644 --- a/filebeat/prospector/registry.go +++ b/filebeat/prospector/registry.go @@ -53,6 +53,12 @@ func (hr *harvesterRegistry) start(h *harvester.Harvester, r reader.Reader) { hr.wg.Add(1) hr.add(h) + + // Update state before staring harvester + // This makes sure the states is set to Finished: false + // This is synchronous state update as part of the scan + h.SendStateUpdate() + go func() { defer func() { hr.remove(h) From 1344e4f0cd2457342ac422f575143cd6c97623f8 Mon Sep 17 00:00:00 2001 From: yeer Date: Thu, 27 Apr 2017 21:27:42 +0800 Subject: [PATCH 27/40] Support Alibaba Cloud provider for add_cloud_metadata proccessor (#4111) * Support Alibaba Cloud provider for add_cloud_metadata processor * Add note for Alibaba Cloud provider in add_cloud_metadata processor documentation. --- CHANGELOG.asciidoc | 1 + libbeat/docs/processors-config.asciidoc | 19 +++++ .../add_cloud_metadata/add_cloud_metadata.go | 5 ++ .../provider_alibaba_cloud.go | 41 +++++++++++ .../provider_alibaba_cloud_test.go | 69 +++++++++++++++++++ .../provider_tencent_cloud.go | 3 +- 6 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go create mode 100644 libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 1edfbb55bf9..a077f61861b 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -294,6 +294,7 @@ https://github.com/elastic/beats/compare/v5.0.2...v5.1.1[View commits] - Add add_cloud_metadata processor for collecting cloud provider metadata. {pull}2728[2728] - Added decode_json_fields processor for decoding fields containing JSON strings. {pull}2605[2605] - Add Tencent Cloud provider for add_cloud_metadata processor. {pull}4023[4023] +- Add Alibaba Cloud provider for add_cloud_metadata processor. {pull}4111[4111] *Metricbeat* diff --git a/libbeat/docs/processors-config.asciidoc b/libbeat/docs/processors-config.asciidoc index b7d3cc79e9a..119cd809bb6 100644 --- a/libbeat/docs/processors-config.asciidoc +++ b/libbeat/docs/processors-config.asciidoc @@ -273,6 +273,7 @@ The following cloud providers are supported: - Digital Ocean - Google Compute Engine (GCE) - https://www.qcloud.com/?lang=en[Tencent Cloud] (QCloud) +- Alibaba Cloud (ECS) The simple configuration below enables the processor. @@ -359,6 +360,24 @@ _Tencent Cloud_ } ------------------------------------------------------------------------------- +_Alibaba Cloud_ + +This metadata is only available when VPC is selected as the network type of the ECS instance. + +[source,json] +------------------------------------------------------------------------------- +{ + "meta": { + "cloud": { + "availability_zone": "cn-shenzhen", + "instance_id": "i-wz9g2hqiikg0aliyun2b", + "provider": "ecs", + "region": "cn-shenzhen-a" + } + } +} +------------------------------------------------------------------------------- + [[add-locale]] === add_locale diff --git a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go index d1b86e150fe..630cb8502a8 100644 --- a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go +++ b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go @@ -254,12 +254,17 @@ func setupFetchers(c common.Config) ([]*metadataFetcher, error) { if err != nil { return fetchers, err } + ecsFetcher, err := newAlibabaCloudMetadataFetcher(c) + if err != nil { + return fetchers, err + } fetchers = []*metadataFetcher{ doFetcher, ec2Fetcher, gceFetcher, qcloudFetcher, + ecsFetcher, } return fetchers, nil } diff --git a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go new file mode 100644 index 00000000000..f981c403322 --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go @@ -0,0 +1,41 @@ +package add_cloud_metadata + +import "github.com/elastic/beats/libbeat/common" + +// Alibaba Cloud Metadata Service +// Document https://help.aliyun.com/knowledge_detail/49122.html +func newAlibabaCloudMetadataFetcher(c common.Config) (*metadataFetcher, error) { + ecsMetadataHost := "100.100.100.200" + ecsMetadataInstanceIDURI := "/latest/meta-data/instance-id" + ecsMetadataRegionURI := "/latest/meta-data/region-id" + ecsMetadataZoneURI := "/latest/meta-data/zone-id" + + ecsSchema := func(m map[string]interface{}) common.MapStr { + return common.MapStr(m) + } + + urls, err := getMetadataURLs(c, ecsMetadataHost, []string{ + ecsMetadataInstanceIDURI, + ecsMetadataRegionURI, + ecsMetadataZoneURI, + }) + if err != nil { + return nil, err + } + responseHandlers := map[string]responseHandler{ + urls[0]: func(all []byte, result *result) error { + result.metadata["instance_id"] = string(all) + return nil + }, + urls[1]: func(all []byte, result *result) error { + result.metadata["region"] = string(all) + return nil + }, + urls[2]: func(all []byte, result *result) error { + result.metadata["availability_zone"] = string(all) + return nil + }, + } + fetcher := &metadataFetcher{"ecs", nil, responseHandlers, ecsSchema} + return fetcher, nil +} diff --git a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go new file mode 100644 index 00000000000..5d4569aab4f --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go @@ -0,0 +1,69 @@ +package add_cloud_metadata + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/stretchr/testify/assert" +) + +func initECSTestServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/latest/meta-data/instance-id" { + w.Write([]byte("i-wz9g2hqiikg0aliyun2b")) + return + } + if r.RequestURI == "/latest/meta-data/region-id" { + w.Write([]byte("cn-shenzhen")) + return + } + if r.RequestURI == "/latest/meta-data/zone-id" { + w.Write([]byte("cn-shenzhen-a")) + return + } + + http.Error(w, "not found", http.StatusNotFound) + })) +} + +func TestRetrieveAlibabaCloudMetadata(t *testing.T) { + if testing.Verbose() { + logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"*"}) + } + + server := initECSTestServer() + defer server.Close() + + config, err := common.NewConfigFrom(map[string]interface{}{ + "host": server.Listener.Addr().String(), + }) + + if err != nil { + t.Fatal(err) + } + + p, err := newCloudMetadata(*config) + if err != nil { + t.Fatal(err) + } + + actual, err := p.Run(common.MapStr{}) + if err != nil { + t.Fatal(err) + } + + expected := common.MapStr{ + "meta": common.MapStr{ + "cloud": common.MapStr{ + "provider": "ecs", + "instance_id": "i-wz9g2hqiikg0aliyun2b", + "region": "cn-shenzhen", + "availability_zone": "cn-shenzhen-a", + }, + }, + } + assert.Equal(t, expected, actual) +} diff --git a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go index aec4d9022e8..d182a7e5336 100644 --- a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go +++ b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go @@ -2,7 +2,8 @@ package add_cloud_metadata import "github.com/elastic/beats/libbeat/common" -// Tenccent Cloud Metadata Service +// Tencent Cloud Metadata Service +// Document https://www.qcloud.com/document/product/213/4934 func newQcloudMetadataFetcher(c common.Config) (*metadataFetcher, error) { qcloudMetadataHost := "metadata.tencentyun.com" qcloudMetadataInstanceIDURI := "/meta-data/instance-id" From b9f1b50dd153c1970f3df74c18dce751be4ef1c2 Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Thu, 27 Apr 2017 17:53:39 -0700 Subject: [PATCH 28/40] Add community beats topic to devguide --- docs/devguide/index.asciidoc | 2 ++ libbeat/docs/communitybeats.asciidoc | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/devguide/index.asciidoc b/docs/devguide/index.asciidoc index d93c8b2150b..1b06c9cf3b0 100644 --- a/docs/devguide/index.asciidoc +++ b/docs/devguide/index.asciidoc @@ -21,6 +21,8 @@ include::../../libbeat/docs/version.asciidoc[] include::./contributing.asciidoc[] +include::../../libbeat/docs/communitybeats.asciidoc[] + include::./newbeat.asciidoc[] include::./event-conventions.asciidoc[] diff --git a/libbeat/docs/communitybeats.asciidoc b/libbeat/docs/communitybeats.asciidoc index 749dc3ea3bd..40a27c232fc 100644 --- a/libbeat/docs/communitybeats.asciidoc +++ b/libbeat/docs/communitybeats.asciidoc @@ -1,9 +1,21 @@ +////////////////////////////////////////////////////////////////////////// +//// This content appears in both the Beats Platform Reference and the +//// Beats Developer Guide. +////////////////////////////////////////////////////////////////////////// + [[community-beats]] == Community Beats The open source community has been hard at work developing new Beats. You can check out some of them here. +Have a question about a community Beat? You can post questions and discuss issues in the +https://discuss.elastic.co/c/beats/community-beats[Community Beats] category of the Beats discussion forum. + +Have you created a Beat that's not listed? If so, add the name and description of your Beat to the source document for +https://github.com/elastic/beats/blob/master/libbeat/docs/communitybeats.asciidoc[Community Beats] and https://help.github.com/articles/using-pull-requests[open a pull request] in the https://github.com/elastic/beats[Beats GitHub repository] to get your change merged. When you're ready, go ahead and https://discuss.elastic.co/c/annoucements[announce] your new Beat in the Elastic +discussion forum. + NOTE: Elastic provides no warranty or support for community-sourced Beats. [horizontal] @@ -62,15 +74,3 @@ https://github.com/cleesmith/unifiedbeat[unifiedbeat]:: Reads records from Unifi network intrusion detection software and indexes the records in Elasticsearch. https://github.com/mrkschan/uwsgibeat[uwsgibeat]:: Reads stats from uWSGI. https://github.com/eskibars/wmibeat[wmibeat]:: Uses WMI to grab your favorite, configurable Windows metrics. - -Have you created a Beat that's not listed? If so, add the name and description of your Beat to the source document for -https://github.com/elastic/beats/blob/master/libbeat/docs/communitybeats.asciidoc[Community Beats] and https://help.github.com/articles/using-pull-requests[open a pull request] in the https://github.com/elastic/beats[Beats GitHub repository] to get your change merged. When you're ready, go ahead and https://discuss.elastic.co/c/annoucements[announce] your new Beat in the Elastic -discussion forum. - -[float] -[[getting-help-community-beats]] -=== Getting Help with Community Beats - -Have a question about a community Beat? You can post questions and discuss issues in the -https://discuss.elastic.co/c/beats/community-beats[Community Beats] category of the Beats discussion forum. - From 6b01fa9dce0273a22e39f93cb72df5e52209e01a Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Fri, 28 Apr 2017 09:08:12 +0200 Subject: [PATCH 29/40] Heartbeat event format (#4091) See PR #4091 for detailed list of event format changes and sample events. - export transport.DialWith - update heartbeat look: - add look.Status - add some missing godoc - rewrite heartbeat dialchain package - simplify package - have connection layers add standardized connection data to the final event - add some more helpers - add some godocs - update event format. See PR #4091 - add more detailed duration measure to HTTP module - update job settings - introduce explicit structs for job settings - job settings can contain static monitor fields to be added to every event - helpers can add more static monitor fields to settings - update fields.yml structure - update kibana dashboard --- CHANGELOG.asciidoc | 1 + heartbeat/_meta/fields.yml | 335 +++++++++++++----- .../02014c80-29d2-11e7-a68f-bfaa2341cc52.json | 24 ++ .../c49bd160-eb17-11e6-be20-559646f8b9ba.json | 23 -- .../091c3a90-eb1e-11e6-be20-559646f8b9ba.json | 8 +- .../0f4c0560-eb20-11e6-9f11-159ff202874a.json | 8 +- .../1738dbc0-eb1d-11e6-be20-559646f8b9ba.json | 8 +- .../920e8140-eb1a-11e6-be20-559646f8b9ba.json | 8 +- .../c65ef340-eb19-11e6-be20-559646f8b9ba.json | 6 +- heartbeat/beater/manager.go | 48 ++- heartbeat/docs/fields.asciidoc | 335 +++++++++++++++--- heartbeat/look/look.go | 19 +- .../monitors/active/dialchain/dialchain.go | 178 ++-------- heartbeat/monitors/active/dialchain/net.go | 94 +++++ heartbeat/monitors/active/dialchain/socks5.go | 38 ++ heartbeat/monitors/active/dialchain/tls.go | 42 +++ heartbeat/monitors/active/dialchain/util.go | 111 ++++++ .../monitors/active/http/simple_transp.go | 15 +- heartbeat/monitors/active/http/task.go | 118 +++--- heartbeat/monitors/active/icmp/icmp.go | 31 +- heartbeat/monitors/active/icmp/loop.go | 2 +- heartbeat/monitors/active/tcp/task.go | 94 +++-- heartbeat/monitors/util.go | 314 +++++++++++----- libbeat/outputs/transport/proxy.go | 2 +- libbeat/outputs/transport/tcp.go | 2 +- libbeat/outputs/transport/tls.go | 1 + libbeat/outputs/transport/util.go | 5 +- 27 files changed, 1321 insertions(+), 549 deletions(-) create mode 100644 heartbeat/_meta/kibana/search/02014c80-29d2-11e7-a68f-bfaa2341cc52.json delete mode 100644 heartbeat/_meta/kibana/search/c49bd160-eb17-11e6-be20-559646f8b9ba.json create mode 100644 heartbeat/monitors/active/dialchain/net.go create mode 100644 heartbeat/monitors/active/dialchain/socks5.go create mode 100644 heartbeat/monitors/active/dialchain/tls.go create mode 100644 heartbeat/monitors/active/dialchain/util.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index a077f61861b..2dc7db9db68 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -30,6 +30,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Add Icinga module. {pull}3904[3904] *Heartbeat* +- Event format and field naming changes in Heartbeat and sample Dashboard. {pull}4091[4091] *Metricbeat* - Linux cgroup metrics are now enabled by default for the system process diff --git a/heartbeat/_meta/fields.yml b/heartbeat/_meta/fields.yml index 763874c139b..640b2dcbe33 100644 --- a/heartbeat/_meta/fields.yml +++ b/heartbeat/_meta/fields.yml @@ -1,132 +1,283 @@ - key: common - title: "Common monitoring fields" + title: "Common Heartbeat Monitor" description: fields: - - name: type - type: keyword - required: true - description: > - The monitor type. - - name: monitor - type: keyword + type: group description: > - Monitor job name. + Common monitor fields. - - name: scheme - type: keyword - description: > - Address url scheme. For example `tcp`, `ssl`, `http`, and `https`. + fields: + - name: type + type: keyword + description: > + The monitor type. - - name: host - type: keyword - description: > - Hostname of service being monitored. Can be missing, if service is - monitored by IP. + - name: name + type: keyword + description: > + The monitors configured name - - name: port - type: integer - description: > - Service port number. + - name: id + type: keyword + description: > + The monitors full job ID as used by heartbeat. - - name: url - type: text - description: > - Service url used by monitor. + - name: duration + type: group + description: total monitoring test duration + fields: + - name: us + type: long + description: Duration in microseconds - - name: ip - type: ip - description: > - IP of service being monitored. If service is monitored by hostname, - the `ip` field contains the resolved ip address for the current host. + - name: scheme + type: keyword + description: > + Address url scheme. For example `tcp`, `tls`, `http`, and `https`. - - name: duration - type: group - description: total monitoring test duration - fields: - - name: us - type: long - description: Duration in microseconds + - name: host + type: keyword + description: > + Hostname of service being monitored. Can be missing, if service is + monitored by IP. - - name: resolve_rtt - type: group - description: Duration required to resolve an IP from hostname. - fields: - - name: us - type: long - description: Duration in microseconds + - name: ip + type: ip + description: > + IP of service being monitored. If service is monitored by hostname, + the `ip` field contains the resolved ip address for the current host. - - name: icmp_rtt - type: group - description: ICMP Echo Request and Reply round trip time - fields: - - name: us - type: long - description: Duration in microseconds + - name: status + required: true + type: keyword + description: > + Indicator if monitor could validate the service to be available. - - name: tcp_connect_rtt +- key: resolve + title: "Host Lookup" + description: + fields: + - name: resolve type: group description: > - Duration required to establish a TCP connection based on already - available IP address. + Host lookup fields. fields: - - name: us - type: long - description: Duration in microseconds + - name: host + type: keyword + description: > + Hostname of service being monitored. + + - name: ip + type: ip + description: > + IP address found for the given host. - - name: socks5_connect_rtt + - name: rtt + type: group + description: Duration required to resolve an IP from hostname. + fields: + - name: us + type: long + description: Duration in microseconds + +- key: icmp + title: "ICMP" + description: + fields: + - name: icmp type: group description: > - Time required to establish a connection via SOCKS5 to endpoint based on available - connection to SOCKS5 proxy. + IP ping fields. fields: - - name: us - type: long - description: Duration in microseconds + - name: requests + type: integer + description: > + Number if ICMP EchoRequests send. - - name: tls_handshake_rtt + - name: rtt + type: group + description: ICMP Echo Request and Reply round trip time + fields: + - name: us + type: long + description: Duration in microseconds + +- key: tcp + title: "TCP Layer" + description: + fields: + - name: tcp type: group description: > - Time required to finish TLS handshake based on already available network - connection. + TCP network layer related fields. fields: - - name: us - type: long - description: Duration in microseconds + - name: port + type: integer + description: > + Service port number. + + - name: rtt + type: group + description: > + TCP layer round trip times. + fields: + - name: connect + type: group + description: > + Duration required to establish a TCP connection based on already + available IP address. + fields: + - name: us + type: long + description: Duration in microseconds + + - name: validate + type: group + description: > + Duration of validation step based on existing TCP connection. + fields: + - name: us + type: long + description: Duration in microseconds - - name: http_rtt +- key: socks5 + title: "SOCKS5 Proxy" + description: + fields: + - name: socks5 type: group description: > - Time required between sending the HTTP request and first by from HTTP - response being read. Duration based on already available network connection. + SOCKS5 proxy related fields: fields: - - name: us - type: long - description: Duration in microseconds + - name: rtt + type: group + description: > + TLS layer round trip times. + fields: + - name: connect + type: group + description: > + Time required to establish a connection via SOCKS5 to endpoint + based on available connection to SOCKS5 proxy. + fields: + - name: us + type: long + description: Duration in microseconds + - - name: validate_rtt +- key: tls + title: "TLS Encryption Layer" + description: + fields: + - name: tls type: group description: > - Time required for validating the connection if connection checks are configured. + TLS layer related fields. fields: - - name: us - type: long - description: Duration in microseconds + - name: rtt + type: group + description: > + TLS layer round trip times. + fields: + - name: handshake + type: group + description: > + Time required to finish TLS handshake based on already available network + connection. + fields: + - name: us + type: long + description: Duration in microseconds - - name: response +- key: http + title: "HTTP Monitor" + description: + fields: + - name: http type: group description: > - Service response parameters. - + HTTP related fields. fields: - - name: status - type: integer + - name: url + type: text description: > - Response status code. + Service url used by monitor. - - name: up - required: true - type: boolean - description: > - Boolean indicator if monitor could validate the service to be available. + - name: response + type: group + description: > + Service response parameters. + fields: + - name: status + type: integer + description: > + Response status code. + - name: rtt + type: group + description: > + HTTP layer round trip times. + fields: + - name: validate + type: group + description: | + Duration between first byte of HTTP request being written and + response being processed by validator. Duration based on already + available network connection. + + Note: if validator is not reading body or only a prefix, this + number does not fully represent the total time needed + to read the body. + fields: + - name: us + type: long + description: Duration in microseconds + + - name: validate_body + type: group + description: | + Duration of validator required to read and validate the response + body. + + Note: if validator is not reading body or only a prefix, this + number does not fully represent the total time needed + to read the body. + fields: + - name: us + type: long + description: Duration in microseconds + + - name: write_request + type: group + description: + Duration of sending the complete HTTP request. Duration based on + already available network connection. + fields: + - name: us + type: long + description: Duration in microseconds + + - name: response_header + type: group + description: + Time required between sending the start of sending the HTTP + request and first by from HTTP response being read. Duration + based on already available network connection. + fields: + - name: us + type: long + description: Duration in microseconds + + - name: total + type: group + description: | + Duration required to process the HTTP transaction. Starts with + the initial TCP connection attempt. Ends with after validator + did check the response. + Note: if validator is not reading body or only a prefix, this + number does not fully represent the total time needed. + fields: + - name: us + type: long + description: Duration in microseconds diff --git a/heartbeat/_meta/kibana/search/02014c80-29d2-11e7-a68f-bfaa2341cc52.json b/heartbeat/_meta/kibana/search/02014c80-29d2-11e7-a68f-bfaa2341cc52.json new file mode 100644 index 00000000000..53ba45f044e --- /dev/null +++ b/heartbeat/_meta/kibana/search/02014c80-29d2-11e7-a68f-bfaa2341cc52.json @@ -0,0 +1,24 @@ +{ + "sort": [ + "@timestamp", + "desc" + ], + "hits": 0, + "description": "", + "title": "Heartbeat HTTP pings", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\n \"index\": \"heartbeat-*\",\n \"highlightAll\": true,\n \"query\": {\n \"query_string\": {\n \"analyze_wildcard\": true,\n \"query\": \"*\"\n }\n },\n \"filter\": [\n {\n \"$state\": {\n \"store\": \"appState\"\n },\n \"meta\": {\n \"alias\": null,\n \"disabled\": false,\n \"index\": \"heartbeat-*\",\n \"key\": \"monitor.name\",\n \"negate\": false,\n \"value\": \"http\"\n },\n \"query\": {\n \"match\": {\n \"monitor.name\": {\n \"query\": \"http\",\n \"type\": \"phrase\"\n }\n }\n }\n }\n ]\n}" + }, + "columns": [ + "monitor.id", + "http.url", + "monitor.status", + "http.response.status", + "monitor.duration.us", + "tcp.rtt.connect.us", + "tls.rtt.handshake.us", + "resolve.rtt.us", + "http.rtt.content.us" + ] +} \ No newline at end of file diff --git a/heartbeat/_meta/kibana/search/c49bd160-eb17-11e6-be20-559646f8b9ba.json b/heartbeat/_meta/kibana/search/c49bd160-eb17-11e6-be20-559646f8b9ba.json deleted file mode 100644 index bb7b085dc25..00000000000 --- a/heartbeat/_meta/kibana/search/c49bd160-eb17-11e6-be20-559646f8b9ba.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "sort": [ - "@timestamp", - "desc" - ], - "hits": 0, - "description": "", - "title": "Heartbeat HTTP pings", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"heartbeat-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[{\"meta\":{\"negate\":false,\"index\":\"heartbeat-*\",\"key\":\"type\",\"value\":\"http\",\"disabled\":false,\"alias\":null},\"query\":{\"match\":{\"type\":{\"query\":\"http\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" - }, - "columns": [ - "monitor", - "up", - "response.status", - "duration.us", - "tcp_connect_rtt.us", - "tls_handshake_rtt.us", - "resolve_rtt.us", - "http_rtt.us" - ] -} \ No newline at end of file diff --git a/heartbeat/_meta/kibana/visualization/091c3a90-eb1e-11e6-be20-559646f8b9ba.json b/heartbeat/_meta/kibana/visualization/091c3a90-eb1e-11e6-be20-559646f8b9ba.json index e953f3f9a7f..dec5f466c78 100644 --- a/heartbeat/_meta/kibana/visualization/091c3a90-eb1e-11e6-be20-559646f8b9ba.json +++ b/heartbeat/_meta/kibana/visualization/091c3a90-eb1e-11e6-be20-559646f8b9ba.json @@ -1,11 +1,11 @@ { - "visState": "{\"title\":\"HTTP up status\",\"type\":\"area\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"percentage\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"up:true\",\"analyze_wildcard\":true}}},\"label\":\"\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"up:false\",\"analyze_wildcard\":true}}}}]}}],\"listeners\":{}}", + "visState": "{\n \"title\": \"HTTP up status\",\n \"type\": \"area\",\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"scale\": \"linear\",\n \"interpolate\": \"linear\",\n \"mode\": \"percentage\",\n \"times\": [],\n \"addTimeMarker\": false,\n \"defaultYExtents\": false,\n \"setYExtents\": true,\n \"yAxis\": {\n \"max\": 100,\n \"min\": 0\n }\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"date_histogram\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"@timestamp\",\n \"interval\": \"auto\",\n \"customInterval\": \"2h\",\n \"min_doc_count\": 1,\n \"extended_bounds\": {}\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"filters\",\n \"schema\": \"group\",\n \"params\": {\n \"filters\": [\n {\n \"input\": {\n \"query\": {\n \"query_string\": {\n \"query\": \"monitor.status: down\",\n \"analyze_wildcard\": true\n }\n }\n },\n \"label\": \"\"\n },\n {\n \"input\": {\n \"query\": {\n \"query_string\": {\n \"query\": \"monitor.status: up\",\n \"analyze_wildcard\": true\n }\n }\n }\n }\n ]\n }\n }\n ],\n \"listeners\": {}\n}", "description": "", "title": "HTTP up status", - "uiStateJSON": "{\"vis\":{\"colors\":{\"up:false\":\"#BF1B00\",\"up:true\":\"#629E51\"}}}", + "uiStateJSON": "{\n \"vis\": {\n \"colors\": {\n \"monitor.status: up\": \"#629E51\",\n \"monitor.status: down\": \"#E24D42\"\n }\n }\n}", "version": 1, - "savedSearchId": "c49bd160-eb17-11e6-be20-559646f8b9ba", + "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" + "searchSourceJSON": "{\n \"filter\": []\n}" } } \ No newline at end of file diff --git a/heartbeat/_meta/kibana/visualization/0f4c0560-eb20-11e6-9f11-159ff202874a.json b/heartbeat/_meta/kibana/visualization/0f4c0560-eb20-11e6-9f11-159ff202874a.json index 087f16bacbe..f9942e48916 100644 --- a/heartbeat/_meta/kibana/visualization/0f4c0560-eb20-11e6-9f11-159ff202874a.json +++ b/heartbeat/_meta/kibana/visualization/0f4c0560-eb20-11e6-9f11-159ff202874a.json @@ -1,11 +1,11 @@ { - "visState": "{\"title\":\"HTTP duration heatmap\",\"type\":\"heatmap\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":10,\"colorSchema\":\"Blues\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":false,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"color\":\"#555\"}}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"histogram\",\"schema\":\"group\",\"params\":{\"field\":\"duration.us\",\"interval\":50000,\"extended_bounds\":{}}}],\"listeners\":{}}", + "visState": "{\n \"title\": \"HTTP duration heatmap\",\n \"type\": \"heatmap\",\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": true,\n \"enableHover\": false,\n \"legendPosition\": \"right\",\n \"times\": [],\n \"colorsNumber\": 10,\n \"colorSchema\": \"Blues\",\n \"setColorRange\": false,\n \"colorsRange\": [],\n \"invertColors\": false,\n \"percentageMode\": false,\n \"valueAxes\": [\n {\n \"show\": false,\n \"id\": \"ValueAxis-1\",\n \"type\": \"value\",\n \"scale\": {\n \"type\": \"linear\",\n \"defaultYExtents\": false\n },\n \"labels\": {\n \"show\": false,\n \"rotate\": 0,\n \"color\": \"#555\"\n }\n }\n ]\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"schema\": \"metric\",\n \"params\": {}\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"date_histogram\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"@timestamp\",\n \"interval\": \"auto\",\n \"customInterval\": \"2h\",\n \"min_doc_count\": 1,\n \"extended_bounds\": {}\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"histogram\",\n \"schema\": \"group\",\n \"params\": {\n \"field\": \"monitor.duration.us\",\n \"interval\": 50000,\n \"extended_bounds\": {}\n }\n }\n ],\n \"listeners\": {}\n}", "description": "", "title": "HTTP duration heatmap", - "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 2\":\"rgb(247,251,255)\",\"2 - 3\":\"rgb(227,238,249)\",\"3 - 4\":\"rgb(208,225,242)\",\"4 - 5\":\"rgb(182,212,233)\",\"5 - 6\":\"rgb(148,196,223)\",\"6 - 8\":\"rgb(107,174,214)\",\"8 - 9\":\"rgb(74,152,201)\",\"9 - 10\":\"rgb(46,126,188)\",\"10 - 11\":\"rgb(23,100,171)\",\"11 - 12\":\"rgb(8,74,145)\"}}}", + "uiStateJSON": "{\n \"vis\": {\n \"defaultColors\": {\n \"0 - 2\": \"rgb(247,251,255)\",\n \"2 - 3\": \"rgb(227,238,249)\",\n \"3 - 4\": \"rgb(208,225,242)\",\n \"4 - 5\": \"rgb(182,212,233)\",\n \"5 - 6\": \"rgb(148,196,223)\",\n \"6 - 8\": \"rgb(107,174,214)\",\n \"8 - 9\": \"rgb(74,152,201)\",\n \"9 - 10\": \"rgb(46,126,188)\",\n \"10 - 11\": \"rgb(23,100,171)\",\n \"11 - 12\": \"rgb(8,74,145)\"\n }\n }\n}", "version": 1, - "savedSearchId": "c49bd160-eb17-11e6-be20-559646f8b9ba", + "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" + "searchSourceJSON": "{\n \"filter\": []\n}" } } \ No newline at end of file diff --git a/heartbeat/_meta/kibana/visualization/1738dbc0-eb1d-11e6-be20-559646f8b9ba.json b/heartbeat/_meta/kibana/visualization/1738dbc0-eb1d-11e6-be20-559646f8b9ba.json index dca9fa71bff..f6fc8d18e27 100644 --- a/heartbeat/_meta/kibana/visualization/1738dbc0-eb1d-11e6-be20-559646f8b9ba.json +++ b/heartbeat/_meta/kibana/visualization/1738dbc0-eb1d-11e6-be20-559646f8b9ba.json @@ -1,11 +1,11 @@ { - "visState": "{\"title\":\"HTTP monitors\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"duration.us\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"monitor\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"resolve_rtt.us\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"tcp_connect_rtt.us\"}},{\"id\":\"7\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"tls_handshake_rtt.us\"}},{\"id\":\"8\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"http_rtt.us\"}}],\"listeners\":{}}", + "visState": "{\n \"title\": \"HTTP monitors\",\n \"type\": \"table\",\n \"params\": {\n \"perPage\": 10,\n \"showPartialRows\": false,\n \"showMeticsAtAllLevels\": false,\n \"sort\": {\n \"columnIndex\": null,\n \"direction\": null\n },\n \"showTotal\": false,\n \"totalFunc\": \"sum\"\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"max\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"monitor.duration.us\"\n }\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"bucket\",\n \"params\": {\n \"field\": \"monitor.id\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n },\n {\n \"id\": \"5\",\n \"enabled\": true,\n \"type\": \"max\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"resolve.rtt.us\"\n }\n },\n {\n \"id\": \"6\",\n \"enabled\": true,\n \"type\": \"max\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"tcp.rtt.connect.us\"\n }\n },\n {\n \"id\": \"7\",\n \"enabled\": true,\n \"type\": \"max\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"tls.rtt.handshake.us\"\n }\n },\n {\n \"id\": \"8\",\n \"enabled\": true,\n \"type\": \"max\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"http.rtt.response_header.us\"\n }\n }\n ],\n \"listeners\": {}\n}", "description": "", "title": "HTTP monitors", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "uiStateJSON": "{\n \"vis\": {\n \"params\": {\n \"sort\": {\n \"columnIndex\": null,\n \"direction\": null\n }\n }\n }\n}", "version": 1, - "savedSearchId": "c49bd160-eb17-11e6-be20-559646f8b9ba", + "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" + "searchSourceJSON": "{\n \"filter\": []\n}" } } \ No newline at end of file diff --git a/heartbeat/_meta/kibana/visualization/920e8140-eb1a-11e6-be20-559646f8b9ba.json b/heartbeat/_meta/kibana/visualization/920e8140-eb1a-11e6-be20-559646f8b9ba.json index a5b25108c21..4b9caec1d3e 100644 --- a/heartbeat/_meta/kibana/visualization/920e8140-eb1a-11e6-be20-559646f8b9ba.json +++ b/heartbeat/_meta/kibana/visualization/920e8140-eb1a-11e6-be20-559646f8b9ba.json @@ -1,11 +1,11 @@ { - "visState": "{\"title\":\"HTTP monitors status\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"monitor\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"segment\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"up: true\",\"analyze_wildcard\":true}}},\"label\":\"\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"up: false\",\"analyze_wildcard\":true}}}}]}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"response.status\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "visState": "{\n \"title\": \"HTTP monitors status\",\n \"type\": \"pie\",\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"bottom\",\n \"isDonut\": false\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"cardinality\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"monitor.id\"\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"filters\",\n \"schema\": \"segment\",\n \"params\": {\n \"filters\": [\n {\n \"input\": {\n \"query\": {\n \"query_string\": {\n \"query\": \"monitor.status: up\",\n \"analyze_wildcard\": true\n }\n }\n },\n \"label\": \"\"\n },\n {\n \"input\": {\n \"query\": {\n \"query_string\": {\n \"query\": \"monitor.status: down\",\n \"analyze_wildcard\": true\n }\n }\n }\n }\n ]\n }\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"terms\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"http.response.status\",\n \"size\": 5,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}", "description": "", "title": "HTTP monitors status", - "uiStateJSON": "{\"vis\":{\"colors\":{\"200\":\"#B7DBAB\",\"up: true\":\"#629E51\",\"up: false\":\"#E24D42\"},\"legendOpen\":true}}", + "uiStateJSON": "{\n \"vis\": {\n \"colors\": {\n \"200\": \"#B7DBAB\",\n \"monitor.status: up\": \"#629E51\",\n \"monitor.status: down\": \"#E24D42\"\n },\n \"legendOpen\": true\n }\n}", "version": 1, - "savedSearchId": "c49bd160-eb17-11e6-be20-559646f8b9ba", + "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" + "searchSourceJSON": "{\n \"filter\": []\n}" } } \ No newline at end of file diff --git a/heartbeat/_meta/kibana/visualization/c65ef340-eb19-11e6-be20-559646f8b9ba.json b/heartbeat/_meta/kibana/visualization/c65ef340-eb19-11e6-be20-559646f8b9ba.json index d62474012ac..4bc9b3c19d1 100644 --- a/heartbeat/_meta/kibana/visualization/c65ef340-eb19-11e6-be20-559646f8b9ba.json +++ b/heartbeat/_meta/kibana/visualization/c65ef340-eb19-11e6-be20-559646f8b9ba.json @@ -1,11 +1,11 @@ { - "visState": "{\"title\":\"HTTP ping times\",\"type\":\"area\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"resolve_rtt.us\",\"customLabel\":\"\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"tcp_connect_rtt.us\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"tls_handshake_rtt.us\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"http_rtt.us\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", + "visState": "{\n \"title\": \"HTTP ping times\",\n \"type\": \"area\",\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": true,\n \"legendPosition\": \"right\",\n \"scale\": \"linear\",\n \"interpolate\": \"linear\",\n \"mode\": \"stacked\",\n \"times\": [],\n \"addTimeMarker\": false,\n \"defaultYExtents\": false,\n \"setYExtents\": false\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"max\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"resolve.rtt.us\",\n \"customLabel\": \"\"\n }\n },\n {\n \"id\": \"3\",\n \"enabled\": true,\n \"type\": \"max\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"tcp.rtt.connect.us\"\n }\n },\n {\n \"id\": \"5\",\n \"enabled\": true,\n \"type\": \"max\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"tls.rtt.handshake.us\"\n }\n },\n {\n \"id\": \"4\",\n \"enabled\": true,\n \"type\": \"max\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"http.rtt.response_header.us\"\n }\n },\n {\n \"id\": \"2\",\n \"enabled\": true,\n \"type\": \"date_histogram\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"@timestamp\",\n \"interval\": \"auto\",\n \"customInterval\": \"2h\",\n \"min_doc_count\": 1,\n \"extended_bounds\": {}\n }\n }\n ],\n \"listeners\": {}\n}", "description": "", "title": "HTTP ping times", "uiStateJSON": "{}", "version": 1, - "savedSearchId": "c49bd160-eb17-11e6-be20-559646f8b9ba", + "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52", "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" + "searchSourceJSON": "{\n \"filter\": []\n}" } } \ No newline at end of file diff --git a/heartbeat/beater/manager.go b/heartbeat/beater/manager.go index 489d8b1aa1f..90804087dac 100644 --- a/heartbeat/beater/manager.go +++ b/heartbeat/beater/manager.go @@ -36,6 +36,8 @@ type Monitor struct { } type MonitorTask struct { + name, typ string + job monitors.Job schedule scheduler.Schedule cancel JobCanceller @@ -49,6 +51,8 @@ type JobCanceller func() error var defaultFilePollInterval = 5 * time.Second +const defaultEventType = "monitor" + func newMonitorManager( client publisher.Client, jobControl JobControl, @@ -151,6 +155,8 @@ func (m *Monitor) Update(configs []*common.Config) error { } shared := struct { + Name string `config:"name"` + Type string `config:"type"` Schedule *schedule.Schedule `config:"schedule" validate:"required"` }{} if err := config.Unpack(&shared); err != nil { @@ -164,8 +170,15 @@ func (m *Monitor) Update(configs []*common.Config) error { return err } + name := shared.Name + if name == "" { + name = shared.Type + } + for _, job := range jobs { all[job.Name()] = MonitorTask{ + name: name, + typ: shared.Type, job: job, schedule: shared.Schedule, } @@ -179,10 +192,10 @@ func (m *Monitor) Update(configs []*common.Config) error { m.active = map[string]MonitorTask{} // start new and reconfigured tasks - for name, t := range all { - job := createJob(m.manager.client, name, t.job) - t.cancel = m.manager.jobControl.Add(t.schedule, name, job) - m.active[name] = t + for id, t := range all { + job := createJob(m.manager.client, t.job, t.name, t.typ) + t.cancel = m.manager.jobControl.Add(t.schedule, id, job) + m.active[id] = t } return nil @@ -221,19 +234,11 @@ func createWatchUpdater(monitor *Monitor) func(content []byte) { } } -func createJob( - client publisher.Client, - name string, - r monitors.Job, -) scheduler.TaskFunc { - return createJobTask(client, name, r) +func createJob(client publisher.Client, r monitors.Job, name, typ string) scheduler.TaskFunc { + return createJobTask(client, r, name, typ) } -func createJobTask( - client publisher.Client, - name string, - r monitors.TaskRunner, -) scheduler.TaskFunc { +func createJobTask(client publisher.Client, r monitors.TaskRunner, name, typ string) scheduler.TaskFunc { return func() []scheduler.TaskFunc { event, next, err := r.Run() if err != nil { @@ -241,7 +246,16 @@ func createJobTask( } if event != nil { - event["monitor"] = name + event.DeepUpdate(common.MapStr{ + "monitor": common.MapStr{ + "name": name, + "type": typ, + }, + }) + + if _, exists := event["type"]; !exists { + event["type"] = defaultEventType + } client.PublishEvent(event) } @@ -251,7 +265,7 @@ func createJobTask( cont := make([]scheduler.TaskFunc, len(next)) for i, n := range next { - cont[i] = createJobTask(client, name, n) + cont[i] = createJobTask(client, n, name, typ) } return cont } diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc index b64cdbd166a..8a3ced1467f 100644 --- a/heartbeat/docs/fields.asciidoc +++ b/heartbeat/docs/fields.asciidoc @@ -15,7 +15,13 @@ grouped in the following categories: * <> * <> * <> +* <> +* <> * <> +* <> +* <> +* <> +* <> -- [[exported-fields-beat]] @@ -160,63 +166,73 @@ Region in which this host is running. [[exported-fields-common]] -== Common monitoring fields Fields +== Common Heartbeat Monitor Fields None [float] -=== type +== monitor Fields -type: keyword +Common monitor fields. -required: True + + +[float] +=== monitor.type + +type: keyword The monitor type. [float] -=== monitor +=== monitor.name type: keyword -Monitor job name. +The monitors configured name [float] -=== scheme +=== monitor.id type: keyword -Address url scheme. For example `tcp`, `ssl`, `http`, and `https`. +The monitors full job ID as used by heartbeat. [float] -=== host +== duration Fields -type: keyword +total monitoring test duration -Hostname of service being monitored. Can be missing, if service is monitored by IP. +[float] +=== monitor.duration.us + +type: long + +Duration in microseconds [float] -=== port +=== monitor.scheme -type: integer +type: keyword -Service port number. +Address url scheme. For example `tcp`, `tls`, `http`, and `https`. [float] -=== url +=== monitor.host -type: text +type: keyword -Service url used by monitor. +Hostname of service being monitored. Can be missing, if service is monitored by IP. [float] -=== ip +=== monitor.ip type: ip @@ -224,138 +240,175 @@ IP of service being monitored. If service is monitored by hostname, the `ip` fie [float] -== duration Fields +=== monitor.status -total monitoring test duration +type: keyword + +required: True + +Indicator if monitor could validate the service to be available. + + +[[exported-fields-http]] +== HTTP Monitor Fields + +None [float] -=== duration.us +== http Fields + +HTTP related fields. -type: long -Duration in microseconds [float] -== resolve_rtt Fields +=== http.url -Duration required to resolve an IP from hostname. +type: text + +Service url used by monitor. [float] -=== resolve_rtt.us +== response Fields + +Service response parameters. -type: long -Duration in microseconds [float] -== icmp_rtt Fields +=== http.response.status -ICMP Echo Request and Reply round trip time +type: integer + +Response status code. [float] -=== icmp_rtt.us +== rtt Fields + +HTTP layer round trip times. -type: long -Duration in microseconds [float] -== tcp_connect_rtt Fields +== validate Fields -Duration required to establish a TCP connection based on already available IP address. +Duration between first byte of HTTP request being written and +response being processed by validator. Duration based on already +available network connection. + +Note: if validator is not reading body or only a prefix, this + number does not fully represent the total time needed + to read the body. [float] -=== tcp_connect_rtt.us +=== http.rtt.validate.us type: long Duration in microseconds [float] -== socks5_connect_rtt Fields +== validate_body Fields -Time required to establish a connection via SOCKS5 to endpoint based on available connection to SOCKS5 proxy. +Duration of validator required to read and validate the response +body. + +Note: if validator is not reading body or only a prefix, this + number does not fully represent the total time needed + to read the body. [float] -=== socks5_connect_rtt.us +=== http.rtt.validate_body.us type: long Duration in microseconds [float] -== tls_handshake_rtt Fields - -Time required to finish TLS handshake based on already available network connection. +== write_request Fields +Duration of sending the complete HTTP request. Duration based on already available network connection. [float] -=== tls_handshake_rtt.us +=== http.rtt.write_request.us type: long Duration in microseconds [float] -== http_rtt Fields - -Time required between sending the HTTP request and first by from HTTP response being read. Duration based on already available network connection. +== response_header Fields +Time required between sending the start of sending the HTTP request and first by from HTTP response being read. Duration based on already available network connection. [float] -=== http_rtt.us +=== http.rtt.response_header.us type: long Duration in microseconds [float] -== validate_rtt Fields +== total Fields + +Duration required to process the HTTP transaction. Starts with +the initial TCP connection attempt. Ends with after validator +did check the response. -Time required for validating the connection if connection checks are configured. +Note: if validator is not reading body or only a prefix, this + number does not fully represent the total time needed. [float] -=== validate_rtt.us +=== http.rtt.total.us type: long Duration in microseconds +[[exported-fields-icmp]] +== ICMP Fields + +None + + [float] -== response Fields +== icmp Fields -Service response parameters. +IP ping fields. [float] -=== response.status +=== icmp.requests type: integer -Response status code. +Number if ICMP EchoRequests send. [float] -=== up +== rtt Fields -type: boolean +ICMP Echo Request and Reply round trip time -required: True -Boolean indicator if monitor could validate the service to be available. +[float] +=== icmp.rtt.us + +type: long +Duration in microseconds [[exported-fields-kubernetes]] == Kubernetes info Fields @@ -404,3 +457,169 @@ type: keyword Kubernetes container name +[[exported-fields-resolve]] +== Host Lookup Fields + +None + + +[float] +== resolve Fields + +Host lookup fields. + + + +[float] +=== resolve.host + +type: keyword + +Hostname of service being monitored. + + +[float] +=== resolve.ip + +type: ip + +IP address found for the given host. + + +[float] +== rtt Fields + +Duration required to resolve an IP from hostname. + + +[float] +=== resolve.rtt.us + +type: long + +Duration in microseconds + +[[exported-fields-socks5]] +== SOCKS5 Proxy Fields + +None + + +[float] +== socks5 Fields + +SOCKS5 proxy related fields: + + + +[float] +== rtt Fields + +TLS layer round trip times. + + + +[float] +== connect Fields + +Time required to establish a connection via SOCKS5 to endpoint based on available connection to SOCKS5 proxy. + + + +[float] +=== socks5.rtt.connect.us + +type: long + +Duration in microseconds + +[[exported-fields-tcp]] +== TCP Layer Fields + +None + + +[float] +== tcp Fields + +TCP network layer related fields. + + + +[float] +=== tcp.port + +type: integer + +Service port number. + + +[float] +== rtt Fields + +TCP layer round trip times. + + + +[float] +== connect Fields + +Duration required to establish a TCP connection based on already available IP address. + + + +[float] +=== tcp.rtt.connect.us + +type: long + +Duration in microseconds + +[float] +== validate Fields + +Duration of validation step based on existing TCP connection. + + + +[float] +=== tcp.rtt.validate.us + +type: long + +Duration in microseconds + +[[exported-fields-tls]] +== TLS Encryption Layer Fields + +None + + +[float] +== tls Fields + +TLS layer related fields. + + + +[float] +== rtt Fields + +TLS layer round trip times. + + + +[float] +== handshake Fields + +Time required to finish TLS handshake based on already available network connection. + + + +[float] +=== tls.rtt.handshake.us + +type: long + +Duration in microseconds + diff --git a/heartbeat/look/look.go b/heartbeat/look/look.go index c9e64d23f04..0fd0e4a3c3d 100644 --- a/heartbeat/look/look.go +++ b/heartbeat/look/look.go @@ -1,5 +1,5 @@ // Package look defines common formatters for fields/types to be used when -// generating custom events. +// generating heartbeat events. package look import ( @@ -10,12 +10,19 @@ import ( "github.com/elastic/beats/heartbeat/reason" ) +// RTT formats a round-trip-time given as time.Duration into an +// event field. The duration is stored in `{"us": rtt}`. func RTT(rtt time.Duration) common.MapStr { + if rtt < 0 { + rtt = 0 + } + return common.MapStr{ "us": rtt / (time.Microsecond / time.Nanosecond), } } +// Reason formats an error into an error event field. func Reason(err error) common.MapStr { if r, ok := err.(reason.Reason); ok { return reason.Fail(r) @@ -23,6 +30,16 @@ func Reason(err error) common.MapStr { return reason.FailIO(err) } +// Timestamp converts an event timestamp into an compatible event timestamp for +// reporting. func Timestamp(t time.Time) common.Time { return common.Time(t) } + +// Status creates a service status message from an error value. +func Status(err error) string { + if err == nil { + return "up" + } + return "down" +} diff --git a/heartbeat/monitors/active/dialchain/dialchain.go b/heartbeat/monitors/active/dialchain/dialchain.go index 096ed6922f7..ceeab0a4a60 100644 --- a/heartbeat/monitors/active/dialchain/dialchain.go +++ b/heartbeat/monitors/active/dialchain/dialchain.go @@ -1,43 +1,31 @@ package dialchain import ( - "fmt" - "net" - "time" - "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/outputs/transport" - - "github.com/elastic/beats/heartbeat/look" ) -type DialLayerCallback interface { - Start() - Done(err error) -} - -type MeasureLayerRTTCB struct { - Callback MeasureCallback - start time.Time -} - -type MeasureCallback func(start, end time.Time) - +// DialerChain composes builders for multiple network layers, used to build +// the final transport.Dialer object based on the network layers. +// Each layer can hold individual configurations. Use 'Clone' to copy and replace/wrap +// layers at will. +// Once all Layers have been prepared, use Build to build a transport.Dialer that can +// used with any go library network packages relying on standard library based dialers. +// +// For Additional Layering capabilities, DialerChain implements the NetDialer interface. type DialerChain struct { Net NetDialer Layers []Layer } -type NetDialer struct { - Name string - Dialer transport.Dialer -} +// NetDialer provides the most low-level network layer for setting up a network +// connection. NetDialer objects do not support wrapping any lower network layers. +type NetDialer func(common.MapStr) (transport.Dialer, error) -type Layer struct { - Name string - Builder func(transport.Dialer) (transport.Dialer, error) -} +// Layer is a configured network layer, wrapping any lower-level network layers. +type Layer func(common.MapStr, transport.Dialer) (transport.Dialer, error) +// Clone create a shallow copy of c. func (c *DialerChain) Clone() *DialerChain { d := &DialerChain{ Net: c.Net, @@ -47,141 +35,39 @@ func (c *DialerChain) Clone() *DialerChain { return d } -func (c *DialerChain) BuildWith(makeCB func(string) DialLayerCallback) (d transport.Dialer, err error) { - d = LayerCBDialer(makeCB(c.Net.Name), c.Net.Dialer) - for _, layer := range c.Layers { - if d, err = LayerDeltaCBDialer(makeCB(layer.Name), d, layer.Builder); err != nil { - return nil, err - } +// Build create a new transport.Dialer for use with other networking libraries. +func (c *DialerChain) Build(event common.MapStr) (d transport.Dialer, err error) { + d, err = c.Net.build(event) + if err != nil { + return } - return -} - -func (c *DialerChain) BuildWithMeasures(event common.MapStr) (transport.Dialer, error) { - return c.BuildWith(func(name string) DialLayerCallback { - return measureEventRTT(event, name) - }) -} -func (c *DialerChain) Build() (d transport.Dialer, err error) { - d = c.Net.Dialer for _, layer := range c.Layers { - if d, err = layer.Builder(d); err != nil { + if d, err = layer.build(event, d); err != nil { return nil, err } } return } -func (c *DialerChain) TestBuild() error { - _, err := c.Build() - return err -} - -func (c *DialerChain) DialWithMeasurements(network, host string) (fields common.MapStr, conn net.Conn, err error) { - var dialer transport.Dialer - fields = common.MapStr{} - if dialer, err = c.BuildWithMeasures(fields); err == nil { - conn, err = dialer.Dial(network, host) - } - return -} - -func (c *DialerChain) Dial(network, host string) (conn net.Conn, err error) { - var dialer transport.Dialer - if dialer, err = c.Build(); err == nil { - return dialer.Dial(network, host) - } - return -} - +// AddLayer adds another layer to the dialer chain. +// The layer being added is the new topmost network layer using the other +// already present layers on dial. func (c *DialerChain) AddLayer(l Layer) { c.Layers = append(c.Layers, l) } -func measureEventRTT(event common.MapStr, name string) DialLayerCallback { - return &MeasureLayerRTTCB{Callback: func(start, end time.Time) { - event[name] = look.RTT(end.Sub(start)) - }} -} - -func LayerCBDialer(cb DialLayerCallback, d transport.Dialer) transport.Dialer { - return transport.DialerFunc(func(network, address string) (net.Conn, error) { - cb.Start() - c, err := d.Dial(network, address) - cb.Done(err) - return c, err - }) -} - -func LayerDeltaCBDialer( - cb DialLayerCallback, - dialer transport.Dialer, - layer func(transport.Dialer) (transport.Dialer, error), -) (transport.DialerFunc, error) { - starter := transport.DialerFunc(func(network, address string) (net.Conn, error) { - c, err := dialer.Dial(network, address) - cb.Start() - return c, err - }) - - layerInstance, err := layer(starter) - if err != nil { - return nil, err - } - - return func(network, address string) (net.Conn, error) { - c, err := layerInstance.Dial(network, address) - cb.Done(err) - return c, err - }, nil -} - -func ConstAddrDialer(name, addr string, to time.Duration) NetDialer { - return NetDialer{name, transport.DialerFunc(func(network, _ string) (net.Conn, error) { - switch network { - case "tcp", "tcp4", "tcp6", "udp", "udp4", "udp6": - default: - return nil, fmt.Errorf("unsupported network type %v", network) - } - - dialer := &net.Dialer{Timeout: to} - return dialer.Dial(network, addr) - })} -} - -func ConstAddrLayer(addr string, l Layer) Layer { - return Layer{l.Name, func(d transport.Dialer) (transport.Dialer, error) { - forward, err := l.Builder(d) - if err != nil { - return nil, err - } - - return transport.DialerFunc(func(network, _ string) (net.Conn, error) { - return forward.Dial(network, addr) - }), nil - }} -} - -func TCPDialer(name string, to time.Duration) NetDialer { - return NetDialer{name, transport.NetDialer(to)} -} - -func UDPDialer(name string, to time.Duration) NetDialer { - return NetDialer{name, transport.NetDialer(to)} +// TestBuild tries to build the DialerChain and reports any error reported by +// one of the layers. +func (c *DialerChain) TestBuild() error { + _, err := c.Build(common.MapStr{}) + return err } -func SOCKS5Layer(name string, config *transport.ProxyConfig) Layer { - return Layer{name, func(d transport.Dialer) (transport.Dialer, error) { - return transport.ProxyDialer(config, d) - }} +func (d NetDialer) build(event common.MapStr) (transport.Dialer, error) { + return d(event) } -func TLSLayer(name string, config *transport.TLSConfig, timeout time.Duration) Layer { - return Layer{name, func(d transport.Dialer) (transport.Dialer, error) { - return transport.TLSDialer(d, config, timeout) - }} +func (l Layer) build(event common.MapStr, next transport.Dialer) (transport.Dialer, error) { + return l(event, next) } - -func (cb *MeasureLayerRTTCB) Start() { cb.start = time.Now() } -func (cb *MeasureLayerRTTCB) Done(_ error) { cb.Callback(cb.start, time.Now()) } diff --git a/heartbeat/monitors/active/dialchain/net.go b/heartbeat/monitors/active/dialchain/net.go new file mode 100644 index 00000000000..295ccf68f9c --- /dev/null +++ b/heartbeat/monitors/active/dialchain/net.go @@ -0,0 +1,94 @@ +package dialchain + +import ( + "fmt" + "net" + "time" + + "github.com/elastic/beats/heartbeat/look" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/outputs/transport" +) + +// TCPDialer creates a new NetDialer with constant event fields and default +// connection timeout. +// The fields parameter holds additonal constants to be added to the final event +// structure. +// +// The dialer will update the active events with: +// +// { +// "tcp": { +// "port": ..., +// "rtt": { "connect": { "us": ... }} +// } +// } +func TCPDialer(to time.Duration) NetDialer { + return netDialer(to) +} + +// UDPDialer creates a new NetDialer with constant event fields and default +// connection timeout. +// The fields parameter holds additonal constants to be added to the final event +// structure. +// +// The dialer will update the active events with: +// +// { +// "udp": { +// "port": ..., +// "rtt": { "connect": { "us": ... }} +// } +// } +func UDPDialer(to time.Duration) NetDialer { + return netDialer(to) +} + +func netDialer(timeout time.Duration) NetDialer { + return func(event common.MapStr) (transport.Dialer, error) { + return makeDialer(func(network, address string) (net.Conn, error) { + namespace := "" + + switch network { + case "tcp", "tcp4", "tcp6": + namespace = "tcp" + case "udp", "udp4", "udp6": + namespace = "udp" + default: + return nil, fmt.Errorf("unsupported network type %v", network) + } + + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, err + } + + addresses, err := net.LookupHost(host) + if err != nil { + logp.Warn(`DNS lookup failure "%s": %v`, host, err) + return nil, err + } + + // dial via host IP by randomized iteration of known IPs + dialer := &net.Dialer{Timeout: timeout} + + start := time.Now() + conn, err := transport.DialWith(dialer, network, host, addresses, port) + if err != nil { + return nil, err + } + + end := time.Now() + event.DeepUpdate(common.MapStr{ + namespace: common.MapStr{ + "rtt": common.MapStr{ + "connect": look.RTT(end.Sub(start)), + }, + }, + }) + + return conn, nil + }), nil + } +} diff --git a/heartbeat/monitors/active/dialchain/socks5.go b/heartbeat/monitors/active/dialchain/socks5.go new file mode 100644 index 00000000000..59023a855f9 --- /dev/null +++ b/heartbeat/monitors/active/dialchain/socks5.go @@ -0,0 +1,38 @@ +package dialchain + +import ( + "net" + + "github.com/elastic/beats/heartbeat/look" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/outputs/transport" +) + +// SOCKS5Layer configures a SOCKS5 proxy layer in a DialerChain. +// +// The layer will update the active event with: +// +// { +// "socks5": { +// "rtt": { "connect": { "us": ... }} +// } +// } +func SOCKS5Layer(config *transport.ProxyConfig) Layer { + return func(event common.MapStr, next transport.Dialer) (transport.Dialer, error) { + var timer timer + + dialer, err := transport.ProxyDialer(config, startTimerAfterDial(&timer, next)) + if err != nil { + return nil, err + } + + return afterDial(dialer, func(conn net.Conn) (net.Conn, error) { + // TODO: extract connection parameter from connection object? + // TODO: add proxy url to event? + + timer.stop() + event.Put("socks5.rtt.connect", look.RTT(timer.duration())) + return conn, nil + }), nil + } +} diff --git a/heartbeat/monitors/active/dialchain/tls.go b/heartbeat/monitors/active/dialchain/tls.go new file mode 100644 index 00000000000..e3354b5449c --- /dev/null +++ b/heartbeat/monitors/active/dialchain/tls.go @@ -0,0 +1,42 @@ +package dialchain + +import ( + "net" + "time" + + "github.com/elastic/beats/heartbeat/look" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/outputs/transport" +) + +// TLSLayer configures the TLS layer in a DialerChain. +// +// The layer will update the active event with: +// +// { +// "tls": { +// "rtt": { "handshake": { "us": ... }} +// } +// } +func TLSLayer(cfg *transport.TLSConfig, to time.Duration) Layer { + return func(event common.MapStr, next transport.Dialer) (transport.Dialer, error) { + var timer timer + + // Wrap next dialer so to start the timer when 'next' returns. + // This gets us the timestamp for when the TLS layer will start the handshake. + next = startTimerAfterDial(&timer, next) + + dialer, err := transport.TLSDialer(next, cfg, to) + if err != nil { + return nil, err + } + + return afterDial(dialer, func(conn net.Conn) (net.Conn, error) { + // TODO: extract TLS connection parameters from connection object. + + timer.stop() + event.Put("tls.rtt.handshake", look.RTT(timer.duration())) + return conn, nil + }), nil + } +} diff --git a/heartbeat/monitors/active/dialchain/util.go b/heartbeat/monitors/active/dialchain/util.go new file mode 100644 index 00000000000..b5ea4885dd8 --- /dev/null +++ b/heartbeat/monitors/active/dialchain/util.go @@ -0,0 +1,111 @@ +package dialchain + +import ( + "net" + "time" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/outputs/transport" +) + +type timer struct { + s, e time.Time +} + +// IDLayer creates an empty placeholder layer. +func IDLayer() Layer { + return _idLayer +} + +var _idLayer = Layer(func(event common.MapStr, next transport.Dialer) (transport.Dialer, error) { + return next, nil +}) + +// ConstAddrLayer introduces a network layer always passing a constant address +// to the underlying layer. +func ConstAddrLayer(address string) Layer { + build := constAddr(address) + + return func(event common.MapStr, next transport.Dialer) (transport.Dialer, error) { + return build(next), nil + } +} + +// MakeConstAddrLayer always passes the same address to the original Layer. +// This is usefull if a lookup did return multiple IPs for the same hostname, +// but the IP use to connect shall be fixed. +func MakeConstAddrLayer(addr string, origLayer Layer) Layer { + return withLayerDialer(origLayer, constAddr(addr)) +} + +// MakeConstAddrDialer always passes the same address to the original NetDialer. +// This is usefull if a lookup did return multiple IPs for the same hostname, +// but the IP use to connect shall be fixed. +func MakeConstAddrDialer(addr string, origNet NetDialer) NetDialer { + return withNetDialer(origNet, constAddr(addr)) +} + +func (t *timer) start() { t.s = time.Now() } +func (t *timer) stop() { t.e = time.Now() } +func (t *timer) duration() time.Duration { return t.e.Sub(t.s) } + +// makeDialer aliases transport.DialerFunc +func makeDialer(fn func(network, address string) (net.Conn, error)) transport.Dialer { + return transport.DialerFunc(fn) +} + +// beforeDial will always call fn before executing the underlying dialer. +// The callback must return the original or a new address to be used with +// the dialer. +func beforeDial(dialer transport.Dialer, fn func(string) string) transport.Dialer { + return makeDialer(func(network, address string) (net.Conn, error) { + address = fn(address) + return dialer.Dial(network, address) + }) +} + +// afterDial will run fn after the dialer did successfully return a connection. +func afterDial(dialer transport.Dialer, fn func(net.Conn) (net.Conn, error)) transport.Dialer { + return makeDialer(func(network, address string) (net.Conn, error) { + conn, err := dialer.Dial(network, address) + if err == nil { + conn, err = fn(conn) + } + return conn, err + }) +} + +func startTimerAfterDial(t *timer, dialer transport.Dialer) transport.Dialer { + return afterDial(dialer, func(c net.Conn) (net.Conn, error) { + t.start() + return c, nil + }) +} + +func constAddr(addr string) func(transport.Dialer) transport.Dialer { + return func(dialer transport.Dialer) transport.Dialer { + return beforeDial(dialer, func(_ string) string { + return addr + }) + } +} + +func withNetDialer(layer NetDialer, fn func(transport.Dialer) transport.Dialer) NetDialer { + return func(event common.MapStr) (transport.Dialer, error) { + origDialer, err := layer.build(event) + if err != nil { + return nil, err + } + return fn(origDialer), nil + } +} + +func withLayerDialer(layer Layer, fn func(transport.Dialer) transport.Dialer) Layer { + return func(event common.MapStr, next transport.Dialer) (transport.Dialer, error) { + origDialer, err := layer.build(event, next) + if err != nil { + return nil, err + } + return fn(origDialer), nil + } +} diff --git a/heartbeat/monitors/active/http/simple_transp.go b/heartbeat/monitors/active/http/simple_transp.go index ca0a7d0fe34..4a97f997fd6 100644 --- a/heartbeat/monitors/active/http/simple_transp.go +++ b/heartbeat/monitors/active/http/simple_transp.go @@ -26,6 +26,7 @@ type SimpleTransport struct { DisableCompression bool OnStartWrite func() + OnEndWrite func() OnStartRead func() } @@ -125,6 +126,7 @@ func (t *SimpleTransport) writeRequest(conn net.Conn, req *http.Request) error { if err == nil { err = writer.Flush() } + t.sigEndWrite() return err } @@ -138,6 +140,7 @@ func (t *SimpleTransport) readResponse( if err != nil { return nil, err } + t.sigStartRead() if requestedGzip && resp.Header.Get("Content-Encoding") == gzipEncoding { @@ -160,14 +163,12 @@ func (t *SimpleTransport) readResponse( return resp, nil } -func (t *SimpleTransport) sigStartRead() { - if f := t.OnStartRead; f != nil { - f() - } -} +func (t *SimpleTransport) sigStartRead() { call(t.OnStartRead) } +func (t *SimpleTransport) sigStartWrite() { call(t.OnStartWrite) } +func (t *SimpleTransport) sigEndWrite() { call(t.OnEndWrite) } -func (t *SimpleTransport) sigStartWrite() { - if f := t.OnStartWrite; f != nil { +func call(f func()) { + if f != nil { f() } } diff --git a/heartbeat/monitors/active/http/task.go b/heartbeat/monitors/active/http/task.go index a22d1d8b6bd..f48ae831a70 100644 --- a/heartbeat/monitors/active/http/task.go +++ b/heartbeat/monitors/active/http/task.go @@ -47,19 +47,22 @@ func newHTTPMonitorHostJob( } timeout := config.Timeout - fields := common.MapStr{ - "scheme": request.URL.Scheme, - "host": hostname, - "port": port, - "url": request.URL.String(), - } - return monitors.MakeSimpleJob(jobName, typ, func() (common.MapStr, error) { - event, err := execPing(client, request, body, timeout, validator) - if event == nil { - event = common.MapStr{} - } - event.Update(fields) + settings := monitors.MakeJobSetting(jobName).WithFields(common.MapStr{ + "monitor": common.MapStr{ + "scheme": request.URL.Scheme, + "host": hostname, + }, + "http": common.MapStr{ + "url": request.URL.String(), + }, + "tcp": common.MapStr{ + "port": port, + }, + }) + + return monitors.MakeSimpleJob(settings, func() (common.MapStr, error) { + _, _, event, err := execPing(client, request, body, timeout, validator) return event, err }), nil } @@ -85,11 +88,21 @@ func newHTTPMonitorIPsJob( return nil, err } + settings := monitors.MakeHostJobSettings(jobName, hostname, config.Mode) + settings = settings.WithFields(common.MapStr{ + "monitor": common.MapStr{ + "scheme": req.URL.Scheme, + }, + "http": common.MapStr{ + "url": req.URL.String(), + }, + "tcp": common.MapStr{ + "port": port, + }, + }) + pingFactory := createPingFactory(config, hostname, port, tls, req, body, validator) - if ip := net.ParseIP(hostname); ip != nil { - return monitors.MakeByIPJob(jobName, typ, ip, pingFactory) - } - return monitors.MakeByHostJob(jobName, typ, hostname, config.Mode, pingFactory) + return monitors.MakeByHostJob(settings, pingFactory) } func createPingFactory( @@ -101,53 +114,61 @@ func createPingFactory( body []byte, validator RespCheck, ) func(*net.IPAddr) monitors.TaskRunner { - fields := common.MapStr{ - "scheme": request.URL.Scheme, - "port": port, - "url": request.URL.String(), - } - timeout := config.Timeout isTLS := request.URL.Scheme == "https" checkRedirect := makeCheckRedirect(config.MaxRedirects) - return monitors.MakePingIPFactory(fields, func(ip *net.IPAddr) (common.MapStr, error) { + return monitors.MakePingIPFactory(func(ip *net.IPAddr) (common.MapStr, error) { + event := common.MapStr{} addr := net.JoinHostPort(ip.String(), strconv.Itoa(int(port))) d := &dialchain.DialerChain{ - Net: dialchain.ConstAddrDialer("tcp_connect_rtt", addr, timeout), + Net: dialchain.MakeConstAddrDialer(addr, dialchain.TCPDialer(timeout)), } + + // TODO: add socks5 proxy? + if isTLS { - d.AddLayer(dialchain.TLSLayer("tls_handshake_rtt", tls, timeout)) + d.AddLayer(dialchain.TLSLayer(tls, timeout)) } - measures := common.MapStr{} - dialer, err := d.BuildWithMeasures(measures) + dialer, err := d.Build(event) if err != nil { return nil, err } - var httpStart, httpEnd time.Time + var ( + writeStart, readStart, writeEnd time.Time + ) client := &http.Client{ CheckRedirect: checkRedirect, Timeout: timeout, Transport: &SimpleTransport{ Dialer: dialer, - OnStartWrite: func() { httpStart = time.Now() }, - OnStartRead: func() { httpEnd = time.Now() }, + OnStartWrite: func() { writeStart = time.Now() }, + OnEndWrite: func() { writeEnd = time.Now() }, + OnStartRead: func() { readStart = time.Now() }, }, } - event, err := execPing(client, request, body, timeout, validator) - if event == nil { - event = measures - } else { - event.Update(measures) + _, end, result, err := execPing(client, request, body, timeout, validator) + event.DeepUpdate(result) + + if !readStart.IsZero() { + event.DeepUpdate(common.MapStr{ + "http": common.MapStr{ + "rtt": common.MapStr{ + "write_request": look.RTT(writeEnd.Sub(writeStart)), + "response_header": look.RTT(readStart.Sub(writeStart)), + }, + }, + }) } - - if !httpEnd.IsZero() { - event["http_rtt"] = look.RTT(httpEnd.Sub(httpStart)) + if !writeStart.IsZero() { + event.Put("http.rtt.validate", look.RTT(end.Sub(writeStart))) + event.Put("http.rtt.content", look.RTT(end.Sub(readStart))) } + return event, err }) } @@ -180,7 +201,7 @@ func execPing( body []byte, timeout time.Duration, validator func(*http.Response) error, -) (common.MapStr, reason.Reason) { +) (time.Time, time.Time, common.MapStr, reason.Reason) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -194,22 +215,27 @@ func execPing( resp, err := client.Do(req) end := time.Now() if err != nil { - return nil, reason.IOFailed(err) + return start, end, nil, reason.IOFailed(err) } defer resp.Body.Close() - if err := validator(resp); err != nil { - return nil, reason.ValidateFailed(err) - } + err = validator(resp) + end = time.Now() rtt := end.Sub(start) - event := common.MapStr{ + event := common.MapStr{"http": common.MapStr{ "response": common.MapStr{ "status": resp.StatusCode, }, - "rtt": look.RTT(rtt), + "rtt": common.MapStr{ + "total": look.RTT(rtt), + }, + }} + + if err != nil { + return start, end, event, reason.ValidateFailed(err) } - return event, nil + return start, end, event, nil } func splitHostnamePort(requ *http.Request) (string, uint16, error) { diff --git a/heartbeat/monitors/active/icmp/icmp.go b/heartbeat/monitors/active/icmp/icmp.go index b06d3c2ba42..2b85f6805f6 100644 --- a/heartbeat/monitors/active/icmp/icmp.go +++ b/heartbeat/monitors/active/icmp/icmp.go @@ -61,23 +61,17 @@ func create( return nil, err } - typ := config.Name network := config.Mode.Network() - pingFactory := monitors.MakePingIPFactory(nil, createPingIPFactory(&config)) + pingFactory := monitors.MakePingIPFactory(createPingIPFactory(&config)) for _, host := range config.Hosts { - ip := net.ParseIP(host) - if ip != nil { - name := fmt.Sprintf("icmp-ip@%v", ip.String()) - err := addJob(monitors.MakeByIPJob(name, typ, ip, pingFactory)) - if err != nil { - return nil, err - } - continue + jobName := fmt.Sprintf("icmp-%v-host-%v@%v", config.Name, network, host) + if ip := net.ParseIP(host); ip != nil { + jobName = fmt.Sprintf("icmp-%v-ip@%v", config.Name, ip.String()) } - name := fmt.Sprintf("%v-host-%v@%v", config.Name, network, host) - err := addJob(monitors.MakeByHostJob(name, typ, host, config.Mode, pingFactory)) + settings := monitors.MakeHostJobSettings(jobName, host, config.Mode) + err := addJob(monitors.MakeByHostJob(settings, pingFactory)) if err != nil { return nil, err } @@ -88,13 +82,14 @@ func create( func createPingIPFactory(config *Config) func(*net.IPAddr) (common.MapStr, error) { return func(ip *net.IPAddr) (common.MapStr, error) { - rtt, _, err := loop.ping(ip, config.Timeout, config.Wait) - if err != nil { - return nil, err + rtt, n, err := loop.ping(ip, config.Timeout, config.Wait) + + fields := common.MapStr{"requests": n} + if err == nil { + fields["rtt"] = look.RTT(rtt) } - return common.MapStr{ - "icmp_rtt": look.RTT(rtt), - }, nil + event := common.MapStr{"icmp": fields} + return event, err } } diff --git a/heartbeat/monitors/active/icmp/loop.go b/heartbeat/monitors/active/icmp/loop.go index e1ef007c961..8254260c7fe 100644 --- a/heartbeat/monitors/active/icmp/loop.go +++ b/heartbeat/monitors/active/icmp/loop.go @@ -253,7 +253,7 @@ func (l *icmpLoop) ping( } if !success { - return 0, 0, timeoutError{} + return 0, requests, timeoutError{} } return rtt, requests, nil } diff --git a/heartbeat/monitors/active/tcp/task.go b/heartbeat/monitors/active/tcp/task.go index f8f6ca41b3e..e600061af3b 100644 --- a/heartbeat/monitors/active/tcp/task.go +++ b/heartbeat/monitors/active/tcp/task.go @@ -31,19 +31,25 @@ func newTCPMonitorHostJob( return nil, err } - return monitors.MakeSimpleJob(jobName, typ, func() (common.MapStr, error) { - event := common.MapStr{ - "scheme": scheme, - "port": port, + settings := monitors.MakeJobSetting(jobName).WithFields(common.MapStr{ + "monitor": common.MapStr{ "host": host, - } - dialer, err := taskDialer.BuildWithMeasures(event) + "scheme": scheme, + }, + "tcp": common.MapStr{ + "port": port, + }, + }) + + return monitors.MakeSimpleJob(settings, func() (common.MapStr, error) { + event := common.MapStr{} + dialer, err := taskDialer.Build(event) if err != nil { return event, err } results, err := pingHost(dialer, pingAddr, timeout, validator) - event.Update(results) + event.DeepUpdate(results) return event, err }), nil } @@ -64,14 +70,16 @@ func newTCPMonitorIPsJob( return nil, err } - pingFactory := createPingFactory(dialerFactory, addr, timeout, validator) - if ip := net.ParseIP(addr.Host); ip != nil { - debugf("Make TCP by IP job: %v:%v", ip, addr.Ports) - return monitors.MakeByIPJob(jobName, typ, ip, pingFactory) - } + settings := monitors.MakeHostJobSettings(jobName, addr.Host, config.Mode) + settings = settings.WithFields(common.MapStr{ + "monitor": common.MapStr{ + "scheme": addr.Scheme, + }, + }) - debugf("Make TCP by Host job: %v:%v (mode=%#v)", addr.Host, addr.Ports, config.Mode) - return monitors.MakeByHostJob(jobName, typ, addr.Host, config.Mode, pingFactory) + debugf("Make TCP job: %v:%v", addr.Host, addr.Ports) + pingFactory := createPingFactory(dialerFactory, addr, timeout, validator) + return monitors.MakeByHostJob(settings, pingFactory) } func createPingFactory( @@ -80,21 +88,25 @@ func createPingFactory( timeout time.Duration, validator ConnCheck, ) func(*net.IPAddr) monitors.TaskRunner { - fields := common.MapStr{"scheme": addr.Scheme} - - return monitors.MakePingAllIPPortFactory(fields, addr.Ports, + return monitors.MakePingAllIPPortFactory(addr.Ports, func(ip *net.IPAddr, port uint16) (common.MapStr, error) { - host := net.JoinHostPort(ip.String(), strconv.Itoa(int(port))) + ipStr := ip.String() + host := net.JoinHostPort(ipStr, strconv.Itoa(int(port))) pingAddr := net.JoinHostPort(addr.Host, strconv.Itoa(int(port))) - event := common.MapStr{} - dialer, err := makeDialerChain(host).BuildWithMeasures(event) + event := common.MapStr{ + "tcp": common.MapStr{ + "port": port, + }, + } + + dialer, err := makeDialerChain(host).Build(event) if err != nil { return event, err } results, err := pingHost(dialer, pingAddr, timeout, validator) - event.Update(results) + event.DeepUpdate(results) return event, err }) } @@ -133,7 +145,11 @@ func pingHost( end := time.Now() event := common.MapStr{ - "validate_rtt": look.RTT(end.Sub(validateStart)), + "tcp": common.MapStr{ + "rtt": common.MapStr{ + "validate": look.RTT(end.Sub(validateStart)), + }, + }, } if err != nil { event["error"] = reason.FailValidate(err) @@ -169,13 +185,20 @@ func buildDialerChain( config *Config, ) (*dialchain.DialerChain, error) { d := &dialchain.DialerChain{ - Net: dialchain.TCPDialer("tcp_connect_rtt", config.Timeout), + Net: dialchain.TCPDialer(config.Timeout), } - if config.Socks5.URL != "" { - d.AddLayer(dialchain.SOCKS5Layer("socks5_connect_rtt", &config.Socks5)) + + withProxy := config.Socks5.URL != "" + if withProxy { + d.AddLayer(dialchain.SOCKS5Layer(&config.Socks5)) } + + // insert empty placeholder, so address can be replaced in dialer chain + // by replacing this placeholder dialer + d.AddLayer(dialchain.IDLayer()) + if isTLSAddr(scheme) { - d.AddLayer(dialchain.TLSLayer("tls_handshake_rtt", tls, config.Timeout)) + d.AddLayer(dialchain.TLSLayer(tls, config.Timeout)) } if err := d.TestBuild(); err != nil { @@ -195,16 +218,15 @@ func buildHostDialerChainFactory( } withProxy := config.Socks5.URL != "" - return func(addr string) *dialchain.DialerChain { - if withProxy { - d := template.Clone() - d.Layers[0] = dialchain.ConstAddrLayer(addr, d.Layers[0]) - return d - } + addrIndex := 0 + if withProxy { + addrIndex = 1 + } - return &dialchain.DialerChain{ - Net: dialchain.ConstAddrDialer("tcp_connect_rtt", addr, config.Timeout), - Layers: template.Layers, - } + return func(addr string) *dialchain.DialerChain { + // replace IDLayer placeholder in template with ConstAddrLayer + d := template.Clone() + d.Layers[addrIndex] = dialchain.ConstAddrLayer(addr) + return d }, nil } diff --git a/heartbeat/monitors/util.go b/heartbeat/monitors/util.go index 9045d6a6b71..0b4184da3ae 100644 --- a/heartbeat/monitors/util.go +++ b/heartbeat/monitors/util.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "net" - "strconv" "time" "github.com/elastic/beats/libbeat/common" @@ -13,7 +12,7 @@ import ( ) type funcJob struct { - name, typ string + settings JobSettings funcTask } @@ -21,12 +20,31 @@ type funcTask struct { run func() (common.MapStr, []TaskRunner, error) } +// IPSettings provides common configuration settings for IP resolution and ping +// mode. type IPSettings struct { IPv4 bool `config:"ipv4"` IPv6 bool `config:"ipv6"` Mode PingMode `config:"mode"` } +// JobSettings configures a Job name and global fields to be added to every +// event. +type JobSettings struct { + Name string + Fields common.MapStr +} + +// HostJobSettings configures a Job including Host lookups and global fields to be added +// to every event. +type HostJobSettings struct { + Name string + Host string + IP IPSettings + Fields common.MapStr +} + +// PingMode enumeration for configuring `any` or `all` IPs pinging. type PingMode uint8 const ( @@ -35,12 +53,16 @@ const ( PingAll ) +// DefaultIPSettings provides an instance of default IPSettings to be copied +// when unpacking settings from a common.Config object. var DefaultIPSettings = IPSettings{ IPv4: true, IPv6: true, Mode: PingAny, } +// Network determines the Network type used for IP name resolution, based on the +// provided settings. func (s IPSettings) Network() string { switch { case s.IPv4 && !s.IPv6: @@ -53,23 +75,40 @@ func (s IPSettings) Network() string { return "" } -func MakeSimpleJob(name, typ string, f func() (common.MapStr, error)) Job { - return MakeJob(name, typ, func() (common.MapStr, []TaskRunner, error) { +// MakeSimpleJob creates a new Job from a callback function. The callback should +// return an valid event and can not create any sub-tasks to be executed after +// completion. +func MakeSimpleJob(settings JobSettings, f func() (common.MapStr, error)) Job { + return MakeJob(settings, func() (common.MapStr, []TaskRunner, error) { event, err := f() return event, nil, err }) } -func MakeJob(name, typ string, f func() (common.MapStr, []TaskRunner, error)) Job { - return &funcJob{name, typ, funcTask{func() (common.MapStr, []TaskRunner, error) { - return annotated(time.Now(), typ, f).Run() +// MakeJob create a new Job from a callback function. The callback can +// optionally return an event to be published and a set of derived sub-tasks to be +// scheduled. The sub-tasks will be run only once and removed from the scheduler +// after completion. +func MakeJob(settings JobSettings, f func() (common.MapStr, []TaskRunner, error)) Job { + settings.AddFields(common.MapStr{ + "monitor": common.MapStr{ + "id": settings.Name, + }, + }) + + return &funcJob{settings, funcTask{func() (common.MapStr, []TaskRunner, error) { + return annotated(settings, time.Now(), f).Run() }}} } +// MakeCont wraps a function into an executable TaskRunner. The task being generated +// can optionally return an event and/or sub-tasks. func MakeCont(f func() (common.MapStr, []TaskRunner, error)) TaskRunner { return funcTask{f} } +// MakeSimpleCont wraps a function into an executable TaskRunner. The task bein generated +// should return an event to be reported. func MakeSimpleCont(f func() (common.MapStr, error)) TaskRunner { return MakeCont(func() (common.MapStr, []TaskRunner, error) { event, err := f() @@ -77,44 +116,33 @@ func MakeSimpleCont(f func() (common.MapStr, error)) TaskRunner { }) } +// MakePingIPFactory creates a factory for building a Task from a new IP address. func MakePingIPFactory( - fields common.MapStr, f func(*net.IPAddr) (common.MapStr, error), ) func(*net.IPAddr) TaskRunner { return func(ip *net.IPAddr) TaskRunner { - r := MakeSimpleCont(func() (common.MapStr, error) { return f(ip) }) - if len(fields) > 0 { - r = WithFields(fields, r) - } - return r + return MakeSimpleCont(func() (common.MapStr, error) { return f(ip) }) } } var emptyTask = MakeSimpleCont(func() (common.MapStr, error) { return nil, nil }) +// MakePingAllIPFactory wraps a function for building a recursive Task Runner from function callbacks. func MakePingAllIPFactory( - fields common.MapStr, f func(*net.IPAddr) []func() (common.MapStr, error), ) func(*net.IPAddr) TaskRunner { - makeTask := func(f func() (common.MapStr, error)) TaskRunner { - if len(fields) > 0 { - return WithFields(fields, MakeSimpleCont(f)) - } - return MakeSimpleCont(f) - } - return func(ip *net.IPAddr) TaskRunner { cont := f(ip) switch len(cont) { case 0: return emptyTask case 1: - return makeTask(cont[0]) + return MakeSimpleCont(cont[0]) } tasks := make([]TaskRunner, len(cont)) for i, c := range cont { - tasks[i] = makeTask(c) + tasks[i] = MakeSimpleCont(c) } return MakeCont(func() (common.MapStr, []TaskRunner, error) { return nil, tasks, nil @@ -122,39 +150,38 @@ func MakePingAllIPFactory( } } +// MakePingAllIPPortFactory builds a set of TaskRunner supporting a set of +// IP/port-pairs. func MakePingAllIPPortFactory( - fields common.MapStr, ports []uint16, f func(*net.IPAddr, uint16) (common.MapStr, error), ) func(*net.IPAddr) TaskRunner { if len(ports) == 1 { port := ports[0] - fields := fields.Clone() - fields["port"] = strconv.Itoa(int(port)) - return MakePingIPFactory(fields, func(ip *net.IPAddr) (common.MapStr, error) { + return MakePingIPFactory(func(ip *net.IPAddr) (common.MapStr, error) { return f(ip, port) }) } - return MakePingAllIPFactory(fields, func(ip *net.IPAddr) []func() (common.MapStr, error) { + return MakePingAllIPFactory(func(ip *net.IPAddr) []func() (common.MapStr, error) { funcs := make([]func() (common.MapStr, error), len(ports)) for i := range ports { port := ports[i] funcs[i] = func() (common.MapStr, error) { - event, err := f(ip, port) - if event == nil { - event = common.MapStr{} - } - event["port"] = strconv.Itoa(int(port)) - return event, err + return f(ip, port) } } return funcs }) } +// MakeByIPJob builds a new Job based on already known IP. Similar to +// MakeByHostJob, the pingFactory will be used to build the tasks run by the job. +// +// A pingFactory instance is normally build with MakePingIPFactory, +// MakePingAllIPFactory or MakePingAllIPPortFactory. func MakeByIPJob( - name, typ string, + settings JobSettings, ip net.IP, pingFactory func(ip *net.IPAddr) TaskRunner, ) (Job, error) { @@ -165,86 +192,144 @@ func MakeByIPJob( return nil, err } - fields := common.MapStr{"ip": addr.String()} - return MakeJob(name, typ, WithFields(fields, pingFactory(addr)).Run), nil + fields := common.MapStr{ + "monitor": common.MapStr{"ip": addr.String()}, + } + return MakeJob(settings, WithFields(fields, pingFactory(addr)).Run), nil } +// MakeByHostJob creates a new Job including host lookup. The pingFactory will be used to +// build one or multiple Tasks after name lookup according to settings. +// +// A pingFactory instance is normally build with MakePingIPFactory, +// MakePingAllIPFactory or MakePingAllIPPortFactory. func MakeByHostJob( - name, typ string, - host string, - settings IPSettings, + settings HostJobSettings, pingFactory func(ip *net.IPAddr) TaskRunner, ) (Job, error) { - network := settings.Network() + host := settings.Host + + if ip := net.ParseIP(host); ip != nil { + return MakeByIPJob(settings.jobSettings(), ip, pingFactory) + } + + network := settings.IP.Network() if network == "" { return nil, errors.New("pinging hosts requires ipv4 or ipv6 mode enabled") } - mode := settings.Mode + mode := settings.IP.Mode + + settings.AddFields(common.MapStr{ + "monitor": common.MapStr{ + "host": host, + }, + }) + if mode == PingAny { - return MakeJob(name, typ, func() (common.MapStr, []TaskRunner, error) { - event := common.MapStr{"host": host} + return makeByHostAnyIPJob(settings, host, pingFactory), nil + } + return makeByHostAllIPJob(settings, host, pingFactory), nil +} - dnsStart := time.Now() - ip, err := net.ResolveIPAddr(network, host) - if err != nil { - return event, nil, err - } +func makeByHostAnyIPJob( + settings HostJobSettings, + host string, + pingFactory func(ip *net.IPAddr) TaskRunner, +) Job { + network := settings.IP.Network() - dnsEnd := time.Now() - dnsRTT := dnsEnd.Sub(dnsStart) - event["resolve_rtt"] = look.RTT(dnsRTT) - event["ip"] = ip.String() + return MakeJob(settings.jobSettings(), func() (common.MapStr, []TaskRunner, error) { + resolveStart := time.Now() + ip, err := net.ResolveIPAddr(network, host) + if err != nil { + return resolveErr(host, err) + } - return WithFields(event, pingFactory(ip)).Run() - }), nil - } + resolveEnd := time.Now() + resolveRTT := resolveEnd.Sub(resolveStart) + + event := resolveIPEvent(host, ip.String(), resolveRTT) + return WithFields(event, pingFactory(ip)).Run() + }) +} +func makeByHostAllIPJob( + settings HostJobSettings, + host string, + pingFactory func(ip *net.IPAddr) TaskRunner, +) Job { + network := settings.IP.Network() filter := makeIPFilter(network) - return MakeJob(name, typ, func() (common.MapStr, []TaskRunner, error) { - event := common.MapStr{"host": host} + return MakeJob(settings.jobSettings(), func() (common.MapStr, []TaskRunner, error) { // TODO: check for better DNS IP lookup support: // - The net.LookupIP drops ipv6 zone index // - dnsStart := time.Now() + resolveStart := time.Now() ips, err := net.LookupIP(host) if err != nil { - return event, nil, err + return resolveErr(host, err) } - dnsEnd := time.Now() - dnsRTT := dnsEnd.Sub(dnsStart) + resolveEnd := time.Now() + resolveRTT := resolveEnd.Sub(resolveStart) - event["resolve_rtt"] = look.RTT(dnsRTT) if filter != nil { ips = filterIPs(ips, filter) } if len(ips) == 0 { err := fmt.Errorf("no %v address resolvable for host %v", network, host) - return event, nil, err + return resolveErr(host, err) } // create ip ping tasks cont := make([]TaskRunner, len(ips)) for i, ip := range ips { addr := &net.IPAddr{IP: ip} - fields := event.Clone() - fields["ip"] = ip.String() - cont[i] = WithFields(fields, pingFactory(addr)) + event := resolveIPEvent(host, ip.String(), resolveRTT) + cont[i] = WithFields(event, pingFactory(addr)) } return nil, cont, nil - }), nil + }) +} + +func resolveIPEvent(host, ip string, rtt time.Duration) common.MapStr { + return common.MapStr{ + "monitor": common.MapStr{ + "host": host, + "ip": ip, + }, + "resolve": common.MapStr{ + "host": host, + "ip": ip, + "rtt": look.RTT(rtt), + }, + } } +func resolveErr(host string, err error) (common.MapStr, []TaskRunner, error) { + event := common.MapStr{ + "monitor": common.MapStr{ + "host": host, + }, + "resolve": common.MapStr{ + "host": host, + }, + } + return event, nil, err +} + +// WithFields wraps a TaskRunner, updating all events returned with the set of +// fields configured. func WithFields(fields common.MapStr, r TaskRunner) TaskRunner { return MakeCont(func() (common.MapStr, []TaskRunner, error) { event, cont, err := r.Run() if event == nil { event = common.MapStr{} } - event.Update(fields) + event.DeepUpdate(fields) for i := range cont { cont[i] = WithFields(fields, cont[i]) @@ -253,9 +338,11 @@ func WithFields(fields common.MapStr, r TaskRunner) TaskRunner { }) } -func WithDuration(name string, r TaskRunner) TaskRunner { +// WithDuration wraps a TaskRunner, measuring the duration between creation and +// finish of the actual task and sub-tasks. +func WithDuration(field string, r TaskRunner) TaskRunner { return MakeCont(func() (common.MapStr, []TaskRunner, error) { - return withStart(name, time.Now(), r).Run() + return withStart(field, time.Now(), r).Run() }) } @@ -263,7 +350,7 @@ func withStart(field string, start time.Time, r TaskRunner) TaskRunner { return MakeCont(func() (common.MapStr, []TaskRunner, error) { event, cont, err := r.Run() if event != nil { - event[field] = look.RTT(time.Now().Sub(start)) + event.Put(field, look.RTT(time.Since(start))) } for i := range cont { @@ -273,14 +360,16 @@ func withStart(field string, start time.Time, r TaskRunner) TaskRunner { }) } -func (f *funcJob) Name() string { return f.name } +func (f *funcJob) Name() string { return f.settings.Name } func (f funcTask) Run() (common.MapStr, []TaskRunner, error) { return f.run() } -func (f funcTask) annotated(start time.Time, typ string) TaskRunner { - return annotated(start, typ, f.run) +func (f funcTask) annotated(settings JobSettings, start time.Time) TaskRunner { + return annotated(settings, start, f.run) } +// Unpack sets PingMode from a constant string. Unpack will be called by common.Unpack when +// unpacking into an IPSettings type. func (p *PingMode) Unpack(s string) error { switch s { case "all": @@ -293,7 +382,11 @@ func (p *PingMode) Unpack(s string) error { return nil } -func annotated(start time.Time, typ string, fn func() (common.MapStr, []TaskRunner, error)) TaskRunner { +func annotated( + settings JobSettings, + start time.Time, + fn func() (common.MapStr, []TaskRunner, error), +) TaskRunner { return MakeCont(func() (common.MapStr, []TaskRunner, error) { event, cont, err := fn() if err != nil { @@ -304,19 +397,24 @@ func annotated(start time.Time, typ string, fn func() (common.MapStr, []TaskRunn } if event != nil { - event.Update(common.MapStr{ + status := look.Status(err) + event.DeepUpdate(common.MapStr{ "@timestamp": look.Timestamp(start), - "duration": look.RTT(time.Now().Sub(start)), - "type": typ, - "up": err == nil, + "monitor": common.MapStr{ + "duration": look.RTT(time.Since(start)), + "status": status, + }, }) + if fields := settings.Fields; fields != nil { + event.DeepUpdate(fields) + } } for i := range cont { if fcont, ok := cont[i].(funcTask); ok { - cont[i] = fcont.annotated(start, typ) + cont[i] = fcont.annotated(settings, start) } else { - cont[i] = annotated(start, typ, cont[i].Run) + cont[i] = annotated(settings, start, cont[i].Run) } } return event, cont, nil @@ -342,3 +440,55 @@ func filterIPs(ips []net.IP, filt func(net.IP) bool) []net.IP { } return out } + +// MakeJobSetting creates a new JobSettings structure without any global event fields. +func MakeJobSetting(name string) JobSettings { + return JobSettings{Name: name} +} + +// WithFields adds new event fields to a Job. Existing fields will be +// overwritten. +// The fields map will be updated (no copy). +func (s JobSettings) WithFields(m common.MapStr) JobSettings { + s.AddFields(m) + return s +} + +// AddFields adds new event fields to a Job. Existing fields will be +// overwritten. +func (s *JobSettings) AddFields(m common.MapStr) { addFields(&s.Fields, m) } + +// MakeHostJobSettings creates a new HostJobSettings structure without any global +// event fields. +func MakeHostJobSettings(name, host string, ip IPSettings) HostJobSettings { + return HostJobSettings{Name: name, Host: host, IP: ip} +} + +// WithFields adds new event fields to a Job. Existing fields will be +// overwritten. +// The fields map will be updated (no copy). +func (s HostJobSettings) WithFields(m common.MapStr) HostJobSettings { + s.AddFields(m) + return s +} + +// AddFields adds new event fields to a Job. Existing fields will be +// overwritten. +func (s *HostJobSettings) AddFields(m common.MapStr) { addFields(&s.Fields, m) } + +func addFields(to *common.MapStr, m common.MapStr) { + if m == nil { + return + } + + fields := *to + if fields == nil { + fields = common.MapStr{} + *to = fields + } + fields.DeepUpdate(m) +} + +func (s *HostJobSettings) jobSettings() JobSettings { + return JobSettings{Name: s.Name, Fields: s.Fields} +} diff --git a/libbeat/outputs/transport/proxy.go b/libbeat/outputs/transport/proxy.go index 4fe4a8847b9..6f7b1fdd95a 100644 --- a/libbeat/outputs/transport/proxy.go +++ b/libbeat/outputs/transport/proxy.go @@ -76,6 +76,6 @@ func ProxyDialer(config *ProxyConfig, forward Dialer) (Dialer, error) { if err != nil { return nil, err } - return dialWith(dialer, network, host, addresses, port) + return DialWith(dialer, network, host, addresses, port) }), nil } diff --git a/libbeat/outputs/transport/tcp.go b/libbeat/outputs/transport/tcp.go index 857e2d00225..78b76634428 100644 --- a/libbeat/outputs/transport/tcp.go +++ b/libbeat/outputs/transport/tcp.go @@ -29,6 +29,6 @@ func NetDialer(timeout time.Duration) Dialer { // dial via host IP by randomized iteration of known IPs dialer := &net.Dialer{Timeout: timeout} - return dialWith(dialer, network, host, addresses, port) + return DialWith(dialer, network, host, addresses, port) }) } diff --git a/libbeat/outputs/transport/tls.go b/libbeat/outputs/transport/tls.go index fff921ccb74..ebb0f88985e 100644 --- a/libbeat/outputs/transport/tls.go +++ b/libbeat/outputs/transport/tls.go @@ -13,6 +13,7 @@ import ( ) type TLSConfig struct { + // List of allowed SSL/TLS protocol versions. Connections might be dropped // after handshake succeeded, if TLS version in use is not listed. Versions []TLSVersion diff --git a/libbeat/outputs/transport/util.go b/libbeat/outputs/transport/util.go index 59e746511b2..41e4be6d8ef 100644 --- a/libbeat/outputs/transport/util.go +++ b/libbeat/outputs/transport/util.go @@ -20,7 +20,10 @@ func fullAddress(host string, defaultPort int) string { return fmt.Sprintf("%v:%v", host, defaultPort) } -func dialWith( +// DialWith randomly dials one of a number of addresses with a given dialer. +// +// Use this to select and dial one IP being known for one host name. +func DialWith( dialer Dialer, network, host string, addresses []string, From b84568740ff667a2b75fbcf436fd79c6b8c2f5f4 Mon Sep 17 00:00:00 2001 From: Monica Sarbu Date: Fri, 28 Apr 2017 17:07:03 +0200 Subject: [PATCH 30/40] Add example dashboards using the time series visual builder (#4115) --- .../79ffd6e0-faa0-11e6-947f-177f697178b8.json | 20 ++++++++++++++++++ .../CPU-slash-Memory-per-container.json | 6 +++--- .../kibana/dashboard/Metricbeat-cpu.json | 4 ++-- .../Metricbeat-filesystem-per-Host.json | 4 ++-- .../dashboard/Metricbeat-filesystem.json | 4 ++-- .../kibana/dashboard/Metricbeat-memory.json | 4 ++-- .../kibana/dashboard/Metricbeat-network.json | 4 ++-- .../dashboard/Metricbeat-processes.json | 4 ++-- .../dashboard/Metricbeat-system-overview.json | 4 ++-- .../d1f1f9e0-1b1c-11e7-b09e-037021c4f8df.json | 21 +++++++++++++++++++ .../089b85d0-1b16-11e7-b09e-037021c4f8df.json | 10 +++++++++ .../1aae9140-1b93-11e7-8ada-3df93aab833e.json | 10 +++++++++ .../217025e0-2a69-11e7-99f0-399f2a11b723.json | 10 +++++++++ .../26732e20-1b91-11e7-bec4-a5e9ec5cab8b.json | 10 +++++++++ .../2e224660-1b19-11e7-b09e-037021c4f8df.json | 10 +++++++++ .../34f97ee0-1b96-11e7-8ada-3df93aab833e.json | 10 +++++++++ .../4d546850-1b15-11e7-b09e-037021c4f8df.json | 10 +++++++++ .../4e4bb1e0-1b1b-11e7-b09e-037021c4f8df.json | 10 +++++++++ .../522ee670-1b92-11e7-bec4-a5e9ec5cab8b.json | 10 +++++++++ .../6b7b9a40-faa1-11e6-86b1-cd7735ff7e23.json | 10 +++++++++ .../83e12df0-1b91-11e7-bec4-a5e9ec5cab8b.json | 10 +++++++++ .../855899e0-1b1c-11e7-b09e-037021c4f8df.json | 10 +++++++++ .../95490a10-1e23-11e7-958f-490b8dcb96d8.json | 11 ++++++++++ .../visualization/Container-CPU-usage.json | 10 ++++----- .../Disk-space-distribution.json | 2 +- .../visualization/System-Navigation.json | 4 ++-- .../aa984970-1e0b-11e7-852e-cdcfcfdffddd.json | 10 +++++++++ .../ab2d1e90-1b1a-11e7-b09e-037021c4f8df.json | 10 +++++++++ .../bfa5e400-1b16-11e7-b09e-037021c4f8df.json | 10 +++++++++ .../d3166e80-1b91-11e7-bec4-a5e9ec5cab8b.json | 10 +++++++++ .../e0f001c0-1b18-11e7-b09e-037021c4f8df.json | 10 +++++++++ .../fe064790-1b1f-11e7-bec4-a5e9ec5cab8b.json | 10 +++++++++ 32 files changed, 257 insertions(+), 25 deletions(-) create mode 100644 metricbeat/module/system/_meta/kibana/dashboard/79ffd6e0-faa0-11e6-947f-177f697178b8.json create mode 100644 metricbeat/module/system/_meta/kibana/dashboard/d1f1f9e0-1b1c-11e7-b09e-037021c4f8df.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/089b85d0-1b16-11e7-b09e-037021c4f8df.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/1aae9140-1b93-11e7-8ada-3df93aab833e.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/217025e0-2a69-11e7-99f0-399f2a11b723.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/26732e20-1b91-11e7-bec4-a5e9ec5cab8b.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/2e224660-1b19-11e7-b09e-037021c4f8df.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/34f97ee0-1b96-11e7-8ada-3df93aab833e.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/4d546850-1b15-11e7-b09e-037021c4f8df.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/4e4bb1e0-1b1b-11e7-b09e-037021c4f8df.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/522ee670-1b92-11e7-bec4-a5e9ec5cab8b.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/6b7b9a40-faa1-11e6-86b1-cd7735ff7e23.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/83e12df0-1b91-11e7-bec4-a5e9ec5cab8b.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/855899e0-1b1c-11e7-b09e-037021c4f8df.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/95490a10-1e23-11e7-958f-490b8dcb96d8.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/aa984970-1e0b-11e7-852e-cdcfcfdffddd.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/ab2d1e90-1b1a-11e7-b09e-037021c4f8df.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/bfa5e400-1b16-11e7-b09e-037021c4f8df.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/d3166e80-1b91-11e7-bec4-a5e9ec5cab8b.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/e0f001c0-1b18-11e7-b09e-037021c4f8df.json create mode 100644 metricbeat/module/system/_meta/kibana/visualization/fe064790-1b1f-11e7-bec4-a5e9ec5cab8b.json diff --git a/metricbeat/module/system/_meta/kibana/dashboard/79ffd6e0-faa0-11e6-947f-177f697178b8.json b/metricbeat/module/system/_meta/kibana/dashboard/79ffd6e0-faa0-11e6-947f-177f697178b8.json new file mode 100644 index 00000000000..c1cedaffa6f --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/dashboard/79ffd6e0-faa0-11e6-947f-177f697178b8.json @@ -0,0 +1,20 @@ +{ + "hits": 0, + "timeFrom": "now-15m", + "timeRestore": true, + "description": "", + "title": "Metricbeat host overview", + "uiStateJSON": "{\"P-18\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", + "panelsJSON": "[{\"col\":1,\"id\":\"6b7b9a40-faa1-11e6-86b1-cd7735ff7e23\",\"panelIndex\":1,\"row\":13,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"4d546850-1b15-11e7-b09e-037021c4f8df\",\"panelIndex\":2,\"row\":4,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"089b85d0-1b16-11e7-b09e-037021c4f8df\",\"panelIndex\":3,\"row\":10,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"bfa5e400-1b16-11e7-b09e-037021c4f8df\",\"panelIndex\":4,\"row\":10,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"e0f001c0-1b18-11e7-b09e-037021c4f8df\",\"panelIndex\":5,\"row\":16,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"2e224660-1b19-11e7-b09e-037021c4f8df\",\"panelIndex\":6,\"row\":16,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"ab2d1e90-1b1a-11e7-b09e-037021c4f8df\",\"panelIndex\":7,\"row\":7,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"4e4bb1e0-1b1b-11e7-b09e-037021c4f8df\",\"panelIndex\":8,\"row\":7,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"26732e20-1b91-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":9,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":5,\"id\":\"83e12df0-1b91-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":10,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":9,\"id\":\"d3166e80-1b91-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":11,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"522ee670-1b92-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":12,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":3,\"id\":\"1aae9140-1b93-11e7-8ada-3df93aab833e\",\"panelIndex\":13,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":11,\"id\":\"34f97ee0-1b96-11e7-8ada-3df93aab833e\",\"panelIndex\":14,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":10,\"id\":\"aa984970-1e0b-11e7-852e-cdcfcfdffddd\",\"panelIndex\":15,\"row\":1,\"size_x\":3,\"size_y\":1,\"type\":\"visualization\"},{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":16,\"row\":1,\"size_x\":9,\"size_y\":1,\"type\":\"visualization\"},{\"col\":7,\"id\":\"In-vs-Out-Network-Bytes\",\"panelIndex\":17,\"row\":13,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"217025e0-2a69-11e7-99f0-399f2a11b723\",\"panelIndex\":18,\"row\":4,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"}]", + "timeTo": "now", + "optionsJSON": "{\"darkTheme\":false}", + "version": 1, + "refreshInterval": { + "pause": false, + "display": "Off", + "value": 0 + }, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/dashboard/CPU-slash-Memory-per-container.json b/metricbeat/module/system/_meta/kibana/dashboard/CPU-slash-Memory-per-container.json index 7171f257caa..cebea2992b9 100644 --- a/metricbeat/module/system/_meta/kibana/dashboard/CPU-slash-Memory-per-container.json +++ b/metricbeat/module/system/_meta/kibana/dashboard/CPU-slash-Memory-per-container.json @@ -2,12 +2,12 @@ "hits": 0, "timeRestore": false, "description": "", - "title": "CPU/Memory per container", + "title": "Metricbeat CPU/Memory per container", "uiStateJSON": "{\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-4\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", - "panelsJSON": "[{\"col\":4,\"id\":\"Container-CPU-usage\",\"panelIndex\":2,\"row\":1,\"size_x\":9,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":3,\"row\":1,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Container-Memory-stats\",\"panelIndex\":4,\"row\":5,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Container-Block-IO\",\"panelIndex\":5,\"row\":8,\"size_x\":12,\"size_y\":4,\"type\":\"visualization\"}]", + "panelsJSON": "[{\"col\":1,\"id\":\"Container-CPU-usage\",\"panelIndex\":2,\"row\":2,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":3,\"row\":1,\"size_x\":12,\"size_y\":1,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Container-Memory-stats\",\"panelIndex\":4,\"row\":5,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Container-Block-IO\",\"panelIndex\":5,\"row\":8,\"size_x\":12,\"size_y\":4,\"type\":\"visualization\"}]", "optionsJSON": "{\"darkTheme\":false}", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" } } \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-cpu.json b/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-cpu.json index addfef84141..df7022ad75f 100644 --- a/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-cpu.json +++ b/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-cpu.json @@ -4,10 +4,10 @@ "description": "", "title": "Metricbeat-cpu", "uiStateJSON": "{\"P-9\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", - "panelsJSON": "[{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":2,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"CPU-usage-over-time\",\"panelIndex\":4,\"row\":4,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":9,\"id\":\"System-load\",\"panelIndex\":6,\"row\":1,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"System-Load-over-time\",\"panelIndex\":8,\"row\":4,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-hosts-by-CPU-usage\",\"panelIndex\":9,\"row\":9,\"size_x\":12,\"size_y\":5,\"type\":\"visualization\"},{\"col\":3,\"id\":\"CPU-Usage\",\"panelIndex\":10,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"}]", + "panelsJSON": "[{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":2,\"row\":1,\"size_x\":12,\"size_y\":1,\"type\":\"visualization\"},{\"col\":1,\"id\":\"CPU-usage-over-time\",\"panelIndex\":4,\"row\":5,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":7,\"id\":\"System-load\",\"panelIndex\":6,\"row\":2,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"System-Load-over-time\",\"panelIndex\":8,\"row\":5,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-hosts-by-CPU-usage\",\"panelIndex\":9,\"row\":10,\"size_x\":12,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"CPU-Usage\",\"panelIndex\":10,\"row\":2,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"}]", "optionsJSON": "{\"darkTheme\":false}", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" } } \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-filesystem-per-Host.json b/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-filesystem-per-Host.json index 51869dd0138..87c04480a9c 100644 --- a/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-filesystem-per-Host.json +++ b/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-filesystem-per-Host.json @@ -4,10 +4,10 @@ "description": "", "title": "Metricbeat filesystem per Host", "uiStateJSON": "{\"P-1\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", - "panelsJSON": "[{\"col\":1,\"id\":\"Top-disks-by-memory-usage\",\"panelIndex\":1,\"row\":6,\"size_x\":12,\"size_y\":5,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Disk-utilization-over-time\",\"panelIndex\":2,\"row\":1,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":3,\"row\":1,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Disk-space-distribution\",\"panelIndex\":5,\"row\":1,\"size_x\":3,\"size_y\":5,\"type\":\"visualization\"}]", + "panelsJSON": "[{\"col\":7,\"id\":\"Top-disks-by-memory-usage\",\"panelIndex\":1,\"row\":2,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":3,\"id\":\"Disk-utilization-over-time\",\"panelIndex\":2,\"row\":2,\"size_x\":4,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":3,\"row\":1,\"size_x\":12,\"size_y\":1,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Disk-space-distribution\",\"panelIndex\":5,\"row\":2,\"size_x\":2,\"size_y\":5,\"type\":\"visualization\"}]", "optionsJSON": "{\"darkTheme\":false}", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" } } \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-filesystem.json b/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-filesystem.json index d5ed85b065f..48466174932 100644 --- a/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-filesystem.json +++ b/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-filesystem.json @@ -4,10 +4,10 @@ "description": "", "title": "Metricbeat-filesystem", "uiStateJSON": "{\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", - "panelsJSON": "[{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":1,\"row\":1,\"size_x\":2,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-hosts-by-disk-size\",\"panelIndex\":5,\"row\":10,\"size_x\":12,\"size_y\":4,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Disk-space-overview\",\"panelIndex\":6,\"row\":1,\"size_x\":9,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Free-disk-space-over-days\",\"panelIndex\":7,\"row\":5,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Total-files-over-days\",\"panelIndex\":8,\"row\":5,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"}]", + "panelsJSON": "[{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":1,\"row\":1,\"size_x\":12,\"size_y\":1,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-hosts-by-disk-size\",\"panelIndex\":5,\"row\":9,\"size_x\":12,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Disk-space-overview\",\"panelIndex\":6,\"row\":2,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Free-disk-space-over-days\",\"panelIndex\":7,\"row\":4,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Total-files-over-days\",\"panelIndex\":8,\"row\":4,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"}]", "optionsJSON": "{\"darkTheme\":false}", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" } } \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-memory.json b/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-memory.json index ae084177b78..7791f3aca2b 100644 --- a/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-memory.json +++ b/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-memory.json @@ -4,10 +4,10 @@ "description": "", "title": "Metricbeat-memory", "uiStateJSON": "{\"P-7\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}}}", - "panelsJSON": "[{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":1,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-hosts-by-memory-usage\",\"panelIndex\":7,\"row\":9,\"size_x\":12,\"size_y\":5,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Memory-usage-over-time\",\"panelIndex\":10,\"row\":4,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Swap-usage-over-time\",\"panelIndex\":11,\"row\":4,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":3,\"id\":\"Total-Memory\",\"panelIndex\":12,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"Available-Memory\",\"panelIndex\":13,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Memory-usage\",\"panelIndex\":14,\"row\":1,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Swap-usage\",\"panelIndex\":15,\"row\":1,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"}]", + "panelsJSON": "[{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":1,\"row\":1,\"size_x\":12,\"size_y\":1,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-hosts-by-memory-usage\",\"panelIndex\":7,\"row\":10,\"size_x\":12,\"size_y\":5,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Memory-usage-over-time\",\"panelIndex\":10,\"row\":5,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Swap-usage-over-time\",\"panelIndex\":11,\"row\":5,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Total-Memory\",\"panelIndex\":12,\"row\":2,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Available-Memory\",\"panelIndex\":13,\"row\":2,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Memory-usage\",\"panelIndex\":14,\"row\":2,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":10,\"id\":\"Swap-usage\",\"panelIndex\":15,\"row\":2,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"}]", "optionsJSON": "{\"darkTheme\":false}", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" } } \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-network.json b/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-network.json index 3c5300f7cf9..c39e36a8fee 100644 --- a/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-network.json +++ b/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-network.json @@ -4,10 +4,10 @@ "description": "", "title": "Metricbeat-network", "uiStateJSON": "{\"P-6\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", - "panelsJSON": "[{\"col\":1,\"id\":\"In-vs-Out-Network-Bytes\",\"panelIndex\":5,\"row\":4,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-10-interfaces\",\"panelIndex\":6,\"row\":9,\"size_x\":12,\"size_y\":6,\"type\":\"visualization\"},{\"col\":9,\"id\":\"Network-Packetloss\",\"panelIndex\":13,\"row\":1,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Packet-loss-on-interfaces\",\"panelIndex\":22,\"row\":4,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":23,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":3,\"id\":\"Network-Bytes\",\"panelIndex\":24,\"row\":1,\"size_x\":5,\"size_y\":3,\"type\":\"visualization\"}]", + "panelsJSON": "[{\"col\":1,\"id\":\"In-vs-Out-Network-Bytes\",\"panelIndex\":5,\"row\":5,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-10-interfaces\",\"panelIndex\":6,\"row\":10,\"size_x\":12,\"size_y\":6,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Network-Packetloss\",\"panelIndex\":13,\"row\":2,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Packet-loss-on-interfaces\",\"panelIndex\":22,\"row\":5,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":23,\"row\":1,\"size_x\":12,\"size_y\":1,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Network-Bytes\",\"panelIndex\":24,\"row\":2,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"}]", "optionsJSON": "{\"darkTheme\":false}", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" } } \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-processes.json b/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-processes.json index 4baac0d363f..9e3134cef16 100644 --- a/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-processes.json +++ b/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-processes.json @@ -4,10 +4,10 @@ "description": "", "title": "Metricbeat-processes", "uiStateJSON": "{\"P-1\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-4\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", - "panelsJSON": "[{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":5,\"row\":1,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Number-of-processes\",\"panelIndex\":7,\"row\":4,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Process-state-by-host\",\"panelIndex\":9,\"row\":1,\"size_x\":5,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"Number-of-processes-by-host\",\"panelIndex\":8,\"row\":1,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"CPU-usage-per-process\",\"panelIndex\":2,\"row\":7,\"size_x\":6,\"size_y\":8,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Memory-usage-per-process\",\"panelIndex\":3,\"row\":7,\"size_x\":6,\"size_y\":8,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-processes-by-memory-usage\",\"panelIndex\":1,\"row\":15,\"size_x\":6,\"size_y\":11,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Top-processes-by-CPU-usage\",\"panelIndex\":4,\"row\":15,\"size_x\":6,\"size_y\":11,\"type\":\"visualization\"},{\"id\":\"Number-of-processes-over-time\",\"type\":\"visualization\",\"panelIndex\":10,\"size_x\":9,\"size_y\":3,\"col\":4,\"row\":4}]", + "panelsJSON": "[{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":5,\"row\":1,\"size_x\":12,\"size_y\":1,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Number-of-processes\",\"panelIndex\":7,\"row\":2,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":4,\"id\":\"Process-state-by-host\",\"panelIndex\":9,\"row\":2,\"size_x\":5,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"Number-of-processes-by-host\",\"panelIndex\":8,\"row\":2,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"CPU-usage-per-process\",\"panelIndex\":2,\"row\":8,\"size_x\":6,\"size_y\":8,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Memory-usage-per-process\",\"panelIndex\":3,\"row\":8,\"size_x\":6,\"size_y\":8,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-processes-by-memory-usage\",\"panelIndex\":1,\"row\":16,\"size_x\":6,\"size_y\":11,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Top-processes-by-CPU-usage\",\"panelIndex\":4,\"row\":16,\"size_x\":6,\"size_y\":11,\"type\":\"visualization\"},{\"id\":\"Number-of-processes-over-time\",\"type\":\"visualization\",\"panelIndex\":10,\"size_x\":12,\"size_y\":3,\"col\":1,\"row\":5}]", "optionsJSON": "{\"darkTheme\":false}", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" } } \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-system-overview.json b/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-system-overview.json index 06b39111a3a..f1c3d3c7c4d 100644 --- a/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-system-overview.json +++ b/metricbeat/module/system/_meta/kibana/dashboard/Metricbeat-system-overview.json @@ -4,10 +4,10 @@ "description": "", "title": "Metricbeat system overview", "uiStateJSON": "{\"P-14\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", - "panelsJSON": "[{\"col\":1,\"id\":\"Network-Bytes\",\"panelIndex\":2,\"row\":6,\"size_x\":8,\"size_y\":2,\"type\":\"visualization\"},{\"col\":9,\"id\":\"Network-Packetloss\",\"panelIndex\":3,\"row\":6,\"size_x\":4,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":9,\"row\":1,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Total-Memory\",\"panelIndex\":11,\"row\":4,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":3,\"id\":\"Available-Memory\",\"panelIndex\":12,\"row\":4,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"System-overview-by-host\",\"panelIndex\":14,\"row\":8,\"size_x\":12,\"size_y\":6,\"type\":\"visualization\"},{\"col\":5,\"id\":\"System-load\",\"panelIndex\":15,\"row\":1,\"size_x\":8,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"CPU-Usage\",\"panelIndex\":16,\"row\":4,\"size_x\":8,\"size_y\":2,\"type\":\"visualization\"}]", + "panelsJSON": "[{\"col\":1,\"id\":\"Network-Bytes\",\"panelIndex\":2,\"row\":7,\"size_x\":8,\"size_y\":2,\"type\":\"visualization\"},{\"col\":9,\"id\":\"Network-Packetloss\",\"panelIndex\":3,\"row\":7,\"size_x\":4,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":9,\"row\":1,\"size_x\":12,\"size_y\":1,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Total-Memory\",\"panelIndex\":11,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":3,\"id\":\"Available-Memory\",\"panelIndex\":12,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"System-overview-by-host\",\"panelIndex\":14,\"row\":9,\"size_x\":12,\"size_y\":6,\"type\":\"visualization\"},{\"col\":5,\"id\":\"System-load\",\"panelIndex\":15,\"row\":2,\"size_x\":8,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"CPU-Usage\",\"panelIndex\":16,\"row\":5,\"size_x\":8,\"size_y\":2,\"type\":\"visualization\"}]", "optionsJSON": "{\"darkTheme\":false}", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" } } \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/dashboard/d1f1f9e0-1b1c-11e7-b09e-037021c4f8df.json b/metricbeat/module/system/_meta/kibana/dashboard/d1f1f9e0-1b1c-11e7-b09e-037021c4f8df.json new file mode 100644 index 00000000000..c4c5ed06be3 --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/dashboard/d1f1f9e0-1b1c-11e7-b09e-037021c4f8df.json @@ -0,0 +1,21 @@ +{ + "hits": 0, + "timeFrom": "now-1h", + "timeRestore": true, + "description": "", + "title": "Metricbeat Hosts Overview", + "uiStateJSON": "{\"P-6\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", + "panelsJSON": "[{\"col\":1,\"id\":\"855899e0-1b1c-11e7-b09e-037021c4f8df\",\"panelIndex\":1,\"row\":2,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"fe064790-1b1f-11e7-bec4-a5e9ec5cab8b\",\"panelIndex\":2,\"row\":2,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":10,\"id\":\"aa984970-1e0b-11e7-852e-cdcfcfdffddd\",\"panelIndex\":5,\"row\":1,\"size_x\":3,\"size_y\":1,\"type\":\"visualization\"},{\"col\":1,\"id\":\"95490a10-1e23-11e7-958f-490b8dcb96d8\",\"panelIndex\":6,\"row\":5,\"size_x\":12,\"size_y\":9,\"type\":\"visualization\"},{\"col\":1,\"id\":\"System-Navigation\",\"panelIndex\":7,\"row\":1,\"size_x\":9,\"size_y\":1,\"type\":\"visualization\"}]", + "timeTo": "now", + "optionsJSON": "{\"darkTheme\":false}", + "version": 1, + "refreshInterval": { + "section": 1, + "pause": false, + "display": "10 seconds", + "value": 10000 + }, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/089b85d0-1b16-11e7-b09e-037021c4f8df.json b/metricbeat/module/system/_meta/kibana/visualization/089b85d0-1b16-11e7-b09e-037021c4f8df.json new file mode 100644 index 00000000000..71ecc186477 --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/089b85d0-1b16-11e7-b09e-037021c4f8df.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Mericbeat: Network Traffic (Bytes)\",\"type\":\"metrics\",\"params\":{\"id\":\"da1046f0-faa0-11e6-86b1-cd7735ff7e23\",\"type\":\"timeseries\",\"series\":[{\"id\":\"da1046f1-faa0-11e6-86b1-cd7735ff7e23\",\"color\":\"rgba(0,156,224,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"da1046f2-faa0-11e6-86b1-cd7735ff7e23\",\"type\":\"max\",\"field\":\"system.network.in.bytes\"},{\"unit\":\"1s\",\"id\":\"f41f9280-faa0-11e6-86b1-cd7735ff7e23\",\"type\":\"derivative\",\"field\":\"da1046f2-faa0-11e6-86b1-cd7735ff7e23\"},{\"unit\":\"\",\"id\":\"a87398e0-1b93-11e7-8ada-3df93aab833e\",\"type\":\"positive_only\",\"field\":\"f41f9280-faa0-11e6-86b1-cd7735ff7e23\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":\"1\",\"stacked\":\"none\",\"label\":\"Inbound \",\"value_template\":\"{{value}}/s\"},{\"id\":\"fbbd5720-faa0-11e6-86b1-cd7735ff7e23\",\"color\":\"rgba(250,40,255,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"fbbd7e30-faa0-11e6-86b1-cd7735ff7e23\",\"type\":\"max\",\"field\":\"system.network.out.bytes\"},{\"unit\":\"1s\",\"id\":\"fbbd7e31-faa0-11e6-86b1-cd7735ff7e23\",\"type\":\"derivative\",\"field\":\"fbbd7e30-faa0-11e6-86b1-cd7735ff7e23\"},{\"script\":\"params.rate != null && params.rate > 0 ? params.rate * -1 : null\",\"id\":\"17e597a0-faa1-11e6-86b1-cd7735ff7e23\",\"type\":\"calculation\",\"variables\":[{\"id\":\"1940bad0-faa1-11e6-86b1-cd7735ff7e23\",\"name\":\"rate\",\"field\":\"fbbd7e31-faa0-11e6-86b1-cd7735ff7e23\"}]}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":\"1\",\"stacked\":\"none\",\"label\":\"Outbound \",\"value_template\":\"{{value}}/s\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"filter\":\"-system.network.name:l*\"},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Mericbeat: Network Traffic (Bytes)", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/1aae9140-1b93-11e7-8ada-3df93aab833e.json b/metricbeat/module/system/_meta/kibana/visualization/1aae9140-1b93-11e7-8ada-3df93aab833e.json new file mode 100644 index 00000000000..ec33b70611b --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/1aae9140-1b93-11e7-8ada-3df93aab833e.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Metricbeat: Outbound Traffic Metric\",\"type\":\"metrics\",\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"background_color_rules\":[{\"id\":\"0e346760-1b92-11e7-bec4-a5e9ec5cab8b\"}],\"filter\":\"-system.network.name:l*\",\"id\":\"0c761590-1b92-11e7-bec4-a5e9ec5cab8b\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"0c761591-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Outbound Traffic\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.out.bytes\",\"id\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"field\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"f2074f70-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"unit\":\"\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"everything\",\"stacked\":\"none\",\"value_template\":\"{{value}}/s\"},{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"37f70440-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Total Transfered\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.out.bytes\",\"id\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"unit\":\"\",\"id\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"field\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\"},{\"sigma\":\"\",\"field\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"id\":\"3e63c2f0-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"sum_bucket\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"everything\",\"stacked\":\"none\",\"value_template\":\"{{value}}\"}],\"show_legend\":1,\"time_field\":\"@timestamp\",\"type\":\"metric\"},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Metricbeat: Outbound Traffic Metric", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/217025e0-2a69-11e7-99f0-399f2a11b723.json b/metricbeat/module/system/_meta/kibana/visualization/217025e0-2a69-11e7-99f0-399f2a11b723.json new file mode 100644 index 00000000000..5db5273d077 --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/217025e0-2a69-11e7-99f0-399f2a11b723.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Top 10 hosts by CPU\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"system.cpu.user.pct\",\"customLabel\":\"CPU usage (%)\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"beat.hostname\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"Hostname\"}}],\"listeners\":{}}", + "description": "", + "title": "Top 10 hosts by CPU", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"metricbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[{\"meta\":{\"index\":\"metricbeat-*\",\"key\":\"beat.name\",\"value\":\"mar.local\",\"disabled\":false,\"negate\":false,\"alias\":null},\"query\":{\"match\":{\"beat.name\":{\"query\":\"mar.local\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"globalState\"}}]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/26732e20-1b91-11e7-bec4-a5e9ec5cab8b.json b/metricbeat/module/system/_meta/kibana/visualization/26732e20-1b91-11e7-bec4-a5e9ec5cab8b.json new file mode 100644 index 00000000000..d164cc363cc --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/26732e20-1b91-11e7-bec4-a5e9ec5cab8b.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Metricbeat: 5m Load Gauge\",\"type\":\"metrics\",\"params\":{\"id\":\"fdcc6180-1b90-11e7-bec4-a5e9ec5cab8b\",\"type\":\"gauge\",\"series\":[{\"id\":\"fdcc6181-1b90-11e7-bec4-a5e9ec5cab8b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"fdcc6182-1b90-11e7-bec4-a5e9ec5cab8b\",\"type\":\"avg\",\"field\":\"system.load.5\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"5m Load\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"background_color_rules\":[{\"id\":\"feefabd0-1b90-11e7-bec4-a5e9ec5cab8b\"}],\"gauge_color_rules\":[{\"id\":\"ffd94880-1b90-11e7-bec4-a5e9ec5cab8b\"}],\"gauge_width\":10,\"gauge_inner_width\":10,\"gauge_style\":\"half\"},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Metricbeat: 5m Load Gauge", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/2e224660-1b19-11e7-b09e-037021c4f8df.json b/metricbeat/module/system/_meta/kibana/visualization/2e224660-1b19-11e7-b09e-037021c4f8df.json new file mode 100644 index 00000000000..016e4b6a989 --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/2e224660-1b19-11e7-b09e-037021c4f8df.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Metricbeat: Processes By Memory\",\"type\":\"metrics\",\"params\":{\"id\":\"edfceb30-1b18-11e7-b09e-037021c4f8df\",\"type\":\"top_n\",\"series\":[{\"id\":\"edfceb31-1b18-11e7-b09e-037021c4f8df\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"edfceb32-1b18-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.process.memory.rss.pct\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"terms_field\":\"system.process.name\",\"terms_order_by\":\"edfceb32-1b18-11e7-b09e-037021c4f8df\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"bar_color_rules\":[{\"value\":0,\"id\":\"efb9b660-1b18-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(104,188,0,1)\",\"opperator\":\"gte\"},{\"value\":0.7,\"id\":\"17fcb820-1b19-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(254,146,0,1)\",\"opperator\":\"gte\"},{\"value\":0.85,\"id\":\"1dd61070-1b19-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(211,49,21,1)\",\"opperator\":\"gte\"}],\"drilldown_url\":\"\",\"filter\":\"\"},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Metricbeat: Processes By Memory", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/34f97ee0-1b96-11e7-8ada-3df93aab833e.json b/metricbeat/module/system/_meta/kibana/visualization/34f97ee0-1b96-11e7-8ada-3df93aab833e.json new file mode 100644 index 00000000000..e105709bd70 --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/34f97ee0-1b96-11e7-8ada-3df93aab833e.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Metricbeat: Disk Usage\",\"type\":\"metrics\",\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"bar_color_rules\":[{\"bar_color\":\"rgba(104,188,0,1)\",\"id\":\"bf525310-1b95-11e7-8ada-3df93aab833e\",\"opperator\":\"gte\",\"value\":0},{\"bar_color\":\"rgba(254,146,0,1)\",\"id\":\"125fc4c0-1b96-11e7-8ada-3df93aab833e\",\"opperator\":\"gte\",\"value\":0.7},{\"bar_color\":\"rgba(211,49,21,1)\",\"id\":\"1a5c7240-1b96-11e7-8ada-3df93aab833e\",\"opperator\":\"gte\",\"value\":0.85}],\"drilldown_url\":\"\",\"filter\":\"-system.filesystem.mount_point:\\\\/run* AND -system.filesystem.mount_point:\\\\/sys* AND -system.filesystem.mount_point:\\\\/dev* AND -system.filesystem.mount_point:\\\\/proc* AND -system.filesystem.mount_point:\\\\/var* AND -system.filesystem.mount_point:\\\\/boot\",\"id\":\"9f7e48a0-1b95-11e7-8ada-3df93aab833e\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"percent\",\"id\":\"9f7e48a1-1b95-11e7-8ada-3df93aab833e\",\"line_width\":1,\"metrics\":[{\"field\":\"system.filesystem.used.pct\",\"id\":\"9f7e48a2-1b95-11e7-8ada-3df93aab833e\",\"type\":\"avg\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"terms_field\":\"system.filesystem.mount_point\"}],\"show_legend\":1,\"time_field\":\"@timestamp\",\"type\":\"top_n\"},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Metricbeat: Disk Usage", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/4d546850-1b15-11e7-b09e-037021c4f8df.json b/metricbeat/module/system/_meta/kibana/visualization/4d546850-1b15-11e7-b09e-037021c4f8df.json new file mode 100644 index 00000000000..e56c422d4dc --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/4d546850-1b15-11e7-b09e-037021c4f8df.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Metricbeat: System Load\",\"type\":\"metrics\",\"params\":{\"id\":\"f6264ad0-1b14-11e7-b09e-037021c4f8df\",\"type\":\"timeseries\",\"series\":[{\"id\":\"f62671e0-1b14-11e7-b09e-037021c4f8df\",\"color\":\"rgba(115,216,255,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"f62671e1-1b14-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.load.1\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"3\",\"point_size\":1,\"fill\":\"0\",\"stacked\":\"none\",\"label\":\"1m\"},{\"id\":\"1c324850-1b15-11e7-b09e-037021c4f8df\",\"color\":\"rgba(0,156,224,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"1c324851-1b15-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.load.5\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"3\",\"point_size\":\"0\",\"fill\":\"0\",\"stacked\":\"none\",\"label\":\"5m\"},{\"id\":\"3287e740-1b15-11e7-b09e-037021c4f8df\",\"color\":\"rgba(0,98,177,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"32880e50-1b15-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.load.15\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"3\",\"point_size\":\"0\",\"fill\":\"0\",\"stacked\":\"none\",\"label\":\"15m\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Metricbeat: System Load", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/4e4bb1e0-1b1b-11e7-b09e-037021c4f8df.json b/metricbeat/module/system/_meta/kibana/visualization/4e4bb1e0-1b1b-11e7-b09e-037021c4f8df.json new file mode 100644 index 00000000000..87fefa277eb --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/4e4bb1e0-1b1b-11e7-b09e-037021c4f8df.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Metricbeat: Disk IO (Bytes)\",\"type\":\"metrics\",\"params\":{\"id\":\"d3c67db0-1b1a-11e7-b09e-037021c4f8df\",\"type\":\"timeseries\",\"series\":[{\"id\":\"d3c67db1-1b1a-11e7-b09e-037021c4f8df\",\"color\":\"rgba(22,165,165,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"d3c67db2-1b1a-11e7-b09e-037021c4f8df\",\"type\":\"max\",\"field\":\"system.diskio.read.bytes\"},{\"unit\":\"\",\"id\":\"f55b9910-1b1a-11e7-b09e-037021c4f8df\",\"type\":\"derivative\",\"field\":\"d3c67db2-1b1a-11e7-b09e-037021c4f8df\"},{\"unit\":\"\",\"id\":\"dcbbb100-1b93-11e7-8ada-3df93aab833e\",\"type\":\"positive_only\",\"field\":\"f55b9910-1b1a-11e7-b09e-037021c4f8df\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":\"0\",\"fill\":\"1\",\"stacked\":\"none\",\"label\":\"reads\",\"value_template\":\"{{value}}/s\"},{\"id\":\"144124d0-1b1b-11e7-b09e-037021c4f8df\",\"color\":\"rgba(251,158,0,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"144124d1-1b1b-11e7-b09e-037021c4f8df\",\"type\":\"max\",\"field\":\"system.diskio.write.bytes\"},{\"unit\":\"\",\"id\":\"144124d2-1b1b-11e7-b09e-037021c4f8df\",\"type\":\"derivative\",\"field\":\"144124d1-1b1b-11e7-b09e-037021c4f8df\"},{\"script\":\"params.rate > 0 ? params.rate * -1 : 0\",\"id\":\"144124d4-1b1b-11e7-b09e-037021c4f8df\",\"type\":\"calculation\",\"variables\":[{\"id\":\"144124d3-1b1b-11e7-b09e-037021c4f8df\",\"name\":\"rate\",\"field\":\"144124d2-1b1b-11e7-b09e-037021c4f8df\"}]}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":\"0\",\"fill\":\"1\",\"stacked\":\"none\",\"label\":\"writes\",\"value_template\":\"{{value}}/s\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"filter\":\"\"},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Metricbeat: Disk IO (Bytes)", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/522ee670-1b92-11e7-bec4-a5e9ec5cab8b.json b/metricbeat/module/system/_meta/kibana/visualization/522ee670-1b92-11e7-bec4-a5e9ec5cab8b.json new file mode 100644 index 00000000000..0c48b6422b1 --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/522ee670-1b92-11e7-bec4-a5e9ec5cab8b.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Metricbeat: Inbound Traffic Metric\",\"type\":\"metrics\",\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"background_color_rules\":[{\"id\":\"0e346760-1b92-11e7-bec4-a5e9ec5cab8b\"}],\"filter\":\"-system.network.name:l*\",\"id\":\"0c761590-1b92-11e7-bec4-a5e9ec5cab8b\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"0c761591-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Inbound Traffic\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.in.bytes\",\"id\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"0c761592-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"field\":\"1d659060-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"f2074f70-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"unit\":\"\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"everything\",\"stacked\":\"none\",\"value_template\":\"{{value}}/s\"},{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"#68BC00\",\"fill\":0.5,\"formatter\":\"bytes\",\"id\":\"37f70440-1b92-11e7-bec4-a5e9ec5cab8b\",\"label\":\"Total Transfered\",\"line_width\":1,\"metrics\":[{\"field\":\"system.network.in.bytes\",\"id\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"max\"},{\"field\":\"37f72b50-1b92-11e7-bec4-a5e9ec5cab8b\",\"id\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"derivative\",\"unit\":\"1s\"},{\"unit\":\"\",\"id\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"type\":\"positive_only\",\"field\":\"37f72b51-1b92-11e7-bec4-a5e9ec5cab8b\"},{\"sigma\":\"\",\"field\":\"f9da2dd0-1b92-11e7-a416-41f5ccdba2e6\",\"id\":\"3e63c2f0-1b92-11e7-bec4-a5e9ec5cab8b\",\"type\":\"sum_bucket\"}],\"point_size\":1,\"seperate_axis\":0,\"split_mode\":\"everything\",\"stacked\":\"none\",\"value_template\":\"{{value}}\"}],\"show_legend\":1,\"time_field\":\"@timestamp\",\"type\":\"metric\"},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Metricbeat: Inbound Traffic Metric", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/6b7b9a40-faa1-11e6-86b1-cd7735ff7e23.json b/metricbeat/module/system/_meta/kibana/visualization/6b7b9a40-faa1-11e6-86b1-cd7735ff7e23.json new file mode 100644 index 00000000000..27baeb2ee72 --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/6b7b9a40-faa1-11e6-86b1-cd7735ff7e23.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Mericbeat: Network Traffic (Packets)\",\"type\":\"metrics\",\"params\":{\"id\":\"da1046f0-faa0-11e6-86b1-cd7735ff7e23\",\"type\":\"timeseries\",\"series\":[{\"id\":\"da1046f1-faa0-11e6-86b1-cd7735ff7e23\",\"color\":\"rgba(0,156,224,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"da1046f2-faa0-11e6-86b1-cd7735ff7e23\",\"type\":\"max\",\"field\":\"system.network.in.packets\"},{\"unit\":\"1s\",\"id\":\"f41f9280-faa0-11e6-86b1-cd7735ff7e23\",\"type\":\"derivative\",\"field\":\"da1046f2-faa0-11e6-86b1-cd7735ff7e23\"},{\"unit\":\"\",\"id\":\"c0da3d80-1b93-11e7-8ada-3df93aab833e\",\"type\":\"positive_only\",\"field\":\"f41f9280-faa0-11e6-86b1-cd7735ff7e23\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"0.[00]a\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":\"1\",\"stacked\":\"none\",\"label\":\"Inbound\",\"value_template\":\"{{value}}/s\"},{\"id\":\"fbbd5720-faa0-11e6-86b1-cd7735ff7e23\",\"color\":\"rgba(250,40,255,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"fbbd7e30-faa0-11e6-86b1-cd7735ff7e23\",\"type\":\"max\",\"field\":\"system.network.out.packets\"},{\"unit\":\"1s\",\"id\":\"fbbd7e31-faa0-11e6-86b1-cd7735ff7e23\",\"type\":\"derivative\",\"field\":\"fbbd7e30-faa0-11e6-86b1-cd7735ff7e23\"},{\"script\":\"params.rate != null && params.rate > 0 ? params.rate * -1 : null\",\"id\":\"17e597a0-faa1-11e6-86b1-cd7735ff7e23\",\"type\":\"calculation\",\"variables\":[{\"id\":\"1940bad0-faa1-11e6-86b1-cd7735ff7e23\",\"name\":\"rate\",\"field\":\"fbbd7e31-faa0-11e6-86b1-cd7735ff7e23\"}]}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"0.[00]a\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":\"1\",\"stacked\":\"none\",\"label\":\"Outbound\",\"value_template\":\"{{value}}/s\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"filter\":\"-system.network.name:l*\"},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Mericbeat: Network Traffic (Packets)", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/83e12df0-1b91-11e7-bec4-a5e9ec5cab8b.json b/metricbeat/module/system/_meta/kibana/visualization/83e12df0-1b91-11e7-bec4-a5e9ec5cab8b.json new file mode 100644 index 00000000000..d864ec2b61b --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/83e12df0-1b91-11e7-bec4-a5e9ec5cab8b.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Metricbeat: CPU Usage Gauge\",\"type\":\"metrics\",\"params\":{\"id\":\"4c9e2550-1b91-11e7-bec4-a5e9ec5cab8b\",\"type\":\"gauge\",\"series\":[{\"id\":\"4c9e2551-1b91-11e7-bec4-a5e9ec5cab8b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"4c9e2552-1b91-11e7-bec4-a5e9ec5cab8b\",\"type\":\"avg\",\"field\":\"system.cpu.idle.pct\"},{\"script\":\"params.idle > 0 ? 1 - params.idle : null\",\"id\":\"587aa510-1b91-11e7-bec4-a5e9ec5cab8b\",\"type\":\"calculation\",\"variables\":[{\"id\":\"5a19af10-1b91-11e7-bec4-a5e9ec5cab8b\",\"name\":\"idle\",\"field\":\"4c9e2552-1b91-11e7-bec4-a5e9ec5cab8b\"}]}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"CPU Usage\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"gauge_color_rules\":[{\"value\":0,\"id\":\"4ef2c3b0-1b91-11e7-bec4-a5e9ec5cab8b\",\"gauge\":\"rgba(104,188,0,1)\",\"opperator\":\"gte\"},{\"value\":0.7,\"id\":\"e6561ae0-1b91-11e7-bec4-a5e9ec5cab8b\",\"gauge\":\"rgba(254,146,0,1)\",\"opperator\":\"gte\"},{\"value\":0.85,\"id\":\"ec655040-1b91-11e7-bec4-a5e9ec5cab8b\",\"gauge\":\"rgba(211,49,21,1)\",\"opperator\":\"gte\"}],\"gauge_width\":10,\"gauge_inner_width\":10,\"gauge_style\":\"half\",\"gauge_max\":\"1\",\"filter\":\"\"},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Metricbeat: CPU Usage Gauge", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/855899e0-1b1c-11e7-b09e-037021c4f8df.json b/metricbeat/module/system/_meta/kibana/visualization/855899e0-1b1c-11e7-b09e-037021c4f8df.json new file mode 100644 index 00000000000..a3bddca10e4 --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/855899e0-1b1c-11e7-b09e-037021c4f8df.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Top Hosts By CPU (Realtime)\",\"type\":\"metrics\",\"params\":{\"id\":\"31e5afa0-1b1c-11e7-b09e-037021c4f8df\",\"type\":\"top_n\",\"series\":[{\"id\":\"31e5afa1-1b1c-11e7-b09e-037021c4f8df\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"31e5afa2-1b1c-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.cpu.user.pct\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"terms_field\":\"beat.hostname\",\"terms_order_by\":\"31e5afa2-1b1c-11e7-b09e-037021c4f8df\",\"terms_size\":\"10\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"bar_color_rules\":[{\"value\":0,\"id\":\"33349dd0-1b1c-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(104,188,0,1)\",\"opperator\":\"gte\"},{\"value\":0.6,\"id\":\"997dc440-1b1c-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(254,146,0,1)\",\"opperator\":\"gte\"},{\"value\":0.85,\"id\":\"a10d7f20-1b1c-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(211,49,21,1)\",\"opperator\":\"gte\"}],\"drilldown_url\":\"../app/kibana#/dashboard/79ffd6e0-faa0-11e6-947f-177f697178b8?_a=(query:(query_string:(analyze_wildcard:!t,query:'beat.hostname:\\\"{{key}}\\\"')))\",\"filter\":\"\"},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Top Hosts By CPU (Realtime)", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/95490a10-1e23-11e7-958f-490b8dcb96d8.json b/metricbeat/module/system/_meta/kibana/visualization/95490a10-1e23-11e7-958f-490b8dcb96d8.json new file mode 100644 index 00000000000..2b07841d21c --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/95490a10-1e23-11e7-958f-490b8dcb96d8.json @@ -0,0 +1,11 @@ +{ + "visState": "{\"title\":\"Top 100 Host By CPU\",\"type\":\"table\",\"params\":{\"perPage\":50,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"beat.name\",\"size\":100,\"orderAgg\":{\"id\":\"2-orderAgg\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"orderAgg\",\"params\":{\"field\":\"system.cpu.user.pct\"}},\"order\":\"desc\",\"orderBy\":\"custom\",\"customLabel\":\"Host\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"system.cpu.user.pct\",\"customLabel\":\"CPU usage\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"system.load.norm.1\",\"customLabel\":\"System load / cores\"}},{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"system.memory.used.pct\",\"customLabel\":\"Used memory\"}},{\"id\":\"8\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"system.memory.total\",\"customLabel\":\"Total memory\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"system.fsstat.total_size.used\",\"customLabel\":\"Used disk space\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"system.network.in.dropped\",\"customLabel\":\"In dropped\"}},{\"id\":\"7\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"system.network.out.dropped\",\"customLabel\":\"Out dropped\"}}],\"listeners\":{}}", + "description": "", + "title": "Top 100 Host By CPU", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "savedSearchId": "System-stats", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/Container-CPU-usage.json b/metricbeat/module/system/_meta/kibana/visualization/Container-CPU-usage.json index 0fdff1d6fe3..ec5ae368060 100644 --- a/metricbeat/module/system/_meta/kibana/visualization/Container-CPU-usage.json +++ b/metricbeat/module/system/_meta/kibana/visualization/Container-CPU-usage.json @@ -1,10 +1,10 @@ { "visState": "{\"title\":\"Container CPU usage\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"system.process.cgroup.cpuacct.stats.user.ns\",\"customLabel\":\"CPU user\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"system.process.cgroup.cpu.cfs.quota.us\",\"customLabel\":\"CPU quota\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"system.process.cgroup.id\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Container ID\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"system.process.cgroup.cpu.stats.throttled.ns\",\"customLabel\":\"CPU throttling\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"system.process.cgroup.cpuacct.stats.system.ns\",\"customLabel\":\"CPU kernel\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"system.process.name\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Process name\"}}],\"listeners\":{}}", - "description": "", - "title": "Container CPU usage", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "version": 1, + "description": "", + "title": "Container CPU usage", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"index\":\"metricbeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" } -} +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/Disk-space-distribution.json b/metricbeat/module/system/_meta/kibana/visualization/Disk-space-distribution.json index a98057b0d70..d3229e2ac73 100644 --- a/metricbeat/module/system/_meta/kibana/visualization/Disk-space-distribution.json +++ b/metricbeat/module/system/_meta/kibana/visualization/Disk-space-distribution.json @@ -1,5 +1,5 @@ { - "visState": "{\"title\":\"Disk space distribution\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"system.filesystem.total\",\"customLabel\":\"Total size\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"system.filesystem.mount_point\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Mount point\"}}],\"listeners\":{}}", + "visState": "{\"title\":\"Disk space distribution\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false,\"legendPosition\":\"bottom\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"system.filesystem.total\",\"customLabel\":\"Total size\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"system.filesystem.mount_point\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"Mount point\"}}],\"listeners\":{}}", "description": "", "title": "Disk space distribution", "uiStateJSON": "{}", diff --git a/metricbeat/module/system/_meta/kibana/visualization/System-Navigation.json b/metricbeat/module/system/_meta/kibana/visualization/System-Navigation.json index 177f8cd09d9..80e59adb254 100644 --- a/metricbeat/module/system/_meta/kibana/visualization/System-Navigation.json +++ b/metricbeat/module/system/_meta/kibana/visualization/System-Navigation.json @@ -1,10 +1,10 @@ { - "visState": "{\"title\":\"System Navigation\",\"type\":\"markdown\",\"params\":{\"markdown\":\"- [Overview](#/dashboard/Metricbeat-system-overview)\\n\\n- [Load/CPU](#/dashboard/Metricbeat-cpu)\\n\\n- [Memory](#/dashboard/Metricbeat-memory)\\n\\n- [Processes](#/dashboard/Metricbeat-processes)\\n\\n- [Network](#/dashboard/Metricbeat-network)\\n\\n- [Filesystem](#/dashboard/Metricbeat-filesystem)\\n\\n- [Filesystem per Host](#/dashboard/Metricbeat-filesystem-per-Host)\\n\\n- [CPU/Memory per container](#/dashboard/CPU-slash-Memory-per-container)\"},\"aggs\":[],\"listeners\":{}}", + "visState": "{\"title\":\"System Navigation\",\"type\":\"markdown\",\"params\":{\"markdown\":\"| [System Overview](#/dashboard/Metricbeat-system-overview) | [Hosts Overview (requires Kibana > 5.4)](#/dashboard/d1f1f9e0-1b1c-11e7-b09e-037021c4f8df) | [Host Overview (requires Kibana > 5.4)](#/dashboard/79ffd6e0-faa0-11e6-947f-177f697178b8) | [Load/CPU](#/dashboard/Metricbeat-cpu) | [Memory](#/dashboard/Metricbeat-memory) | [Processes](#/dashboard/Metricbeat-processes) | [Network](#/dashboard/Metricbeat-network) | [Filesystem](#/dashboard/Metricbeat-filesystem) | [Filesystem per Host](#/dashboard/Metricbeat-filesystem-per-Host) | [CPU/Memory per container](#/dashboard/CPU-slash-Memory-per-container) |\"},\"aggs\":[],\"listeners\":{}}", "description": "", "title": "System Navigation", "uiStateJSON": "{}", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}" + "searchSourceJSON": "{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[{\"meta\":{\"index\":\"metricbeat-*\",\"key\":\"beat.name\",\"value\":\"mar.local\",\"disabled\":false,\"negate\":false,\"alias\":null},\"query\":{\"match\":{\"beat.name\":{\"query\":\"mar.local\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}" } } \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/aa984970-1e0b-11e7-852e-cdcfcfdffddd.json b/metricbeat/module/system/_meta/kibana/visualization/aa984970-1e0b-11e7-852e-cdcfcfdffddd.json new file mode 100644 index 00000000000..d7ebc9de772 --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/aa984970-1e0b-11e7-852e-cdcfcfdffddd.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Warning\",\"type\":\"markdown\",\"params\":{\"markdown\":\"**This dashboard is only compatible with Kibana 5.4 and above. **\"},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Warning", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/ab2d1e90-1b1a-11e7-b09e-037021c4f8df.json b/metricbeat/module/system/_meta/kibana/visualization/ab2d1e90-1b1a-11e7-b09e-037021c4f8df.json new file mode 100644 index 00000000000..a8256e7e11f --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/ab2d1e90-1b1a-11e7-b09e-037021c4f8df.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Metricbeat: CPU Usage\",\"type\":\"metrics\",\"params\":{\"id\":\"80a04950-1b19-11e7-b09e-037021c4f8df\",\"type\":\"timeseries\",\"series\":[{\"id\":\"80a04951-1b19-11e7-b09e-037021c4f8df\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"80a04952-1b19-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.cpu.user.pct\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":\"0\",\"fill\":\"1\",\"stacked\":\"stacked\",\"label\":\"user\"},{\"id\":\"993acf30-1b19-11e7-b09e-037021c4f8df\",\"color\":\"rgba(211,49,21,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"993acf31-1b19-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.cpu.system.pct\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":\"0\",\"fill\":\"1\",\"stacked\":\"stacked\",\"label\":\"system\"},{\"id\":\"65ca35e0-1b1a-11e7-b09e-037021c4f8df\",\"color\":\"rgba(123,100,255,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"65ca5cf0-1b1a-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.cpu.nice.pct\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":\"0\",\"fill\":\"1\",\"stacked\":\"stacked\",\"label\":\"nice\"},{\"id\":\"741b5f20-1b1a-11e7-b09e-037021c4f8df\",\"color\":\"rgba(226,115,0,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"741b5f21-1b1a-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.cpu.irq.pct\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":\"0\",\"fill\":\"1\",\"stacked\":\"stacked\",\"label\":\"irq\"},{\"id\":\"2efc5d40-1b1a-11e7-b09e-037021c4f8df\",\"color\":\"rgba(176,188,0,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"2efc5d41-1b1a-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.cpu.softirq.pct\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":\"0\",\"fill\":\"1\",\"stacked\":\"stacked\",\"label\":\"softirq\"},{\"id\":\"ae644a30-1b19-11e7-b09e-037021c4f8df\",\"color\":\"rgba(15,20,25,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"ae644a31-1b19-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.cpu.iowait.pct\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":\"0\",\"fill\":\"1\",\"stacked\":\"stacked\",\"label\":\"iowait\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Metricbeat: CPU Usage", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/bfa5e400-1b16-11e7-b09e-037021c4f8df.json b/metricbeat/module/system/_meta/kibana/visualization/bfa5e400-1b16-11e7-b09e-037021c4f8df.json new file mode 100644 index 00000000000..e902c4e902a --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/bfa5e400-1b16-11e7-b09e-037021c4f8df.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Metricbeat: Memory Usage\",\"type\":\"metrics\",\"params\":{\"id\":\"32f46f40-1b16-11e7-b09e-037021c4f8df\",\"type\":\"timeseries\",\"series\":[{\"id\":\"4ff61fd0-1b16-11e7-b09e-037021c4f8df\",\"color\":\"rgba(211,49,21,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"4ff61fd1-1b16-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.memory.actual.used.bytes\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":\"0\",\"fill\":\"1\",\"stacked\":\"stacked\",\"label\":\"Used\"},{\"id\":\"753a6080-1b16-11e7-b09e-037021c4f8df\",\"color\":\"rgba(0,156,224,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"753a6081-1b16-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.memory.actual.used.bytes\"},{\"id\":\"7c9d3f00-1b16-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.memory.used.bytes\"},{\"script\":\"params.actual != null && params.used != null ? params.used - params.actual : null\",\"id\":\"869cc160-1b16-11e7-b09e-037021c4f8df\",\"type\":\"calculation\",\"variables\":[{\"id\":\"890f9620-1b16-11e7-b09e-037021c4f8df\",\"name\":\"actual\",\"field\":\"753a6081-1b16-11e7-b09e-037021c4f8df\"},{\"id\":\"8f3ab7f0-1b16-11e7-b09e-037021c4f8df\",\"name\":\"used\",\"field\":\"7c9d3f00-1b16-11e7-b09e-037021c4f8df\"}]}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":\"0\",\"fill\":\"1\",\"stacked\":\"stacked\",\"label\":\"Cache\"},{\"id\":\"32f46f41-1b16-11e7-b09e-037021c4f8df\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"32f46f42-1b16-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.memory.free\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":\"0\",\"fill\":\"1\",\"stacked\":\"stacked\",\"label\":\"Free\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Metricbeat: Memory Usage", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/d3166e80-1b91-11e7-bec4-a5e9ec5cab8b.json b/metricbeat/module/system/_meta/kibana/visualization/d3166e80-1b91-11e7-bec4-a5e9ec5cab8b.json new file mode 100644 index 00000000000..6ddf8958fb3 --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/d3166e80-1b91-11e7-bec4-a5e9ec5cab8b.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Metricbeat: Memory Usage Gauge\",\"type\":\"metrics\",\"params\":{\"id\":\"9f51b730-1b91-11e7-bec4-a5e9ec5cab8b\",\"type\":\"gauge\",\"series\":[{\"id\":\"9f51b731-1b91-11e7-bec4-a5e9ec5cab8b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"9f51b732-1b91-11e7-bec4-a5e9ec5cab8b\",\"type\":\"avg\",\"field\":\"system.memory.actual.used.pct\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Memory Usage\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"gauge_color_rules\":[{\"value\":0,\"id\":\"a0d522e0-1b91-11e7-bec4-a5e9ec5cab8b\",\"gauge\":\"rgba(104,188,0,1)\",\"opperator\":\"gte\"},{\"value\":0.7,\"id\":\"b45ad8f0-1b91-11e7-bec4-a5e9ec5cab8b\",\"gauge\":\"rgba(254,146,0,1)\",\"opperator\":\"gte\"},{\"value\":0.85,\"id\":\"c06e9550-1b91-11e7-bec4-a5e9ec5cab8b\",\"gauge\":\"rgba(211,49,21,1)\",\"opperator\":\"gte\"}],\"gauge_width\":10,\"gauge_inner_width\":10,\"gauge_style\":\"half\",\"gauge_max\":\"1\",\"filter\":\"\"},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Metricbeat: Memory Usage Gauge", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/e0f001c0-1b18-11e7-b09e-037021c4f8df.json b/metricbeat/module/system/_meta/kibana/visualization/e0f001c0-1b18-11e7-b09e-037021c4f8df.json new file mode 100644 index 00000000000..2695dacfb4f --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/e0f001c0-1b18-11e7-b09e-037021c4f8df.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Metricbeat: Top Processes By CPU\",\"type\":\"metrics\",\"params\":{\"id\":\"5f5b8d50-1b18-11e7-b09e-037021c4f8df\",\"type\":\"top_n\",\"series\":[{\"id\":\"5f5b8d51-1b18-11e7-b09e-037021c4f8df\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"5f5b8d52-1b18-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.process.cpu.total.pct\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"terms_field\":\"system.process.name\",\"terms_order_by\":\"5f5b8d52-1b18-11e7-b09e-037021c4f8df\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"bar_color_rules\":[{\"value\":0,\"id\":\"60e11be0-1b18-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(104,188,0,1)\",\"opperator\":\"gte\"}],\"drilldown_url\":\"\",\"filter\":\"\"},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Metricbeat: Top Processes By CPU", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/metricbeat/module/system/_meta/kibana/visualization/fe064790-1b1f-11e7-bec4-a5e9ec5cab8b.json b/metricbeat/module/system/_meta/kibana/visualization/fe064790-1b1f-11e7-bec4-a5e9ec5cab8b.json new file mode 100644 index 00000000000..78f6aea1eaf --- /dev/null +++ b/metricbeat/module/system/_meta/kibana/visualization/fe064790-1b1f-11e7-bec4-a5e9ec5cab8b.json @@ -0,0 +1,10 @@ +{ + "visState": "{\"title\":\"Top Hosts By Memory (Realtime)\",\"type\":\"metrics\",\"params\":{\"id\":\"31e5afa0-1b1c-11e7-b09e-037021c4f8df\",\"type\":\"top_n\",\"series\":[{\"id\":\"31e5afa1-1b1c-11e7-b09e-037021c4f8df\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"31e5afa2-1b1c-11e7-b09e-037021c4f8df\",\"type\":\"avg\",\"field\":\"system.memory.actual.used.pct\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"terms_field\":\"beat.hostname\",\"terms_order_by\":\"31e5afa2-1b1c-11e7-b09e-037021c4f8df\",\"terms_size\":\"10\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"bar_color_rules\":[{\"value\":0,\"id\":\"33349dd0-1b1c-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(104,188,0,1)\",\"opperator\":\"gte\"},{\"value\":0.6,\"id\":\"997dc440-1b1c-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(254,146,0,1)\",\"opperator\":\"gte\"},{\"value\":0.85,\"id\":\"a10d7f20-1b1c-11e7-b09e-037021c4f8df\",\"bar_color\":\"rgba(211,49,21,1)\",\"opperator\":\"gte\"}],\"drilldown_url\":\"../app/kibana#/dashboard/79ffd6e0-faa0-11e6-947f-177f697178b8?_a=(query:(query_string:(analyze_wildcard:!t,query:'beat.hostname:\\\"{{key}}\\\"')))\",\"filter\":\"\"},\"aggs\":[],\"listeners\":{}}", + "description": "", + "title": "Top Hosts By Memory (Realtime)", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}" + } +} \ No newline at end of file From 946bfd84db6c835c1eababb8c8c4929ef68db5ed Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Fri, 21 Apr 2017 16:44:24 +0200 Subject: [PATCH 31/40] Initial commit http metricbeat module --- metricbeat/docs/modules_list.asciidoc | 2 ++ metricbeat/include/list.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 418728dc554..dcab36e12ab 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -10,6 +10,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -37,6 +38,7 @@ include::modules/dropwizard.asciidoc[] include::modules/elasticsearch.asciidoc[] include::modules/golang.asciidoc[] include::modules/haproxy.asciidoc[] +include::modules/http.asciidoc[] include::modules/jolokia.asciidoc[] include::modules/kafka.asciidoc[] include::modules/kibana.asciidoc[] diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 565b1b5df1e..c3da777c3b3 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -39,6 +39,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/haproxy" _ "github.com/elastic/beats/metricbeat/module/haproxy/info" _ "github.com/elastic/beats/metricbeat/module/haproxy/stat" + _ "github.com/elastic/beats/metricbeat/module/http" + _ "github.com/elastic/beats/metricbeat/module/http/json" _ "github.com/elastic/beats/metricbeat/module/jolokia" _ "github.com/elastic/beats/metricbeat/module/jolokia/jmx" _ "github.com/elastic/beats/metricbeat/module/kafka" From 92ee85ecdd66b539652c7c00bdbd5b62963606bc Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Fri, 21 Apr 2017 16:44:39 +0200 Subject: [PATCH 32/40] Initial commit http metricbeat module --- metricbeat/docs/modules/http.asciidoc | 37 +++++++++++++ metricbeat/docs/modules/http/json.asciidoc | 19 +++++++ metricbeat/module/http/_meta/config.yml | 6 +++ metricbeat/module/http/_meta/docs.asciidoc | 4 ++ metricbeat/module/http/_meta/fields.yml | 9 ++++ metricbeat/module/http/doc.go | 4 ++ metricbeat/module/http/json/_meta/data.json | 19 +++++++ .../module/http/json/_meta/docs.asciidoc | 3 ++ metricbeat/module/http/json/_meta/fields.yml | 9 ++++ metricbeat/module/http/json/json.go | 53 +++++++++++++++++++ 10 files changed, 163 insertions(+) create mode 100644 metricbeat/docs/modules/http.asciidoc create mode 100644 metricbeat/docs/modules/http/json.asciidoc create mode 100644 metricbeat/module/http/_meta/config.yml create mode 100644 metricbeat/module/http/_meta/docs.asciidoc create mode 100644 metricbeat/module/http/_meta/fields.yml create mode 100644 metricbeat/module/http/doc.go create mode 100644 metricbeat/module/http/json/_meta/data.json create mode 100644 metricbeat/module/http/json/_meta/docs.asciidoc create mode 100644 metricbeat/module/http/json/_meta/fields.yml create mode 100644 metricbeat/module/http/json/json.go diff --git a/metricbeat/docs/modules/http.asciidoc b/metricbeat/docs/modules/http.asciidoc new file mode 100644 index 00000000000..e4f6e470323 --- /dev/null +++ b/metricbeat/docs/modules/http.asciidoc @@ -0,0 +1,37 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-module-http]] +== http Module + +This is the http Module. + + + +[float] +=== Example Configuration + +The http module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: http + metricsets: ["json"] + enabled: true + period: 10s + hosts: ["localhost"] + +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::http/json.asciidoc[] + diff --git a/metricbeat/docs/modules/http/json.asciidoc b/metricbeat/docs/modules/http/json.asciidoc new file mode 100644 index 00000000000..f9173b08d8a --- /dev/null +++ b/metricbeat/docs/modules/http/json.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-http-json]] +include::../../../module/http/json/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/http/json/_meta/data.json[] +---- diff --git a/metricbeat/module/http/_meta/config.yml b/metricbeat/module/http/_meta/config.yml new file mode 100644 index 00000000000..b74b5214045 --- /dev/null +++ b/metricbeat/module/http/_meta/config.yml @@ -0,0 +1,6 @@ +- module: http + metricsets: ["json"] + enabled: true + period: 10s + hosts: ["localhost"] + diff --git a/metricbeat/module/http/_meta/docs.asciidoc b/metricbeat/module/http/_meta/docs.asciidoc new file mode 100644 index 00000000000..72f5d9cae11 --- /dev/null +++ b/metricbeat/module/http/_meta/docs.asciidoc @@ -0,0 +1,4 @@ +== http Module + +This is the http Module. + diff --git a/metricbeat/module/http/_meta/fields.yml b/metricbeat/module/http/_meta/fields.yml new file mode 100644 index 00000000000..82c8ea4aee8 --- /dev/null +++ b/metricbeat/module/http/_meta/fields.yml @@ -0,0 +1,9 @@ +- key: http + title: "http" + description: > + http Module + fields: + - name: http + type: group + description: > + fields: diff --git a/metricbeat/module/http/doc.go b/metricbeat/module/http/doc.go new file mode 100644 index 00000000000..fadec87d6e7 --- /dev/null +++ b/metricbeat/module/http/doc.go @@ -0,0 +1,4 @@ +/* +Package http is a Metricbeat module that contains MetricSets. +*/ +package http diff --git a/metricbeat/module/http/json/_meta/data.json b/metricbeat/module/http/json/_meta/data.json new file mode 100644 index 00000000000..7324e6a3c68 --- /dev/null +++ b/metricbeat/module/http/json/_meta/data.json @@ -0,0 +1,19 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"http", + "name":"json", + "rtt":44269 + }, + "http":{ + "json":{ + "example": "json" + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/http/json/_meta/docs.asciidoc b/metricbeat/module/http/json/_meta/docs.asciidoc new file mode 100644 index 00000000000..8094e53468c --- /dev/null +++ b/metricbeat/module/http/json/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +=== http json MetricSet + +This is the json metricset of the module http. diff --git a/metricbeat/module/http/json/_meta/fields.yml b/metricbeat/module/http/json/_meta/fields.yml new file mode 100644 index 00000000000..29e0c11cb8f --- /dev/null +++ b/metricbeat/module/http/json/_meta/fields.yml @@ -0,0 +1,9 @@ +- name: json + type: group + description: > + json + fields: + - name: example + type: keyword + description: > + Example field diff --git a/metricbeat/module/http/json/json.go b/metricbeat/module/http/json/json.go new file mode 100644 index 00000000000..9893064ad6e --- /dev/null +++ b/metricbeat/module/http/json/json.go @@ -0,0 +1,53 @@ +package json + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + if err := mb.Registry.AddMetricSet("http", "json", New); err != nil { + panic(err) + } +} + +// MetricSet type defines all fields of the MetricSet +// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with +// additional entries. These variables can be used to persist data or configuration between +// multiple fetch calls. +type MetricSet struct { + mb.BaseMetricSet + counter int +} + +// New create a new instance of the MetricSet +// Part of new is also setting up the configuration by processing additional +// configuration entries if needed. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + + config := struct{}{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + counter: 1, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch() (common.MapStr, error) { + + event := common.MapStr{ + "counter": m.counter, + } + m.counter++ + + return event, nil +} From 81da4f65078aa396ff6a321c8a06ac890bdf8f68 Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Sun, 23 Apr 2017 07:50:23 +0200 Subject: [PATCH 33/40] Add first fields information Added first request/event --- metricbeat/module/http/json/_meta/fields.yml | 22 ++++++++-- metricbeat/module/http/json/json.go | 44 +++++++++++++++++--- 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/metricbeat/module/http/json/_meta/fields.yml b/metricbeat/module/http/json/_meta/fields.yml index 29e0c11cb8f..ef432c3dbda 100644 --- a/metricbeat/module/http/json/_meta/fields.yml +++ b/metricbeat/module/http/json/_meta/fields.yml @@ -1,9 +1,23 @@ - name: json type: group description: > - json + json metricset fields: - - name: example - type: keyword + - name: request + type: group description: > - Example field + HTTP request information + fields: + - name: method + type: string + description: > + The HTTP method used + - name: response + type: group + description: > + HTTP response information + fields: + - name: body + type: string + description: > + The HTTP payload returned diff --git a/metricbeat/module/http/json/json.go b/metricbeat/module/http/json/json.go index 9893064ad6e..86b1644032b 100644 --- a/metricbeat/module/http/json/json.go +++ b/metricbeat/module/http/json/json.go @@ -2,6 +2,7 @@ package json import ( "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" ) @@ -19,7 +20,9 @@ func init() { // multiple fetch calls. type MetricSet struct { mb.BaseMetricSet - counter int + http *helper.HTTP + body string + headers map[string]string } // New create a new instance of the MetricSet @@ -27,15 +30,28 @@ type MetricSet struct { // configuration entries if needed. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - config := struct{}{} + config := struct { + Method string `config:"method"` + Body string `config:"body"` + Headers map[string]string `config:"headers"` + }{} if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } + http := helper.NewHTTP(base) + http.SetMethod(config.Method) + http.SetBody([]byte(config.Body)) + for key, value := range config.Headers { + http.SetHeader(key,value) + } + return &MetricSet{ BaseMetricSet: base, - counter: 1, + http: http, + body: config.Body, + headers: config.Headers, }, nil } @@ -44,10 +60,28 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // descriptive error must be returned. func (m *MetricSet) Fetch() (common.MapStr, error) { + response, err := m.http.FetchResponse() + if err != nil { + return nil, err + } + defer response.Body.Close() + event := common.MapStr{ - "counter": m.counter, + "response.status_code": response.StatusCode, + } + + event["request"] = common.MapStr{ + "header": response.Request.Header, + "method": response.Request.Method, + "body": response.Request.Body, + + } + + event["response"] = common.MapStr{ + "status_code": response.StatusCode, + "header": response.Header, + "body": response.Body, } - m.counter++ return event, nil } From 2c84adfc313208811b695bcd5d447347c95c9331 Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Sun, 23 Apr 2017 13:29:12 +0200 Subject: [PATCH 34/40] Add first fields information Added first request/event --- metricbeat/module/http/json/_meta/fields.yml | 22 ++++++-- metricbeat/module/http/json/json.go | 55 ++++++++++++++------ 2 files changed, 58 insertions(+), 19 deletions(-) diff --git a/metricbeat/module/http/json/_meta/fields.yml b/metricbeat/module/http/json/_meta/fields.yml index ef432c3dbda..4d0148e3df4 100644 --- a/metricbeat/module/http/json/_meta/fields.yml +++ b/metricbeat/module/http/json/_meta/fields.yml @@ -8,16 +8,32 @@ description: > HTTP request information fields: + - name: header + type: nested + description: > + The HTTP headers sent - name: method - type: string + type: keyword description: > The HTTP method used + - name: body + type: keyword + description: > + The HTTP payload sent - name: response type: group description: > HTTP response information fields: + - name: header + type: nested + description: > + The HTTP headers received + - name: status_code + type: keyword + description: > + The HTTP status code - name: body - type: string + type: keyword description: > - The HTTP payload returned + The HTTP payload received diff --git a/metricbeat/module/http/json/json.go b/metricbeat/module/http/json/json.go index 86b1644032b..3bb9629851e 100644 --- a/metricbeat/module/http/json/json.go +++ b/metricbeat/module/http/json/json.go @@ -4,6 +4,9 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" + "net/http" + "strings" + "io/ioutil" ) // init registers the MetricSet with the central registry. @@ -20,9 +23,11 @@ func init() { // multiple fetch calls. type MetricSet struct { mb.BaseMetricSet - http *helper.HTTP - body string - headers map[string]string + http *helper.HTTP + headers map[string]string + method string + body string + } // New create a new instance of the MetricSet @@ -31,9 +36,9 @@ type MetricSet struct { func New(base mb.BaseMetricSet) (mb.MetricSet, error) { config := struct { - Method string `config:"method"` - Body string `config:"body"` - Headers map[string]string `config:"headers"` + Method string `config:"method"` + Body string `config:"body"` + Headers map[string]string `config:"headers"` }{} if err := base.Module().UnpackConfig(&config); err != nil { @@ -44,14 +49,16 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { http.SetMethod(config.Method) http.SetBody([]byte(config.Body)) for key, value := range config.Headers { - http.SetHeader(key,value) + http.SetHeader(key, value) } return &MetricSet{ BaseMetricSet: base, http: http, - body: config.Body, headers: config.Headers, + method: config.Method, + body: config.Body, + }, nil } @@ -66,22 +73,38 @@ func (m *MetricSet) Fetch() (common.MapStr, error) { } defer response.Body.Close() - event := common.MapStr{ - "response.status_code": response.StatusCode, + responseBody, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, err } - event["request"] = common.MapStr{ - "header": response.Request.Header, - "method": response.Request.Method, - "body": response.Request.Body, + event := common.MapStr{} + event["request"] = common.MapStr{ + "headers": m.headers, + "method": m.method, + "body": m.body, } event["response"] = common.MapStr{ "status_code": response.StatusCode, - "header": response.Header, - "body": response.Body, + "headers": m.getHeaders(response.Header), + "body": responseBody, } return event, nil } + +func (m *MetricSet) getHeaders(header http.Header) map[string]string { + + headers := make(map[string]string) + for k, v := range header { + value := "" + for _, h := range v { + value += h + " ," + } + value = strings.TrimRight(value, " ,") + headers[k] = value + } + return headers +} From 521c11cb438c83021c2748da059410ef1c330990 Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Sun, 23 Apr 2017 22:04:51 +0200 Subject: [PATCH 35/40] Updated config and docu files. --- metricbeat/docs/fields.asciidoc | 83 ++++++++++++++++++++ metricbeat/metricbeat.full.yml | 8 ++ metricbeat/module/http/json/_meta/fields.yml | 4 +- 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index ced1e37c9f6..5d4a1716034 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -23,6 +23,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -3444,6 +3445,88 @@ type: integer The average queue time in ms over the last 1024 requests. +[[exported-fields-http]] +== http Fields + +http Module + + + +[float] +== http Fields + + + + +[float] +== json Fields + +json metricset + + + +[float] +== request Fields + +HTTP request information + + + +[float] +=== http.json.request.header + +type: nested + +The HTTP headers sent + + +[float] +=== http.json.request.method + +type: keyword + +The HTTP method used + + +[float] +=== http.json.request.body + +type: keyword + +The HTTP payload sent + + +[float] +== response Fields + +HTTP response information + + + +[float] +=== http.json.response.header + +type: nested + +The HTTP headers received + + +[float] +=== http.json.response.status_code + +type: keyword + +The HTTP status code + + +[float] +=== http.json.response.body + +type: keyword + +The HTTP payload received + + [[exported-fields-jolokia]] == Jolokia Fields diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 121eed7b6ca..32db9990267 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -160,6 +160,14 @@ metricbeat.modules: period: 10s hosts: ["tcp://127.0.0.1:14567"] +#-------------------------------- http Module -------------------------------- +- module: http + metricsets: ["json"] + enabled: true + period: 10s + hosts: ["localhost"] + + #------------------------------- Jolokia Module ------------------------------ - module: jolokia metricsets: ["jmx"] diff --git a/metricbeat/module/http/json/_meta/fields.yml b/metricbeat/module/http/json/_meta/fields.yml index 4d0148e3df4..be59c8ef14b 100644 --- a/metricbeat/module/http/json/_meta/fields.yml +++ b/metricbeat/module/http/json/_meta/fields.yml @@ -11,7 +11,7 @@ - name: header type: nested description: > - The HTTP headers sent + The HTTP headers sent - name: method type: keyword description: > @@ -28,7 +28,7 @@ - name: header type: nested description: > - The HTTP headers received + The HTTP headers received - name: status_code type: keyword description: > From 9d69fd6561f3dec0c36735eac3865e0c35964367 Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Sun, 23 Apr 2017 22:12:03 +0200 Subject: [PATCH 36/40] Fix fmt findings --- metricbeat/module/http/json/json.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/http/json/json.go b/metricbeat/module/http/json/json.go index 3bb9629851e..e7c7070a3ed 100644 --- a/metricbeat/module/http/json/json.go +++ b/metricbeat/module/http/json/json.go @@ -4,9 +4,9 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" + "io/ioutil" "net/http" "strings" - "io/ioutil" ) // init registers the MetricSet with the central registry. @@ -25,9 +25,8 @@ type MetricSet struct { mb.BaseMetricSet http *helper.HTTP headers map[string]string - method string + method string body string - } // New create a new instance of the MetricSet @@ -58,7 +57,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { headers: config.Headers, method: config.Method, body: config.Body, - }, nil } From 47a046c8f30220eac3925cead6ce7f2fe26c6237 Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Sat, 29 Apr 2017 07:56:49 +0200 Subject: [PATCH 37/40] Fixed review findings added first config for testing --- metricbeat/module/http/_meta/Dockerfile | 11 +++++ metricbeat/module/http/_meta/config.yml | 5 ++- .../module/http/json/_meta/test/config.yml | 42 +++++++++++++++++++ metricbeat/module/http/json/json.go | 35 +++++++++------- 4 files changed, 76 insertions(+), 17 deletions(-) create mode 100644 metricbeat/module/http/_meta/Dockerfile create mode 100644 metricbeat/module/http/json/_meta/test/config.yml diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile new file mode 100644 index 00000000000..7d0fa83cfe6 --- /dev/null +++ b/metricbeat/module/http/_meta/Dockerfile @@ -0,0 +1,11 @@ +# Tomcat is started to fetch Jolokia metrics from it +FROM jolokia/java-jolokia:7 +ENV TOMCAT_VERSION 7.0.55 +ENV TC apache-tomcat-${TOMCAT_VERSION} + +HEALTHCHECK CMD curl -f curl localhost:8778/jolokia/ +EXPOSE 8778 +RUN wget http://archive.apache.org/dist/tomcat/tomcat-7/v${TOMCAT_VERSION}/bin/${TC}.tar.gz +RUN tar xzf ${TC}.tar.gz -C /opt + +CMD env CATALINA_OPTS=$(jolokia_opts) /opt/${TC}/bin/catalina.sh run diff --git a/metricbeat/module/http/_meta/config.yml b/metricbeat/module/http/_meta/config.yml index b74b5214045..56d9d00056d 100644 --- a/metricbeat/module/http/_meta/config.yml +++ b/metricbeat/module/http/_meta/config.yml @@ -2,5 +2,6 @@ metricsets: ["json"] enabled: true period: 10s - hosts: ["localhost"] - + hosts: ["http://httpbin.org"] + namespace: "test" + path: "/headers" diff --git a/metricbeat/module/http/json/_meta/test/config.yml b/metricbeat/module/http/json/_meta/test/config.yml new file mode 100644 index 00000000000..7fb793709b0 --- /dev/null +++ b/metricbeat/module/http/json/_meta/test/config.yml @@ -0,0 +1,42 @@ +###################### Metricbeat Configuration Example ####################### + +#========================== Modules configuration ============================ +metricbeat.modules: + +#------------------------------ Http Module ----------------------------- +- module: http + metricsets: ["json"] + enabled: true + period: 10s + namespace: "http_json" + hosts: ["http://httpbin.org"] + namespace: "test" + path: "/headers" + +#================================ Outputs ===================================== + +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["localhost:9200"] + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "elastic" + #password: "changeme" + + +output.file: + # Boolean flag to enable or disable the output module. + enabled: true + + # Path to the directory where to save the generated files. The option is + # mandatory. + path: "/tmp/httpmetric" + +logging.level: debug + +# At debug level, you can selectively enable logging only for some components. +# To enable all selectors use ["*"]. Examples of other selectors are "beat", +# "publish", "service". +logging.selectors: ["*"] diff --git a/metricbeat/module/http/json/json.go b/metricbeat/module/http/json/json.go index e7c7070a3ed..b93fc8b712c 100644 --- a/metricbeat/module/http/json/json.go +++ b/metricbeat/module/http/json/json.go @@ -2,11 +2,13 @@ package json import ( "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "io/ioutil" "net/http" "strings" + "encoding/json" ) // init registers the MetricSet with the central registry. @@ -23,10 +25,10 @@ func init() { // multiple fetch calls. type MetricSet struct { mb.BaseMetricSet - http *helper.HTTP - headers map[string]string - method string - body string + namespace string + http *helper.HTTP + method string + body string } // New create a new instance of the MetricSet @@ -34,10 +36,12 @@ type MetricSet struct { // configuration entries if needed. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + logp.Warn("The http json metricset is in beta.") + config := struct { - Method string `config:"method"` - Body string `config:"body"` - Headers map[string]string `config:"headers"` + Namespace string `config:"namespace" validate:"required"` + Method string `config:"method"` + Body string `config:"body"` }{} if err := base.Module().UnpackConfig(&config); err != nil { @@ -46,15 +50,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { http := helper.NewHTTP(base) http.SetMethod(config.Method) - http.SetBody([]byte(config.Body)) - for key, value := range config.Headers { - http.SetHeader(key, value) - } +// http.SetBody([]byte(config.Body)) return &MetricSet{ BaseMetricSet: base, - http: http, - headers: config.Headers, + namespace: config.Namespace, method: config.Method, body: config.Body, }, nil @@ -71,15 +71,17 @@ func (m *MetricSet) Fetch() (common.MapStr, error) { } defer response.Body.Close() + var raw map[string]interface{} responseBody, err := ioutil.ReadAll(response.Body) if err != nil { return nil, err } + json.Unmarshal(responseBody, &raw) event := common.MapStr{} event["request"] = common.MapStr{ - "headers": m.headers, + "headers": m.getHeaders(response.Request.Header), "method": m.method, "body": m.body, } @@ -87,9 +89,12 @@ func (m *MetricSet) Fetch() (common.MapStr, error) { event["response"] = common.MapStr{ "status_code": response.StatusCode, "headers": m.getHeaders(response.Header), - "body": responseBody, + "body": raw, } + // Set dynamic namespace + event["_namespace"] = m.namespace + return event, nil } From a3bbadb424c8d709b9ef7ce88600c1151b50535d Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Sun, 30 Apr 2017 07:27:49 +0200 Subject: [PATCH 38/40] Added integration tess Moved fields to modules Update (some) documentation Open: - complete documentation --- metricbeat/docs/modules/http.asciidoc | 5 +- metricbeat/module/http/_meta/config.yml | 4 +- metricbeat/module/http/_meta/env | 2 + metricbeat/module/http/_meta/fields.yml | 34 +++++++++ metricbeat/module/http/json/_meta/fields.yml | 36 +--------- .../module/http/json/_meta/test/config.yml | 8 +-- metricbeat/module/http/json/json.go | 69 +++++++++++++------ .../module/http/json/json_integration_test.go | 57 +++++++++++++++ 8 files changed, 152 insertions(+), 63 deletions(-) create mode 100644 metricbeat/module/http/_meta/env create mode 100644 metricbeat/module/http/json/json_integration_test.go diff --git a/metricbeat/docs/modules/http.asciidoc b/metricbeat/docs/modules/http.asciidoc index e4f6e470323..89b4471d921 100644 --- a/metricbeat/docs/modules/http.asciidoc +++ b/metricbeat/docs/modules/http.asciidoc @@ -22,8 +22,9 @@ metricbeat.modules: metricsets: ["json"] enabled: true period: 10s - hosts: ["localhost"] - + hosts: ["httpbin.org"] + namespace: "http_json_namespace" + path: "/headers" ---- [float] diff --git a/metricbeat/module/http/_meta/config.yml b/metricbeat/module/http/_meta/config.yml index 56d9d00056d..ee9288665a6 100644 --- a/metricbeat/module/http/_meta/config.yml +++ b/metricbeat/module/http/_meta/config.yml @@ -2,6 +2,6 @@ metricsets: ["json"] enabled: true period: 10s - hosts: ["http://httpbin.org"] - namespace: "test" + hosts: ["httpbin.org"] + namespace: "http_json_namespace" path: "/headers" diff --git a/metricbeat/module/http/_meta/env b/metricbeat/module/http/_meta/env new file mode 100644 index 00000000000..9c0340b6f3c --- /dev/null +++ b/metricbeat/module/http/_meta/env @@ -0,0 +1,2 @@ +JOLOKIA_HOST=jolokia +JOLOKIA_PORT=8778 diff --git a/metricbeat/module/http/_meta/fields.yml b/metricbeat/module/http/_meta/fields.yml index 82c8ea4aee8..e3799fb3c94 100644 --- a/metricbeat/module/http/_meta/fields.yml +++ b/metricbeat/module/http/_meta/fields.yml @@ -7,3 +7,37 @@ type: group description: > fields: + - name: request + type: group + description: > + HTTP request information + fields: + - name: header + type: nested + description: > + The HTTP headers sent + - name: method + type: keyword + description: > + The HTTP method used + - name: body + type: keyword + description: > + The HTTP payload sent + - name: response + type: group + description: > + HTTP response information + fields: + - name: header + type: nested + description: > + The HTTP headers received + - name: status_code + type: keyword + description: > + The HTTP status code + - name: body + type: keyword + description: > + The HTTP payload received diff --git a/metricbeat/module/http/json/_meta/fields.yml b/metricbeat/module/http/json/_meta/fields.yml index be59c8ef14b..b1683407a89 100644 --- a/metricbeat/module/http/json/_meta/fields.yml +++ b/metricbeat/module/http/json/_meta/fields.yml @@ -3,37 +3,7 @@ description: > json metricset fields: - - name: request - type: group + - name: body + type: keyword description: > - HTTP request information - fields: - - name: header - type: nested - description: > - The HTTP headers sent - - name: method - type: keyword - description: > - The HTTP method used - - name: body - type: keyword - description: > - The HTTP payload sent - - name: response - type: group - description: > - HTTP response information - fields: - - name: header - type: nested - description: > - The HTTP headers received - - name: status_code - type: keyword - description: > - The HTTP status code - - name: body - type: keyword - description: > - The HTTP payload received + The HTTP payload received diff --git a/metricbeat/module/http/json/_meta/test/config.yml b/metricbeat/module/http/json/_meta/test/config.yml index 7fb793709b0..5b5932e4737 100644 --- a/metricbeat/module/http/json/_meta/test/config.yml +++ b/metricbeat/module/http/json/_meta/test/config.yml @@ -8,10 +8,10 @@ metricbeat.modules: metricsets: ["json"] enabled: true period: 10s - namespace: "http_json" - hosts: ["http://httpbin.org"] - namespace: "test" - path: "/headers" + hosts: ["http://date.jsontest.com"] + namespace: "http_json_namespace" + headers: + Accept: application/json #================================ Outputs ===================================== diff --git a/metricbeat/module/http/json/json.go b/metricbeat/module/http/json/json.go index b93fc8b712c..f59b0cff22f 100644 --- a/metricbeat/module/http/json/json.go +++ b/metricbeat/module/http/json/json.go @@ -1,24 +1,42 @@ package json import ( + "encoding/json" + "io/ioutil" + "net/http" + "strings" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" - "io/ioutil" - "net/http" - "strings" - "encoding/json" + "github.com/elastic/beats/metricbeat/mb/parse" ) // init registers the MetricSet with the central registry. // The New method will be called after the setup of the module and before starting to fetch data func init() { - if err := mb.Registry.AddMetricSet("http", "json", New); err != nil { + if err := mb.Registry.AddMetricSet("http", "json", New, hostParser); err != nil { panic(err) } } +const ( + // defaultScheme is the default scheme to use when it is not specified in the host config. + defaultScheme = "http" + + // defaultPath is the dto use when it is not specified in the host config. + defaultPath = "" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + PathConfigKey: "path", + DefaultPath: defaultPath, + }.Build() +) + // MetricSet type defines all fields of the MetricSet // As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with // additional entries. These variables can be used to persist data or configuration between @@ -39,9 +57,9 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { logp.Warn("The http json metricset is in beta.") config := struct { - Namespace string `config:"namespace" validate:"required"` - Method string `config:"method"` - Body string `config:"body"` + Namespace string `config:"namespace" validate:"required"` + Method string `config:"method"` + Body string `config:"body"` }{} if err := base.Module().UnpackConfig(&config); err != nil { @@ -50,13 +68,14 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { http := helper.NewHTTP(base) http.SetMethod(config.Method) -// http.SetBody([]byte(config.Body)) + http.SetBody([]byte(config.Body)) return &MetricSet{ BaseMetricSet: base, namespace: config.Namespace, method: config.Method, body: config.Body, + http: http, }, nil } @@ -71,25 +90,31 @@ func (m *MetricSet) Fetch() (common.MapStr, error) { } defer response.Body.Close() - var raw map[string]interface{} - responseBody, err := ioutil.ReadAll(response.Body) + var jsonBody map[string]interface{} + + body, err := ioutil.ReadAll(response.Body) if err != nil { return nil, err } - json.Unmarshal(responseBody, &raw) - - event := common.MapStr{} - event["request"] = common.MapStr{ - "headers": m.getHeaders(response.Request.Header), - "method": m.method, - "body": m.body, + err = json.Unmarshal(body, &jsonBody) + if err != nil { + return nil, err } - event["response"] = common.MapStr{ - "status_code": response.StatusCode, - "headers": m.getHeaders(response.Header), - "body": raw, + event := common.MapStr{ + mb.ModuleData: common.MapStr{ + "request": common.MapStr{ + "headers": m.getHeaders(response.Request.Header), + "method": response.Request.Method, + "body": m.body, + }, + "response": common.MapStr{ + "status_code": response.StatusCode, + "headers": m.getHeaders(response.Header), + }, + }, + "body": jsonBody, } // Set dynamic namespace diff --git a/metricbeat/module/http/json/json_integration_test.go b/metricbeat/module/http/json/json_integration_test.go new file mode 100644 index 00000000000..0f1698d7f6e --- /dev/null +++ b/metricbeat/module/http/json/json_integration_test.go @@ -0,0 +1,57 @@ +// +build integration + +package json + +import ( + "os" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/stretchr/testify/assert" +) + +func TestFetch(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + event, err := f.Fetch() + if !assert.NoError(t, err) { + t.FailNow() + } + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) +} + +func TestData(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + err := mbtest.WriteEvent(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "http", + "metricsets": []string{"json"}, + "hosts": []string{getEnvHost() + ":" + getEnvPort()}, + "path": "/jolokia/?ignoreErrors=true&canonicalNaming=false", + "namespace": "testnamespace", + } +} + +func getEnvHost() string { + host := os.Getenv("JOLOKIA_HOST") + + if len(host) == 0 { + host = "127.0.0.1" + } + return host +} + +func getEnvPort() string { + port := os.Getenv("JOLOKIA_PORT") + + if len(port) == 0 { + port = "8778" + } + return port +} From 1fca1b0611f698f15d1070a9f282e8b01dfc7c98 Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Sun, 30 Apr 2017 10:19:35 +0200 Subject: [PATCH 39/40] Generated docucmentation --- metricbeat/docs/fields.asciidoc | 34 ++++++++++++++++++++------------- metricbeat/metricbeat.full.yml | 5 +++-- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 5d4a1716034..574e928baab 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3458,13 +3458,6 @@ http Module -[float] -== json Fields - -json metricset - - - [float] == request Fields @@ -3473,7 +3466,7 @@ HTTP request information [float] -=== http.json.request.header +=== http.request.header type: nested @@ -3481,7 +3474,7 @@ The HTTP headers sent [float] -=== http.json.request.method +=== http.request.method type: keyword @@ -3489,7 +3482,7 @@ The HTTP method used [float] -=== http.json.request.body +=== http.request.body type: keyword @@ -3504,7 +3497,7 @@ HTTP response information [float] -=== http.json.response.header +=== http.response.header type: nested @@ -3512,7 +3505,7 @@ The HTTP headers received [float] -=== http.json.response.status_code +=== http.response.status_code type: keyword @@ -3520,7 +3513,22 @@ The HTTP status code [float] -=== http.json.response.body +=== http.response.body + +type: keyword + +The HTTP payload received + + +[float] +== json Fields + +json metricset + + + +[float] +=== http.json.body type: keyword diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 32db9990267..cf721e7bfc0 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -165,8 +165,9 @@ metricbeat.modules: metricsets: ["json"] enabled: true period: 10s - hosts: ["localhost"] - + hosts: ["httpbin.org"] + namespace: "http_json_namespace" + path: "/headers" #------------------------------- Jolokia Module ------------------------------ - module: jolokia From 8eb085ff61439f898f6792a9b3ae90ed4478aa56 Mon Sep 17 00:00:00 2001 From: Christian Galsterer Date: Mon, 1 May 2017 08:12:26 +0200 Subject: [PATCH 40/40] Updated documentation and added entry to changelog --- CHANGELOG.asciidoc | 1 + metricbeat/docs/modules/http.asciidoc | 11 ++- metricbeat/module/http/_meta/docs.asciidoc | 11 ++- .../module/http/json/_meta/docs.asciidoc | 70 +++++++++++++++++++ 4 files changed, 91 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 2dc7db9db68..55dc6cd87c9 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -149,6 +149,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Add dropwizard module {pull}4022[4022] - Adding query APIs for metricsets and modules from metricbeat registry {pull}4102[4102] - Fixing nil pointer on prometheus collector when http response is nil {pull}4119[4119] +- Add http module with json metricset. {pull}4092[4092] *Packetbeat* - Add `fields` and `fields_under_root` to packetbeat protocols configurations. {pull}3518[3518] diff --git a/metricbeat/docs/modules/http.asciidoc b/metricbeat/docs/modules/http.asciidoc index 89b4471d921..267f7a41056 100644 --- a/metricbeat/docs/modules/http.asciidoc +++ b/metricbeat/docs/modules/http.asciidoc @@ -5,8 +5,17 @@ This file is generated! See scripts/docs_collector.py [[metricbeat-module-http]] == http Module -This is the http Module. +Http module is a [Metricbeat](https://www.elastic.co/products/beats/metricbeat) used to call HTTP endpoints. +Multiple endpoints can be configured which are polled in a regular interval and the result is shipped to the configured output channel. +Httpbeat is inspired by the Logstash [http_poller](https://www.elastic.co/guide/en/logstash/current/plugins-inputs-http_poller.html) input filter but doesn't require that the endpoint is reachable by Logstash as the Metricbeat module pushes the data to the configured outpust channels, e.g. Logstash or Elasticsearch. +This is often necessary in security restricted network setups, where Logstash is not able to reach all servers. Instead the server to be monitored itself has Metricbeat installed and can send the data or a collector server has Metricbeat installed which is deployed in the secured network environment and can reach all servers to be monitored. + +Example use cases are: +* Monitor [Apache Stats](https://httpd.apache.org/docs/2.4/mod/mod_status.html) +* Monitor Java application with [Jolokia](https://jolokia.org) +* Monitor [Spring Boot Actuators](http://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#production-ready) +* Monitor [Docker Remote API](https://docs.docker.com/engine/reference/api/docker_remote_api/) [float] diff --git a/metricbeat/module/http/_meta/docs.asciidoc b/metricbeat/module/http/_meta/docs.asciidoc index 72f5d9cae11..d1587e44271 100644 --- a/metricbeat/module/http/_meta/docs.asciidoc +++ b/metricbeat/module/http/_meta/docs.asciidoc @@ -1,4 +1,13 @@ == http Module -This is the http Module. +Http module is a [Metricbeat](https://www.elastic.co/products/beats/metricbeat) used to call HTTP endpoints. +Multiple endpoints can be configured which are polled in a regular interval and the result is shipped to the configured output channel. +Httpbeat is inspired by the Logstash [http_poller](https://www.elastic.co/guide/en/logstash/current/plugins-inputs-http_poller.html) input filter but doesn't require that the endpoint is reachable by Logstash as the Metricbeat module pushes the data to the configured outpust channels, e.g. Logstash or Elasticsearch. +This is often necessary in security restricted network setups, where Logstash is not able to reach all servers. Instead the server to be monitored itself has Metricbeat installed and can send the data or a collector server has Metricbeat installed which is deployed in the secured network environment and can reach all servers to be monitored. + +Example use cases are: +* Monitor [Apache Stats](https://httpd.apache.org/docs/2.4/mod/mod_status.html) +* Monitor Java application with [Jolokia](https://jolokia.org) +* Monitor [Spring Boot Actuators](http://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#production-ready) +* Monitor [Docker Remote API](https://docs.docker.com/engine/reference/api/docker_remote_api/) diff --git a/metricbeat/module/http/json/_meta/docs.asciidoc b/metricbeat/module/http/json/_meta/docs.asciidoc index 8094e53468c..f0d47e0061c 100644 --- a/metricbeat/module/http/json/_meta/docs.asciidoc +++ b/metricbeat/module/http/json/_meta/docs.asciidoc @@ -1,3 +1,73 @@ === http json MetricSet This is the json metricset of the module http. + +[float] +=== Features and Configuration + +The JSON structure returned by the HTTP endpoint will be send added as the `body` field of the provided `namespace` field as showned in the following example: + +```json +{ + "@timestamp": "2017-05-01T06:06:31.787Z", + "beat": { + "hostname": "mbp", + "name": "mbp", + "version": "6.0.0-alpha1" + }, + "http": { + "http_json_namespace": { + "body": { + "date": "05-01-2017", + "milliseconds_since_epoch": 1493618791986.000000, + "time": "06:06:31 AM" + } + }, + "request": { + "body": "", + "headers": { + "Accept": "application/json" + }, + "method": "GET" + }, + "response": { + "headers": { + "Access-Control-Allow-Origin": "*", + "Content-Length": "100", + "Content-Type": "application/json; charset=ISO-8859-1", + "Date": "Mon, 01 May 2017 06:06:31 GMT", + "Server": "Google Frontend", + "X-Cloud-Trace-Context": "580c9e1993ffbe156fcf869c33b84b78" + }, + "status_code": 200 + } + }, + "metricset": { + "host": "date.jsontest.com", + "module": "http", + "name": "json", + "namespace": "http_json_namespace", + "rtt": 257411 + }, + "type": "metricsets" +} + +``` + +Here the response from `date.jsontest.com` is returned in the configured `http_json_namespace` namespace in the `body` element as +```json +"http_json_namespace": { + "body": { + "date": "05-01-2017", + "milliseconds_since_epoch": 1493618791986.000000, + "time": "06:06:31 AM" + } + } +``` + +It is required to set a namespace in the general module config section. + +[float] +=== Exposed fields, Dashboards, Indexes, etc. +Since this is a general purpose module that can be tailored for any application that exposes a JSON structure, it +comes with no exposed fields description, dashboards or index patterns.