From 215b308b7f0f98aeba7c88e32ab4a872c1538392 Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Mon, 16 Apr 2018 20:51:17 +0200 Subject: [PATCH 01/20] added ip and hw addresses --- .../add_host_metadata/add_host_metadata.go | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index 65d8dbb65446..46f9b904ced3 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -1,6 +1,7 @@ package add_host_metadata import ( + "net" "time" "github.com/elastic/beats/libbeat/beat" @@ -71,10 +72,54 @@ func (p *addHostMetadata) loadData() { if p.info.OS.Build != "" { p.data.Put("host.os.build", p.info.OS.Build) } + + // IP-address and MAC-address + var ipList, hwList = p.getNetInfo() + p.data.Put("host.net.ip", ipList) + p.data.Put("host.net.hw", hwList) + p.lastUpdate = time.Now() } } +func (p addHostMetadata) getNetInfo() ([]string, []string) { + var ipList []string + var hwList []string + + // Get all interfaces and loop through them + ifaces, err := net.Interfaces() + if err != nil { + return ipList, hwList + } + for _, i := range ifaces { + // Skip loopback interfaces + if i.Flags&net.FlagLoopback == net.FlagLoopback { + continue + } + + hw := i.HardwareAddr.String() + // Skip empty hardware addresses + if hw != "" { + hwList = append(hwList, hw) + } + + addrs, err := i.Addrs() + if err != nil { + return ipList, hwList + } + for _, addr := range addrs { + switch v := addr.(type) { + case *net.IPNet: + ipList = append(ipList, v.IP.String()) + case *net.IPAddr: + ipList = append(ipList, v.IP.String()) + } + } + } + + return ipList, hwList +} + func (p addHostMetadata) String() string { return "add_host_metadata=[]" } From 06c1a3d488f9403a6c221751caf40ce1f1b74721 Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Mon, 16 Apr 2018 20:51:41 +0200 Subject: [PATCH 02/20] added new test case for ip and hw addresses --- .../add_host_metadata/add_host_metadata_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index 0192f57b3253..1d8386647b9c 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -31,4 +31,12 @@ func TestRun(t *testing.T) { v, err := newEvent.GetValue("host.os.family") assert.NoError(t, err) assert.NotNil(t, v) + + v, err = newEvent.GetValue("host.net.ip") + assert.NoError(t, err) + assert.NotNil(t, v) + + v, err = newEvent.GetValue("host.net.hw") + assert.NoError(t, err) + assert.NotNil(t, v) } From bf696b12787870d62b809261fbb049823d0e7ac0 Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Mon, 16 Apr 2018 20:52:21 +0200 Subject: [PATCH 03/20] added description of net.ip and net.hw fields --- libbeat/processors/add_host_metadata/_meta/fields.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libbeat/processors/add_host_metadata/_meta/fields.yml b/libbeat/processors/add_host_metadata/_meta/fields.yml index 9897d7707628..d819516e43f2 100644 --- a/libbeat/processors/add_host_metadata/_meta/fields.yml +++ b/libbeat/processors/add_host_metadata/_meta/fields.yml @@ -31,3 +31,10 @@ type: keyword description: > OS family (e.g. redhat, debian, freebsd, windows). + - name: net.ip + description: > + List of IP-addresses. + - name: net.hw + description: > + List of hardware-addresses, usually MAC-addresses. + From 70bf0573c0a032171012d03b5f103a9ac68024fe Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Tue, 17 Apr 2018 19:34:42 +0200 Subject: [PATCH 04/20] added types to fields --- libbeat/processors/add_host_metadata/_meta/fields.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libbeat/processors/add_host_metadata/_meta/fields.yml b/libbeat/processors/add_host_metadata/_meta/fields.yml index d819516e43f2..e9f677fe969d 100644 --- a/libbeat/processors/add_host_metadata/_meta/fields.yml +++ b/libbeat/processors/add_host_metadata/_meta/fields.yml @@ -32,9 +32,11 @@ description: > OS family (e.g. redhat, debian, freebsd, windows). - name: net.ip + type: ip description: > List of IP-addresses. - name: net.hw + type: keyword description: > List of hardware-addresses, usually MAC-addresses. From 5bd56a1004828a7f6132fcbe82d27e4d511fbdfd Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Tue, 17 Apr 2018 19:37:31 +0200 Subject: [PATCH 05/20] added changelog --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 221823cc3fe4..7ca7ebe502d1 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -104,6 +104,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Added logging of system info at Beat startup. {issue}5946[5946] - Do not log errors if X-Pack Monitoring is enabled but Elastisearch X-Pack is not. {pull}6627[6627] - Add rename processor. {pull}6292[6292] +- Add IP-addresses and MAC-addresses to add_host_metadata. {pull}6878[6878] *Auditbeat* From 5538abf9a7287148695fdedaf7a6bd0eee9001c7 Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Fri, 20 Apr 2018 19:46:08 +0200 Subject: [PATCH 06/20] rename to host.mac and host.ip --- libbeat/processors/add_host_metadata/_meta/fields.yml | 4 ++-- libbeat/processors/add_host_metadata/add_host_metadata.go | 4 ++-- .../processors/add_host_metadata/add_host_metadata_test.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libbeat/processors/add_host_metadata/_meta/fields.yml b/libbeat/processors/add_host_metadata/_meta/fields.yml index e9f677fe969d..b579df3b2724 100644 --- a/libbeat/processors/add_host_metadata/_meta/fields.yml +++ b/libbeat/processors/add_host_metadata/_meta/fields.yml @@ -31,11 +31,11 @@ type: keyword description: > OS family (e.g. redhat, debian, freebsd, windows). - - name: net.ip + - name: ip type: ip description: > List of IP-addresses. - - name: net.hw + - name: mac type: keyword description: > List of hardware-addresses, usually MAC-addresses. diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index 46f9b904ced3..c26808de3574 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -75,8 +75,8 @@ func (p *addHostMetadata) loadData() { // IP-address and MAC-address var ipList, hwList = p.getNetInfo() - p.data.Put("host.net.ip", ipList) - p.data.Put("host.net.hw", hwList) + p.data.Put("host.ip", ipList) + p.data.Put("host.mac", hwList) p.lastUpdate = time.Now() } diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index 1d8386647b9c..a74c02932c9a 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -32,11 +32,11 @@ func TestRun(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.net.ip") + v, err = newEvent.GetValue("host.ip") assert.NoError(t, err) assert.NotNil(t, v) - v, err = newEvent.GetValue("host.net.hw") + v, err = newEvent.GetValue("host.mac") assert.NoError(t, err) assert.NotNil(t, v) } From 65463abd2aa69ff2c93f56491f477bc60af43df3 Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Fri, 27 Apr 2018 17:06:46 +0200 Subject: [PATCH 07/20] Added configuration option netinfo.enabled for IP and MAC --- .../add_host_metadata/add_host_metadata.go | 27 ++++++++++++---- .../add_host_metadata_test.go | 32 ++++++++++++++++++- .../processors/add_host_metadata/config.go | 16 ++++++++++ 3 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 libbeat/processors/add_host_metadata/config.go diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index c26808de3574..4aeda85032cb 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -1,6 +1,7 @@ package add_host_metadata import ( + "fmt" "net" "time" @@ -9,6 +10,7 @@ import ( "github.com/elastic/beats/libbeat/processors" "github.com/elastic/go-sysinfo" "github.com/elastic/go-sysinfo/types" + "github.com/pkg/errors" ) func init() { @@ -19,19 +21,27 @@ type addHostMetadata struct { info types.HostInfo lastUpdate time.Time data common.MapStr + config Config } const ( + processorName = "add_host_metadata" cacheExpiration = time.Minute * 5 ) -func newHostMetadataProcessor(_ *common.Config) (processors.Processor, error) { +func newHostMetadataProcessor(cfg *common.Config) (processors.Processor, error) { + config := defaultConfig() + if err := cfg.Unpack(&config); err != nil { + return nil, errors.Wrapf(err, "fail to unpack the %v configuration", processorName) + } + h, err := sysinfo.Host() if err != nil { return nil, err } p := &addHostMetadata{ - info: h.Info(), + info: h.Info(), + config: config, } return p, nil } @@ -73,10 +83,12 @@ func (p *addHostMetadata) loadData() { p.data.Put("host.os.build", p.info.OS.Build) } - // IP-address and MAC-address - var ipList, hwList = p.getNetInfo() - p.data.Put("host.ip", ipList) - p.data.Put("host.mac", hwList) + if p.config.NetInfoEnabled { + // IP-address and MAC-address + var ipList, hwList = p.getNetInfo() + p.data.Put("host.ip", ipList) + p.data.Put("host.mac", hwList) + } p.lastUpdate = time.Now() } @@ -121,5 +133,6 @@ func (p addHostMetadata) getNetInfo() ([]string, []string) { } func (p addHostMetadata) String() string { - return "add_host_metadata=[]" + return fmt.Sprintf("%v=[netinfo.enabled=[%v]]", + processorName, p.config.NetInfoEnabled) } diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index a74c02932c9a..0f9feae785d0 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -18,7 +18,10 @@ func TestRun(t *testing.T) { Fields: common.MapStr{}, Timestamp: time.Now(), } - p, err := newHostMetadataProcessor(nil) + testConfig, err := common.NewConfigFrom(map[string]interface{}{}) + assert.NoError(t, err) + + p, err := newHostMetadataProcessor(testConfig) if runtime.GOOS != "windows" && runtime.GOOS != "darwin" && runtime.GOOS != "linux" { assert.IsType(t, types.ErrNotImplemented, err) return @@ -32,6 +35,33 @@ func TestRun(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, v) + v, err = newEvent.GetValue("host.ip") + assert.Error(t, err) + assert.Nil(t, v) + + v, err = newEvent.GetValue("host.mac") + assert.Error(t, err) + assert.Nil(t, v) + + event = &beat.Event{ + Fields: common.MapStr{}, + Timestamp: time.Now(), + } + testConfig, err = common.NewConfigFrom(map[string]interface{}{ + "netinfo.enabled": true, + }) + assert.NoError(t, err) + + p, err = newHostMetadataProcessor(testConfig) + if runtime.GOOS != "windows" && runtime.GOOS != "darwin" && runtime.GOOS != "linux" { + assert.IsType(t, types.ErrNotImplemented, err) + return + } + assert.NoError(t, err) + + newEvent, err = p.Run(event) + assert.NoError(t, err) + v, err = newEvent.GetValue("host.ip") assert.NoError(t, err) assert.NotNil(t, v) diff --git a/libbeat/processors/add_host_metadata/config.go b/libbeat/processors/add_host_metadata/config.go new file mode 100644 index 000000000000..29a47f7e22fd --- /dev/null +++ b/libbeat/processors/add_host_metadata/config.go @@ -0,0 +1,16 @@ +package add_host_metadata + +//import ( +// "github.com/elastic/beats/libbeat/processors" +//) + +// Config for add_host_metadata processor. +type Config struct { + NetInfoEnabled bool `config:"netinfo.enabled"` // Add IP and MAC to event +} + +func defaultConfig() Config { + return Config{ + // netInfoEnabled: "false", + } +} From 3e438f8431d59d894f71bee8f39f02cfd60afb8f Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Fri, 27 Apr 2018 17:14:02 +0200 Subject: [PATCH 08/20] Explicitly set NetInfoEnabled to false --- libbeat/processors/add_host_metadata/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/processors/add_host_metadata/config.go b/libbeat/processors/add_host_metadata/config.go index 29a47f7e22fd..575ae1a60961 100644 --- a/libbeat/processors/add_host_metadata/config.go +++ b/libbeat/processors/add_host_metadata/config.go @@ -11,6 +11,6 @@ type Config struct { func defaultConfig() Config { return Config{ - // netInfoEnabled: "false", + NetInfoEnabled: false, } } From 959e2f649281f53096f48d5b0dc0e488f4c724f5 Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Mon, 30 Apr 2018 08:59:36 +0200 Subject: [PATCH 09/20] after make fmt --- libbeat/processors/add_host_metadata/add_host_metadata.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index 4aeda85032cb..d2c3b543f891 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -5,12 +5,13 @@ import ( "net" "time" + "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/go-sysinfo" "github.com/elastic/go-sysinfo/types" - "github.com/pkg/errors" ) func init() { From 5c20a52d41be788b5fadd0ea5a4c29771c18de16 Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Mon, 30 Apr 2018 09:04:17 +0200 Subject: [PATCH 10/20] split testcase into two --- .../add_host_metadata/add_host_metadata_test.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index 0f9feae785d0..d8e42594c82c 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -13,7 +13,7 @@ import ( "github.com/elastic/go-sysinfo/types" ) -func TestRun(t *testing.T) { +func TestConfigDefault(t *testing.T) { event := &beat.Event{ Fields: common.MapStr{}, Timestamp: time.Now(), @@ -42,25 +42,31 @@ func TestRun(t *testing.T) { v, err = newEvent.GetValue("host.mac") assert.Error(t, err) assert.Nil(t, v) +} - event = &beat.Event{ +func TestConfigNetInfoEnabled(t *testing.T) { + event := &beat.Event{ Fields: common.MapStr{}, Timestamp: time.Now(), } - testConfig, err = common.NewConfigFrom(map[string]interface{}{ + testConfig, err := common.NewConfigFrom(map[string]interface{}{ "netinfo.enabled": true, }) assert.NoError(t, err) - p, err = newHostMetadataProcessor(testConfig) + p, err := newHostMetadataProcessor(testConfig) if runtime.GOOS != "windows" && runtime.GOOS != "darwin" && runtime.GOOS != "linux" { assert.IsType(t, types.ErrNotImplemented, err) return } assert.NoError(t, err) - newEvent, err = p.Run(event) + newEvent, err := p.Run(event) + assert.NoError(t, err) + + v, err := newEvent.GetValue("host.os.family") assert.NoError(t, err) + assert.NotNil(t, v) v, err = newEvent.GetValue("host.ip") assert.NoError(t, err) From df588257938bc4167cc2af56d5c405a055182ba3 Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Mon, 30 Apr 2018 10:17:43 +0200 Subject: [PATCH 11/20] after make update --- auditbeat/docs/fields.asciidoc | 20 ++++++++++++++++++++ filebeat/docs/fields.asciidoc | 20 ++++++++++++++++++++ heartbeat/docs/fields.asciidoc | 20 ++++++++++++++++++++ metricbeat/docs/fields.asciidoc | 20 ++++++++++++++++++++ packetbeat/docs/fields.asciidoc | 20 ++++++++++++++++++++ winlogbeat/docs/fields.asciidoc | 20 ++++++++++++++++++++ 6 files changed, 120 insertions(+) diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc index b68b03182855..29f229e38f2a 100644 --- a/auditbeat/docs/fields.asciidoc +++ b/auditbeat/docs/fields.asciidoc @@ -3258,6 +3258,26 @@ type: keyword OS family (e.g. redhat, debian, freebsd, windows). +-- + +*`host.ip`*:: ++ +-- +type: ip + +List of IP-addresses. + + +-- + +*`host.mac`*:: ++ +-- +type: keyword + +List of hardware-addresses, usually MAC-addresses. + + -- [[exported-fields-kubernetes-processor]] diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 323759c6c054..ca69a19e7af8 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -851,6 +851,26 @@ type: keyword OS family (e.g. redhat, debian, freebsd, windows). +-- + +*`host.ip`*:: ++ +-- +type: ip + +List of IP-addresses. + + +-- + +*`host.mac`*:: ++ +-- +type: keyword + +List of hardware-addresses, usually MAC-addresses. + + -- [[exported-fields-icinga]] diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc index 679dc6051296..c353af7c42e8 100644 --- a/heartbeat/docs/fields.asciidoc +++ b/heartbeat/docs/fields.asciidoc @@ -421,6 +421,26 @@ type: keyword OS family (e.g. redhat, debian, freebsd, windows). +-- + +*`host.ip`*:: ++ +-- +type: ip + +List of IP-addresses. + + +-- + +*`host.mac`*:: ++ +-- +type: keyword + +List of hardware-addresses, usually MAC-addresses. + + -- [[exported-fields-http]] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index f12ca0cf74c0..d9e61cf04208 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -5653,6 +5653,26 @@ type: keyword OS family (e.g. redhat, debian, freebsd, windows). +-- + +*`host.ip`*:: ++ +-- +type: ip + +List of IP-addresses. + + +-- + +*`host.mac`*:: ++ +-- +type: keyword + +List of hardware-addresses, usually MAC-addresses. + + -- [[exported-fields-http]] diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index f3d6ae692c6b..9c405c0906dc 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -2316,6 +2316,26 @@ type: keyword OS family (e.g. redhat, debian, freebsd, windows). +-- + +*`host.ip`*:: ++ +-- +type: ip + +List of IP-addresses. + + +-- + +*`host.mac`*:: ++ +-- +type: keyword + +List of hardware-addresses, usually MAC-addresses. + + -- [[exported-fields-http]] diff --git a/winlogbeat/docs/fields.asciidoc b/winlogbeat/docs/fields.asciidoc index 89fc00914d38..48b30ad9c3a9 100644 --- a/winlogbeat/docs/fields.asciidoc +++ b/winlogbeat/docs/fields.asciidoc @@ -634,6 +634,26 @@ type: keyword OS family (e.g. redhat, debian, freebsd, windows). +-- + +*`host.ip`*:: ++ +-- +type: ip + +List of IP-addresses. + + +-- + +*`host.mac`*:: ++ +-- +type: keyword + +List of hardware-addresses, usually MAC-addresses. + + -- [[exported-fields-kubernetes-processor]] From 1b09cd61f18db454f04de59db8fc0265fe8dfbba Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Wed, 2 May 2018 17:38:43 +0200 Subject: [PATCH 12/20] updated docs --- libbeat/_meta/config.reference.yml | 5 ++++- libbeat/docs/processors-using.asciidoc | 14 +++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index 986aab97f6c4..564086738617 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -183,7 +183,10 @@ # #processors: #- add_docker_metadata: ~ -#- add_host_metadata: ~ +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false #============================= Elastic Cloud ================================== diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index 49d72f5051c7..7ec19edda863 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -727,6 +727,16 @@ forget metadata for a container, 60s by default. beta[] +[source,yaml] +------------------------------------------------------------------------------- +processors: +- add_host_metadata: + netinfo.enabled: false + +It has the following settings: + +`netinfo.enabled`:: (Optional) Default false. Include IP adresses and MAC addresses as fields host.ip and host.mac + The `add_host_metadata` processor annotates each event with relevant metadata from the host machine. The fields added to the event are looking as following: @@ -742,7 +752,9 @@ The fields added to the event are looking as following: "build":"16G1212", "platform":"darwin", "version":"10.12.6" - } + }, + ip: ["192.168.0.1", "10.0.0.1"], + mac: ["00:25:96:12:34:56", "72:00:06:ff:79:f1"] } } ------------------------------------------------------------------------------- From 1cba03f349b386fdfa026de93a107a1db1eae7db Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Wed, 2 May 2018 17:46:13 +0200 Subject: [PATCH 13/20] after make update --- auditbeat/auditbeat.reference.yml | 5 ++++- filebeat/filebeat.reference.yml | 5 ++++- heartbeat/heartbeat.reference.yml | 5 ++++- metricbeat/metricbeat.reference.yml | 5 ++++- packetbeat/packetbeat.reference.yml | 5 ++++- winlogbeat/winlogbeat.reference.yml | 5 ++++- 6 files changed, 24 insertions(+), 6 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index cbce75f67aff..5208c7ace91f 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -288,7 +288,10 @@ auditbeat.modules: # #processors: #- add_docker_metadata: ~ -#- add_host_metadata: ~ +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false #============================= Elastic Cloud ================================== diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index bbca33f8e09f..d1ad30b9a907 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -806,7 +806,10 @@ filebeat.inputs: # #processors: #- add_docker_metadata: ~ -#- add_host_metadata: ~ +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false #============================= Elastic Cloud ================================== diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 1acb33bbda8f..a2eb91cf82fb 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -397,7 +397,10 @@ heartbeat.scheduler: # #processors: #- add_docker_metadata: ~ -#- add_host_metadata: ~ +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false #============================= Elastic Cloud ================================== diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 049ce10527e1..f43b58081fbe 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -763,7 +763,10 @@ metricbeat.modules: # #processors: #- add_docker_metadata: ~ -#- add_host_metadata: ~ +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false #============================= Elastic Cloud ================================== diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 88c5013f6140..b4863faee205 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -660,7 +660,10 @@ packetbeat.protocols: # #processors: #- add_docker_metadata: ~ -#- add_host_metadata: ~ +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false #============================= Elastic Cloud ================================== diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index cbd999a7a863..48ce0068ee48 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -212,7 +212,10 @@ winlogbeat.event_logs: # #processors: #- add_docker_metadata: ~ -#- add_host_metadata: ~ +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false #============================= Elastic Cloud ================================== From dfef568c1941c6a29235c2c6afe1ccba619487b2 Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Wed, 2 May 2018 21:04:59 +0200 Subject: [PATCH 14/20] yet another make update --- auditbeat/auditbeat.reference.yml | 3 +++ filebeat/filebeat.reference.yml | 3 +++ heartbeat/heartbeat.reference.yml | 3 +++ libbeat/_meta/config.reference.yml | 3 +++ metricbeat/metricbeat.reference.yml | 3 +++ packetbeat/packetbeat.reference.yml | 3 +++ winlogbeat/winlogbeat.reference.yml | 3 +++ 7 files changed, 21 insertions(+) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 5208c7ace91f..c7deedb152a2 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -289,9 +289,12 @@ auditbeat.modules: #processors: #- add_docker_metadata: ~ # +# The following example enriches each event with host metadata. +# #processors: #- add_host_metadata: # netinfo.enabled: false +# #============================= Elastic Cloud ================================== diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index d1ad30b9a907..37058c349808 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -807,9 +807,12 @@ filebeat.inputs: #processors: #- add_docker_metadata: ~ # +# The following example enriches each event with host metadata. +# #processors: #- add_host_metadata: # netinfo.enabled: false +# #============================= Elastic Cloud ================================== diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index a2eb91cf82fb..4eae7eb17f8d 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -398,9 +398,12 @@ heartbeat.scheduler: #processors: #- add_docker_metadata: ~ # +# The following example enriches each event with host metadata. +# #processors: #- add_host_metadata: # netinfo.enabled: false +# #============================= Elastic Cloud ================================== diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index 564086738617..ee10b06e1855 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -184,9 +184,12 @@ #processors: #- add_docker_metadata: ~ # +# The following example enriches each event with host metadata. +# #processors: #- add_host_metadata: # netinfo.enabled: false +# #============================= Elastic Cloud ================================== diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index f43b58081fbe..9f9a865af1dc 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -764,9 +764,12 @@ metricbeat.modules: #processors: #- add_docker_metadata: ~ # +# The following example enriches each event with host metadata. +# #processors: #- add_host_metadata: # netinfo.enabled: false +# #============================= Elastic Cloud ================================== diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index b4863faee205..f8773f3f62a4 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -661,9 +661,12 @@ packetbeat.protocols: #processors: #- add_docker_metadata: ~ # +# The following example enriches each event with host metadata. +# #processors: #- add_host_metadata: # netinfo.enabled: false +# #============================= Elastic Cloud ================================== diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 48ce0068ee48..c2e7fed1b8d9 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -213,9 +213,12 @@ winlogbeat.event_logs: #processors: #- add_docker_metadata: ~ # +# The following example enriches each event with host metadata. +# #processors: #- add_host_metadata: # netinfo.enabled: false +# #============================= Elastic Cloud ================================== From 1ef240d62b979fae931e0d968e1ac11f35f8d050 Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Wed, 2 May 2018 21:05:49 +0200 Subject: [PATCH 15/20] removed left over comment --- libbeat/processors/add_host_metadata/config.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libbeat/processors/add_host_metadata/config.go b/libbeat/processors/add_host_metadata/config.go index 575ae1a60961..5ead35b4c0b6 100644 --- a/libbeat/processors/add_host_metadata/config.go +++ b/libbeat/processors/add_host_metadata/config.go @@ -1,9 +1,5 @@ package add_host_metadata -//import ( -// "github.com/elastic/beats/libbeat/processors" -//) - // Config for add_host_metadata processor. type Config struct { NetInfoEnabled bool `config:"netinfo.enabled"` // Add IP and MAC to event From 7c44e2adbd8f20d15d03d8c8717bb9d5d88f1b77 Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Wed, 2 May 2018 22:06:00 +0200 Subject: [PATCH 16/20] added logging on error --- .../add_host_metadata/add_host_metadata.go | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index d2c3b543f891..10e32aa14efd 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -9,6 +9,7 @@ import ( "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/go-sysinfo" "github.com/elastic/go-sysinfo/types" @@ -86,23 +87,31 @@ func (p *addHostMetadata) loadData() { if p.config.NetInfoEnabled { // IP-address and MAC-address - var ipList, hwList = p.getNetInfo() - p.data.Put("host.ip", ipList) - p.data.Put("host.mac", hwList) + var ipList, hwList, err = p.getNetInfo() + if err != nil { + logp.Warn("Error when getting network information %v", err) + } + + if len(ipList) > 0 { + p.data.Put("host.ip", ipList) + } + if len(hwList) > 0 { + p.data.Put("host.mac", hwList) + } } p.lastUpdate = time.Now() } } -func (p addHostMetadata) getNetInfo() ([]string, []string) { +func (p addHostMetadata) getNetInfo() ([]string, []string, error) { var ipList []string var hwList []string // Get all interfaces and loop through them ifaces, err := net.Interfaces() if err != nil { - return ipList, hwList + return ipList, hwList, err } for _, i := range ifaces { // Skip loopback interfaces @@ -118,8 +127,11 @@ func (p addHostMetadata) getNetInfo() ([]string, []string) { addrs, err := i.Addrs() if err != nil { - return ipList, hwList + // If we get an error, log it and continue with the next interface + logp.Warn("Error when getting IP address %v", err) + continue } + for _, addr := range addrs { switch v := addr.(type) { case *net.IPNet: @@ -130,7 +142,7 @@ func (p addHostMetadata) getNetInfo() ([]string, []string) { } } - return ipList, hwList + return ipList, hwList, nil } func (p addHostMetadata) String() string { From e909f81e0ff1d54f4f6691c6dde2248f67800715 Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Thu, 3 May 2018 21:54:45 +0200 Subject: [PATCH 17/20] use Info instead of Warn --- libbeat/processors/add_host_metadata/add_host_metadata.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index 10e32aa14efd..fcd5f1b7b87c 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -89,7 +89,7 @@ func (p *addHostMetadata) loadData() { // IP-address and MAC-address var ipList, hwList, err = p.getNetInfo() if err != nil { - logp.Warn("Error when getting network information %v", err) + logp.Info("Error when getting network information %v", err) } if len(ipList) > 0 { @@ -128,7 +128,7 @@ func (p addHostMetadata) getNetInfo() ([]string, []string, error) { addrs, err := i.Addrs() if err != nil { // If we get an error, log it and continue with the next interface - logp.Warn("Error when getting IP address %v", err) + logp.Info("Error when getting IP address %v", err) continue } From dde24f7611c5ba97bd0f512359eab9839b5b91be Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Thu, 3 May 2018 21:56:17 +0200 Subject: [PATCH 18/20] added missing end of source --- libbeat/docs/processors-using.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index 7ec19edda863..71a0874709a4 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -732,6 +732,7 @@ beta[] processors: - add_host_metadata: netinfo.enabled: false +------------------------------------------------------------------------------- It has the following settings: From 993b2bc9388f245ee7d51c8a1773c3c1d25980b8 Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Thu, 3 May 2018 21:58:44 +0200 Subject: [PATCH 19/20] return nil, nil instead of ipList, hwList --- libbeat/processors/add_host_metadata/add_host_metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index fcd5f1b7b87c..ae296e6407ff 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -111,7 +111,7 @@ func (p addHostMetadata) getNetInfo() ([]string, []string, error) { // Get all interfaces and loop through them ifaces, err := net.Interfaces() if err != nil { - return ipList, hwList, err + return nil, nil, err } for _, i := range ifaces { // Skip loopback interfaces From 014f5d5732a3b21bd6e646f9a688c20febb89e13 Mon Sep 17 00:00:00 2001 From: Mathias Olsson Date: Thu, 3 May 2018 22:06:23 +0200 Subject: [PATCH 20/20] keep track of all errors using multierror --- .../processors/add_host_metadata/add_host_metadata.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index ae296e6407ff..f27557e7d136 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -5,6 +5,7 @@ import ( "net" "time" + "github.com/joeshaw/multierror" "github.com/pkg/errors" "github.com/elastic/beats/libbeat/beat" @@ -113,6 +114,10 @@ func (p addHostMetadata) getNetInfo() ([]string, []string, error) { if err != nil { return nil, nil, err } + + // Keep track of all errors + var errs multierror.Errors + for _, i := range ifaces { // Skip loopback interfaces if i.Flags&net.FlagLoopback == net.FlagLoopback { @@ -127,8 +132,8 @@ func (p addHostMetadata) getNetInfo() ([]string, []string, error) { addrs, err := i.Addrs() if err != nil { - // If we get an error, log it and continue with the next interface - logp.Info("Error when getting IP address %v", err) + // If we get an error, keep track of it and continue with the next interface + errs = append(errs, err) continue } @@ -142,7 +147,7 @@ func (p addHostMetadata) getNetInfo() ([]string, []string, error) { } } - return ipList, hwList, nil + return ipList, hwList, errs.Err() } func (p addHostMetadata) String() string {