From 2cca46ed1de46b52fd9befbce98834f53ab20133 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Wed, 9 May 2018 12:51:53 +0200 Subject: [PATCH 1/9] Add host.name in the events As a solution for #7050, we adding a `host.name` field to all events. This is duplicate information from `beat.name`, but is used to avoid the mapping conflict and to slowly introduce the "host as an object" approach. To remove the duplication, you can remove `beat.name` like this: processors: - drop_fields.fields: ["beat.name"] Closes #7050. --- libbeat/publisher/pipeline/module.go | 3 +++ libbeat/publisher/pipeline/pipeline.go | 6 ++++++ libbeat/publisher/pipeline/processor.go | 5 ++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/libbeat/publisher/pipeline/module.go b/libbeat/publisher/pipeline/module.go index 594ce572a17..a86e6d65c03 100644 --- a/libbeat/publisher/pipeline/module.go +++ b/libbeat/publisher/pipeline/module.go @@ -55,6 +55,9 @@ func Load( "hostname": beatInfo.Hostname, "version": beatInfo.Version, }, + Host: common.MapStr{ + "name": name, + }, }, } diff --git a/libbeat/publisher/pipeline/pipeline.go b/libbeat/publisher/pipeline/pipeline.go index fb468d2a074..e2179e5e07c 100644 --- a/libbeat/publisher/pipeline/pipeline.go +++ b/libbeat/publisher/pipeline/pipeline.go @@ -63,6 +63,7 @@ type pipelineProcessors struct { // constructing the clients complete processor // pipeline on connect. beatsMeta common.MapStr + hostMeta common.MapStr fields common.MapStr tags []string @@ -93,6 +94,7 @@ type Settings struct { type Annotations struct { Beat common.MapStr Event common.EventMetadata + Host common.MapStr } // WaitCloseMode enumerates the possible behaviors of WaitClose in a pipeline. @@ -405,6 +407,10 @@ func makePipelineProcessors( p.beatsMeta = common.MapStr{"beat": meta} } + if meta := annotations.Host; meta != nil { + p.hostMeta = common.MapStr{"host": meta} + } + if em := annotations.Event; len(em.Fields) > 0 { fields := common.MapStr{} common.MergeFields(fields, em.Fields.Clone(), em.FieldsUnderRoot) diff --git a/libbeat/publisher/pipeline/processor.go b/libbeat/publisher/pipeline/processor.go index e8e68fbbd14..eab6fb5a551 100644 --- a/libbeat/publisher/pipeline/processor.go +++ b/libbeat/publisher/pipeline/processor.go @@ -99,10 +99,13 @@ func newProcessorPipeline( // setup 5: client processor list processors.add(localProcessors) - // setup 6: add beats metadata + // setup 6: add beats and metadata if meta := global.beatsMeta; len(meta) > 0 { processors.add(makeAddFieldsProcessor("beatsMeta", meta, needsCopy)) } + if meta := global.hostMeta; len(meta) > 0 { + processors.add(makeAddFieldsProcessor("hostMeta", meta, needsCopy)) + } // setup 7: pipeline processors list processors.add(global.processors) From 9c60a5c52ba709bb5a000ef24d5095405f54e327 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Wed, 9 May 2018 12:59:09 +0200 Subject: [PATCH 2/9] fix comment --- libbeat/publisher/pipeline/processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/publisher/pipeline/processor.go b/libbeat/publisher/pipeline/processor.go index eab6fb5a551..ee97ef70d00 100644 --- a/libbeat/publisher/pipeline/processor.go +++ b/libbeat/publisher/pipeline/processor.go @@ -99,7 +99,7 @@ func newProcessorPipeline( // setup 5: client processor list processors.add(localProcessors) - // setup 6: add beats and metadata + // setup 6: add beats and host metadata if meta := global.beatsMeta; len(meta) > 0 { processors.add(makeAddFieldsProcessor("beatsMeta", meta, needsCopy)) } From 49a1537d5094db983e51dc5ee2b857ee8fc6943f Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Thu, 10 May 2018 00:18:12 +0200 Subject: [PATCH 3/9] Addressed comments --- libbeat/publisher/pipeline/module.go | 16 +++++++++------- libbeat/publisher/pipeline/pipeline.go | 20 +++++++------------- libbeat/publisher/pipeline/processor.go | 5 +---- 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/libbeat/publisher/pipeline/module.go b/libbeat/publisher/pipeline/module.go index a86e6d65c03..6bcefa0a28e 100644 --- a/libbeat/publisher/pipeline/module.go +++ b/libbeat/publisher/pipeline/module.go @@ -50,13 +50,15 @@ func Load( Processors: processors, Annotations: Annotations{ Event: config.EventMetadata, - Beat: common.MapStr{ - "name": name, - "hostname": beatInfo.Hostname, - "version": beatInfo.Version, - }, - Host: common.MapStr{ - "name": name, + Builtin: common.MapStr{ + "beat": common.MapStr{ + "name": name, + "hostname": beatInfo.Hostname, + "version": beatInfo.Version, + }, + "host": common.MapStr{ + "name": name, + }, }, }, } diff --git a/libbeat/publisher/pipeline/pipeline.go b/libbeat/publisher/pipeline/pipeline.go index e2179e5e07c..8ffa191180a 100644 --- a/libbeat/publisher/pipeline/pipeline.go +++ b/libbeat/publisher/pipeline/pipeline.go @@ -62,10 +62,9 @@ type pipelineProcessors struct { // The pipeline its processor settings for // constructing the clients complete processor // pipeline on connect. - beatsMeta common.MapStr - hostMeta common.MapStr - fields common.MapStr - tags []string + builtinMeta common.MapStr + fields common.MapStr + tags []string processors beat.Processor @@ -92,9 +91,8 @@ type Settings struct { // processors, so all processors configured with the pipeline or client will see // the same/complete event. type Annotations struct { - Beat common.MapStr - Event common.EventMetadata - Host common.MapStr + Event common.EventMetadata + Builtin common.MapStr } // WaitCloseMode enumerates the possible behaviors of WaitClose in a pipeline. @@ -403,12 +401,8 @@ func makePipelineProcessors( p.processors = tmp } - if meta := annotations.Beat; meta != nil { - p.beatsMeta = common.MapStr{"beat": meta} - } - - if meta := annotations.Host; meta != nil { - p.hostMeta = common.MapStr{"host": meta} + if meta := annotations.Builtin; meta != nil { + p.builtinMeta = meta } if em := annotations.Event; len(em.Fields) > 0 { diff --git a/libbeat/publisher/pipeline/processor.go b/libbeat/publisher/pipeline/processor.go index ee97ef70d00..5f058cdde81 100644 --- a/libbeat/publisher/pipeline/processor.go +++ b/libbeat/publisher/pipeline/processor.go @@ -100,12 +100,9 @@ func newProcessorPipeline( processors.add(localProcessors) // setup 6: add beats and host metadata - if meta := global.beatsMeta; len(meta) > 0 { + if meta := global.builtinMeta; len(meta) > 0 { processors.add(makeAddFieldsProcessor("beatsMeta", meta, needsCopy)) } - if meta := global.hostMeta; len(meta) > 0 { - processors.add(makeAddFieldsProcessor("hostMeta", meta, needsCopy)) - } // setup 7: pipeline processors list processors.add(global.processors) From b2c6770e4b09073103694688ec1f17193bee5e86 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Thu, 10 May 2018 00:21:02 +0200 Subject: [PATCH 4/9] changelog --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index fdf43bcdcdb..9bd6b024bb4 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -126,6 +126,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff] - Add a default seccomp (secure computing) filter on Linux that prohibits execve, execveat, fork, and vfork syscalls. A custom policy can be configured. {issue}5213[5213] - Update Sarama to v1.16.0, adding better support for kafka 0.11, 1.0, and 1.1 {pull}7025[7025] +- Add `host.name` field to all events, to avoid mapping conflicts and breaking compatibility. {pull}7051[7051] *Auditbeat* From 6be41ce4fec539346b1ed228ebc8afd443c0743d Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Thu, 10 May 2018 00:35:31 +0200 Subject: [PATCH 5/9] moved entry into breaking changes --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 9bd6b024bb4..a67e4d41d8d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -18,6 +18,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff] - Mark `system.syslog.message` and `system.auth.message` as `text` instead of `keyword`. {pull}6589[6589] - Allow override of dynamic template `match_mapping_type` for fields with object_type. {pull}6691[6691] - Set default kafka version to 1.0.0 in kafka output. Older versions are still supported by configuring the `version` setting. {pull}7025[7025] +- Add `host.name` field to all events, to avoid mapping conflicts. This could be breaking Logstash configs if you rely on the `host` field being a string. {pull}7051[7051] *Auditbeat* From 122981d8050fe965e4f864ca137e35094e8bf0d7 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Thu, 10 May 2018 00:39:05 +0200 Subject: [PATCH 6/9] removed duplicate --- CHANGELOG.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index a67e4d41d8d..837cb9caf88 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -127,7 +127,6 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff] - Add a default seccomp (secure computing) filter on Linux that prohibits execve, execveat, fork, and vfork syscalls. A custom policy can be configured. {issue}5213[5213] - Update Sarama to v1.16.0, adding better support for kafka 0.11, 1.0, and 1.1 {pull}7025[7025] -- Add `host.name` field to all events, to avoid mapping conflicts and breaking compatibility. {pull}7051[7051] *Auditbeat* From 0eb69fd167602c88e7e9b6da0bb3cd3a07ba2806 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Thu, 10 May 2018 00:48:12 +0200 Subject: [PATCH 7/9] fix test --- metricbeat/tests/system/test_processors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/tests/system/test_processors.py b/metricbeat/tests/system/test_processors.py index 840a7fefdba..e9192d092d9 100644 --- a/metricbeat/tests/system/test_processors.py +++ b/metricbeat/tests/system/test_processors.py @@ -35,7 +35,7 @@ def test_drop_fields(self): print(evt.keys()) self.assertItemsEqual(self.de_dot([ 'beat', '@timestamp', 'system', 'metricset.module', - 'metricset.rtt', 'metricset.name' + 'metricset.rtt', 'metricset.name', 'host' ]), evt.keys()) cpu = evt["system"]["cpu"] print(cpu.keys()) From 074008c81db7d8e292ca69ce1c33b334bc4d3d77 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Thu, 10 May 2018 08:55:33 +0200 Subject: [PATCH 8/9] another test fix --- metricbeat/tests/system/metricbeat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/tests/system/metricbeat.py b/metricbeat/tests/system/metricbeat.py index f46d8fabf99..74fe07c7ab7 100644 --- a/metricbeat/tests/system/metricbeat.py +++ b/metricbeat/tests/system/metricbeat.py @@ -7,7 +7,7 @@ from beat.beat import TestCase COMMON_FIELDS = ["@timestamp", "beat", "metricset.name", "metricset.host", - "metricset.module", "metricset.rtt"] + "metricset.module", "metricset.rtt", "host.name"] INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) From a4bdd25cb3fb1073ada87150b0d78314fc19b40c Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Thu, 10 May 2018 10:51:40 +0200 Subject: [PATCH 9/9] trying a fix for the windows tests --- dev-tools/jenkins_ci.ps1 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dev-tools/jenkins_ci.ps1 b/dev-tools/jenkins_ci.ps1 index 35e7ee8d362..4db34c03bc8 100755 --- a/dev-tools/jenkins_ci.ps1 +++ b/dev-tools/jenkins_ci.ps1 @@ -36,13 +36,14 @@ exec { go get -u github.com/jstemmer/go-junit-report } echo "Building $env:beat" exec { go build } "Build FAILURE" +# always build the libbeat fields +cp ..\libbeat\_meta\fields.common.yml ..\libbeat\_meta\fields.generated.yml +cat ..\libbeat\processors\*\_meta\fields.yml | Out-File -append -encoding UTF8 -filepath ..\libbeat\_meta\fields.generated.yml +cp ..\libbeat\_meta\fields.generated.yml ..\libbeat\fields.yml + if ($env:beat -eq "metricbeat") { cp .\_meta\fields.common.yml .\_meta\fields.generated.yml python .\scripts\fields_collector.py | out-file -append -encoding UTF8 -filepath .\_meta\fields.generated.yml -} elseif ($env:beat -eq "libbeat") { - cp .\_meta\fields.common.yml .\_meta\fields.generated.yml - cat processors\*\_meta\fields.yml | Out-File -append -encoding UTF8 -filepath .\_meta\fields.generated.yml - cp .\_meta\fields.generated.yml .\fields.yml } echo "Unit testing $env:beat"