Skip to content

Commit

Permalink
Improve ECS categorization field mappings in logstash module (elastic…
Browse files Browse the repository at this point in the history
…#16668)

- event.kind
- event.type
- convert pipeline to yaml

Closes elastic#16169
  • Loading branch information
leehinman authored Mar 6, 2020
1 parent 5c38912 commit 8e8da46
Show file tree
Hide file tree
Showing 16 changed files with 260 additions and 310 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Improve ECS categorization field mapping in kafka module. {issue}16167[16167] {pull}16645[16645]
- Allow users to override pipeline ID in fileset input config. {issue}9531[9531] {pull}16561[16561]
- Add `o365audit` input type for consuming events from Office 365 Management Activity API. {issue}16196[16196] {pull}16244[16244]
- Improve ECS categorization field mappings in logstash module. {issue}16169[16169] {pull}16668[16668]

*Heartbeat*

Expand Down
72 changes: 0 additions & 72 deletions filebeat/module/logstash/log/ingest/pipeline-json.json

This file was deleted.

50 changes: 50 additions & 0 deletions filebeat/module/logstash/log/ingest/pipeline-json.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
description: Pipeline for parsing logstash logs
on_failure:
- set:
field: error.message
value: '{{ _ingest.on_failure_message }}'
processors:
- json:
field: message
target_field: logstash.log
- rename:
field: '@timestamp'
target_field: event.created
- convert:
field: logstash.log.timeMillis
type: string
- date:
field: logstash.log.timeMillis
formats:
- UNIX_MS
target_field: '@timestamp'
- rename:
field: logstash.log.loggerName
target_field: logstash.log.module
- remove:
field:
- message
- logstash.log.timeMillis
- rename:
field: logstash.log.logEvent.message
target_field: message
- rename:
field: logstash.log.logEvent
target_field: logstash.log.log_event
- rename:
field: logstash.log.level
target_field: log.level
- set:
field: event.kind
value: event
- script:
lang: painless
source: >-
def errorLevels = ["ERROR", "FATAL"];
if (ctx?.log?.level != null) {
if (errorLevels.contains(ctx.log.level)) {
ctx.event.type = "error";
} else {
ctx.event.type = "info";
}
}
57 changes: 0 additions & 57 deletions filebeat/module/logstash/log/ingest/pipeline-plain.json

This file was deleted.

59 changes: 59 additions & 0 deletions filebeat/module/logstash/log/ingest/pipeline-plain.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
description: Pipeline for parsing logstash logs in the plain format
on_failure:
- set:
field: error.message
value: '{{ _ingest.on_failure_message }}'
processors:
- grok:
field: message
pattern_definitions:
LOGSTASH_CLASS_MODULE: '[\w\.]+'
LOGSTASH_LOGLEVEL: INFO|ERROR|DEBUG|FATAL|WARN|TRACE
GREEDYMULTILINE: |-
(.|
)*
patterns:
- \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\]\[%{WORD:logstash.log.pipeline_id}\]
%{GREEDYMULTILINE:message}
- \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\]
%{GREEDYMULTILINE:message}
- rename:
field: '@timestamp'
target_field: event.created
- date:
if: ctx.event.timezone == null
field: logstash.log.timestamp
target_field: '@timestamp'
formats:
- yyyy-MM-dd'T'HH:mm:ss,SSS
on_failure:
- append:
field: error.message
value: '{{ _ingest.on_failure_message }}'
- date:
if: ctx.event.timezone != null
field: logstash.log.timestamp
target_field: '@timestamp'
formats:
- yyyy-MM-dd'T'HH:mm:ss,SSS
timezone: '{{ event.timezone }}'
on_failure:
- append:
field: error.message
value: '{{ _ingest.on_failure_message }}'
- remove:
field: logstash.log.timestamp
- set:
field: event.kind
value: event
- script:
lang: painless
source: >-
def errorLevels = ["ERROR", "FATAL"];
if (ctx?.log?.level != null) {
if (errorLevels.contains(ctx.log.level)) {
ctx.event.type = "error";
} else {
ctx.event.type = "info";
}
}
2 changes: 1 addition & 1 deletion filebeat/module/logstash/log/manifest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ var:
os.windows:
- c:/programdata/logstash/logs/logstash-{{.format}}*.log

ingest_pipeline: ingest/pipeline-{{.format}}.json
ingest_pipeline: ingest/pipeline-{{.format}}.yml
input: config/log.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
{
"@timestamp": "2019-01-07T21:25:21.871Z",
"event.dataset": "logstash.log",
"event.kind": "event",
"event.module": "logstash",
"event.type": "info",
"fileset.name": "log",
"input.type": "log",
"log.level": "INFO",
Expand All @@ -28,7 +30,9 @@
{
"@timestamp": "2019-01-07T21:25:22.538Z",
"event.dataset": "logstash.log",
"event.kind": "event",
"event.module": "logstash",
"event.type": "info",
"fileset.name": "log",
"input.type": "log",
"log.level": "INFO",
Expand All @@ -43,7 +47,9 @@
{
"@timestamp": "2019-01-07T21:25:22.594Z",
"event.dataset": "logstash.log",
"event.kind": "event",
"event.module": "logstash",
"event.type": "info",
"fileset.name": "log",
"input.type": "log",
"log.level": "INFO",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@
{
"@timestamp": "2019-11-20T19:04:48.468-02:00",
"event.dataset": "logstash.log",
"event.kind": "event",
"event.module": "logstash",
"event.timezone": "-02:00",
"event.type": "info",
"fileset.name": "log",
"input.type": "log",
"log.level": "WARN",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@
{
"@timestamp": "2017-10-23T14:20:12.046-02:00",
"event.dataset": "logstash.log",
"event.kind": "event",
"event.module": "logstash",
"event.timezone": "-02:00",
"event.type": "info",
"fileset.name": "log",
"input.type": "log",
"log.level": "INFO",
Expand All @@ -15,8 +17,10 @@
{
"@timestamp": "2017-11-20T03:55:00.318-02:00",
"event.dataset": "logstash.log",
"event.kind": "event",
"event.module": "logstash",
"event.timezone": "-02:00",
"event.type": "info",
"fileset.name": "log",
"input.type": "log",
"log.flags": [
Expand Down
Loading

0 comments on commit 8e8da46

Please sign in to comment.