forked from elastic/beats
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add autonomous system info to Filebeat modules
Update module pipelines to enrich events with autonomous system info (number and org name). All of the modules that had a geoip processor now also add autonomous system info. Field naming is based on ECS (elastic/ecs#341).
- Loading branch information
1 parent
83f18e9
commit d152683
Showing
48 changed files
with
1,978 additions
and
292 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,60 +1,99 @@ | ||
| { | ||
| "description": "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins.", | ||
| "processors": [{ | ||
| "grok": { | ||
| "field": "message", | ||
| "patterns":[ | ||
| "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?", | ||
| "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -", | ||
| "\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:http.response.body.bytes:long}" | ||
| ], | ||
| "ignore_missing": true | ||
| } | ||
| },{ | ||
| "remove":{ | ||
| "field": "message" | ||
| } | ||
| }, { | ||
| "grok": { | ||
| "field": "source.address", | ||
| "ignore_missing": true, | ||
| "patterns": [ | ||
| "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" | ||
| ] | ||
| } | ||
| }, { | ||
| "rename": { | ||
| "field": "@timestamp", | ||
| "target_field": "event.created" | ||
| } | ||
| }, { | ||
| "date": { | ||
| "field": "apache.access.time", | ||
| "target_field": "@timestamp", | ||
| "formats": ["dd/MMM/yyyy:H:m:s Z"], | ||
| "ignore_failure": true | ||
| } | ||
| }, { | ||
| "remove": { | ||
| "field": "apache.access.time", | ||
| "ignore_failure": true | ||
| } | ||
| }, { | ||
| "user_agent": { | ||
| "field": "user_agent.original", | ||
| "ignore_failure": true | ||
| } | ||
| }, { | ||
| "geoip": { | ||
| "field": "source.ip", | ||
| "target_field": "source.geo", | ||
| "ignore_missing": true | ||
| } | ||
| }], | ||
| "on_failure" : [{ | ||
| "set" : { | ||
| "field" : "error.message", | ||
| "value" : "{{ _ingest.on_failure_message }}" | ||
| } | ||
| }] | ||
| "description": "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins.", | ||
| "processors": [ | ||
| { | ||
| "grok": { | ||
| "field": "message", | ||
| "patterns": [ | ||
| "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?", | ||
| "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -", | ||
| "\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:http.response.body.bytes:long}" | ||
| ], | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| { | ||
| "remove": { | ||
| "field": "message" | ||
| } | ||
| }, | ||
| { | ||
| "grok": { | ||
| "field": "source.address", | ||
| "ignore_missing": true, | ||
| "patterns": [ | ||
| "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" | ||
| ] | ||
| } | ||
| }, | ||
| { | ||
| "rename": { | ||
| "field": "@timestamp", | ||
| "target_field": "event.created" | ||
| } | ||
| }, | ||
| { | ||
| "date": { | ||
| "field": "apache.access.time", | ||
| "target_field": "@timestamp", | ||
| "formats": [ | ||
| "dd/MMM/yyyy:H:m:s Z" | ||
| ], | ||
| "ignore_failure": true | ||
| } | ||
| }, | ||
| { | ||
| "remove": { | ||
| "field": "apache.access.time", | ||
| "ignore_failure": true | ||
| } | ||
| }, | ||
| { | ||
| "user_agent": { | ||
| "field": "user_agent.original", | ||
| "ignore_failure": true | ||
| } | ||
| }, | ||
| { | ||
| "geoip": { | ||
| "field": "source.ip", | ||
| "target_field": "source.geo", | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| { | ||
| "geoip": { | ||
| "database_file": "GeoLite2-ASN.mmdb", | ||
| "field": "source.ip", | ||
| "target_field": "source.as", | ||
| "properties": [ | ||
| "asn", | ||
| "organization_name" | ||
| ], | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| { | ||
| "rename": { | ||
| "field": "source.as.asn", | ||
| "target_field": "source.as.number", | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| { | ||
| "rename": { | ||
| "field": "source.as.organization_name", | ||
| "target_field": "source.as.organization.name", | ||
| "ignore_missing": true | ||
| } | ||
| } | ||
| ], | ||
| "on_failure": [ | ||
| { | ||
| "set": { | ||
| "field": "error.message", | ||
| "value": "{{ _ingest.on_failure_message }}" | ||
| } | ||
| } | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,55 +1,85 @@ | ||
| { | ||
| "description": "Pipeline for parsing apache error logs", | ||
| "processors": [ | ||
| { | ||
| "grok": { | ||
| "field": "message", | ||
| "patterns": [ | ||
| "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}", | ||
| "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}" | ||
| ], | ||
| "pattern_definitions": { | ||
| "APACHE_TIME": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}" | ||
| "description": "Pipeline for parsing apache error logs", | ||
| "processors": [ | ||
| { | ||
| "grok": { | ||
| "field": "message", | ||
| "patterns": [ | ||
| "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}", | ||
| "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}" | ||
| ], | ||
| "pattern_definitions": { | ||
| "APACHE_TIME": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}" | ||
| }, | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| { | ||
| "date": { | ||
| "field": "apache.error.timestamp", | ||
| "target_field": "@timestamp", | ||
| "formats": ["EEE MMM dd H:m:s yyyy", "EEE MMM dd H:m:s.SSSSSS yyyy"], | ||
| "ignore_failure": true | ||
| } | ||
| }, | ||
| { | ||
| "remove": { | ||
| "field": "apache.error.timestamp", | ||
| "ignore_failure": true | ||
| } | ||
| }, | ||
|
|
||
| { | ||
| "grok": { | ||
| "field": "source.address", | ||
| "ignore_missing": true, | ||
| "patterns": [ | ||
| "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" | ||
| ] | ||
| { | ||
| "date": { | ||
| "field": "apache.error.timestamp", | ||
| "target_field": "@timestamp", | ||
| "formats": [ | ||
| "EEE MMM dd H:m:s yyyy", | ||
| "EEE MMM dd H:m:s.SSSSSS yyyy" | ||
| ], | ||
| "ignore_failure": true | ||
| } | ||
| }, | ||
| { | ||
| "remove": { | ||
| "field": "apache.error.timestamp", | ||
| "ignore_failure": true | ||
| } | ||
| }, | ||
| { | ||
| "grok": { | ||
| "field": "source.address", | ||
| "ignore_missing": true, | ||
| "patterns": [ | ||
| "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" | ||
| ] | ||
| } | ||
| }, | ||
| { | ||
| "geoip": { | ||
| "field": "source.ip", | ||
| "target_field": "source.geo", | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| { | ||
| "geoip": { | ||
| "database_file": "GeoLite2-ASN.mmdb", | ||
| "field": "source.ip", | ||
| "target_field": "source.as", | ||
| "properties": [ | ||
| "asn", | ||
| "organization_name" | ||
| ], | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| { | ||
| "rename": { | ||
| "field": "source.as.asn", | ||
| "target_field": "source.as.number", | ||
| "ignore_missing": true | ||
| } | ||
| }, | ||
| { | ||
| "rename": { | ||
| "field": "source.as.organization_name", | ||
| "target_field": "source.as.organization.name", | ||
| "ignore_missing": true | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "geoip": { | ||
| "field": "source.ip", | ||
| "target_field": "source.geo", | ||
| "ignore_missing": true | ||
| ], | ||
| "on_failure": [ | ||
| { | ||
| "set": { | ||
| "field": "error.message", | ||
| "value": "{{ _ingest.on_failure_message }}" | ||
| } | ||
| } | ||
| } | ||
| ], | ||
| "on_failure" : [{ | ||
| "set" : { | ||
| "field" : "error.message", | ||
| "value" : "{{ _ingest.on_failure_message }}" | ||
| } | ||
| }] | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.