Skip to content

Commit

Permalink
Improve ECS categorization field mappings in traefik module (#19379) (#…
Browse files Browse the repository at this point in the history
…19392)

- event.kind
- event.category
- event.type
- event.outcome
- related.ip
- related.user

Closes #16183

(cherry picked from commit f814f41)
leehinman authored Jul 8, 2020
1 parent 77dc072 commit 72c5c29
Showing 5 changed files with 188 additions and 132 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
@@ -503,6 +503,7 @@ field. You can revert this change by configuring tags for the module and omittin
- Explicitly set ECS version in all Filebeat modules. {pull}19198[19198]
- Add awscloudwatch input. {pull}19025[19025]
- Add support for timezone offsets and `Z` to decode_cef timestamp parser. {pull}19346[19346]
- Improve ECS categorization field mappings in traefik module. {issue}16183[16183] {pull}19379[19379]

*Heartbeat*

131 changes: 0 additions & 131 deletions filebeat/module/traefik/access/ingest/pipeline.json

This file was deleted.

106 changes: 106 additions & 0 deletions filebeat/module/traefik/access/ingest/pipeline.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
description: Pipeline for parsing Traefik access logs. Requires the geoip and user_agent
plugins.
processors:
- dissect:
field: message
pattern: '%{source.address} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}]
"%{http.request.method} %{url.original} HTTP/%{http.version}" %{http.response.status_code}
%{traefik.access.message}'
- grok:
field: traefik.access.message
patterns:
- (?:%{NUMBER:http.response.body.bytes:long}|-)( (?:"%{DATA:http.request.referrer}"|-)?(
(?:"%{DATA:user_agent.original}"|-)?)?( (?:%{NUMBER:traefik.access.request_count:long}|-)?)?(
(?:"%{DATA:traefik.access.frontend_name}"|-)?)?( "%{DATA:traefik.access.backend_url}")?(
%{NUMBER:temp.duration:long}ms)?)?
ignore_missing: true
- remove:
field: message
ignore_missing: true
- remove:
field: traefik.access.message
ignore_missing: true
- rename:
field: '@timestamp'
target_field: event.created
- date:
field: traefik.access.time
target_field: '@timestamp'
formats:
- dd/MMM/yyyy:H:m:s Z
- remove:
field: traefik.access.time
- convert:
field: http.response.status_code
type: long
- grok:
field: source.address
patterns:
- ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$
- script:
lang: painless
source: ctx.event.duration = Math.round(ctx.temp.duration * params.scale)
params:
scale: 1000000
if: ctx.temp?.duration != null
- remove:
field: temp.duration
ignore_missing: true
- user_agent:
field: user_agent.original
ignore_failure: true
- geoip:
field: source.ip
target_field: source.geo
ignore_missing: true
- geoip:
database_file: GeoLite2-ASN.mmdb
field: source.ip
target_field: source.as
properties:
- asn
- organization_name
ignore_missing: true
- rename:
field: source.as.asn
target_field: source.as.number
ignore_missing: true
- rename:
field: source.as.organization_name
target_field: source.as.organization.name
ignore_missing: true
- set:
field: event.kind
value: event
- append:
field: event.category
value: web
if: "ctx?.http?.request?.method != null && ctx.http.request.method != '-'"
- append:
field: event.type
value: access
if: "ctx?.http?.request?.method != null && ctx.http.request.method != '-'"
- set:
field: event.outcome
value: success
if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
- set:
field: event.outcome
value: failure
if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400"
- append:
field: related.ip
value: "{{source.ip}}"
if: "ctx?.source?.ip != null"
- append:
field: related.ip
value: "{{destination.ip}}"
if: "ctx?.destination?.ip != null"
- append:
field: related.user
value: "{{user.name}}"
if: "ctx?.user?.name != null && ctx.user.name != '-'"
on_failure:
- set:
field: error.message
value: '{{ _ingest.on_failure_message }}'
2 changes: 1 addition & 1 deletion filebeat/module/traefik/access/manifest.yml
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ var:
os.windows:
- c:/programdata/traefik/logs/*access.log*

ingest_pipeline: ingest/pipeline.json
ingest_pipeline: ingest/pipeline.yml
input: config/traefik-access.yml

requires.processors:
80 changes: 80 additions & 0 deletions filebeat/module/traefik/access/test/test.log-expected.json
Original file line number Diff line number Diff line change
@@ -1,9 +1,17 @@
[
{
"@timestamp": "2017-10-02T20:22:07.000Z",
"event.category": [
"web"
],
"event.dataset": "traefik.access",
"event.duration": 2000000,
"event.kind": "event",
"event.module": "traefik",
"event.outcome": "success",
"event.type": [
"access"
],
"fileset.name": "access",
"http.request.method": "GET",
"http.request.referrer": "http://example.com/login",
@@ -12,6 +20,9 @@
"http.version": "1.1",
"input.type": "log",
"log.offset": 0,
"related.ip": [
"192.168.33.1"
],
"service.type": "traefik",
"source.address": "192.168.33.1",
"source.ip": "192.168.33.1",
@@ -29,9 +40,17 @@
},
{
"@timestamp": "2017-10-02T20:22:08.000Z",
"event.category": [
"web"
],
"event.dataset": "traefik.access",
"event.duration": 3000000,
"event.kind": "event",
"event.module": "traefik",
"event.outcome": "success",
"event.type": [
"access"
],
"fileset.name": "access",
"http.request.method": "GET",
"http.request.referrer": "http://example.com/login",
@@ -40,6 +59,9 @@
"http.version": "1.1",
"input.type": "log",
"log.offset": 280,
"related.ip": [
"85.181.35.98"
],
"service.type": "traefik",
"source.address": "85.181.35.98",
"source.as.number": 6805,
@@ -66,16 +88,27 @@
},
{
"@timestamp": "2018-02-28T17:30:33.000Z",
"event.category": [
"web"
],
"event.dataset": "traefik.access",
"event.duration": 247000000,
"event.kind": "event",
"event.module": "traefik",
"event.outcome": "success",
"event.type": [
"access"
],
"fileset.name": "access",
"http.request.method": "GET",
"http.response.body.bytes": 2814,
"http.response.status_code": 200,
"http.version": "2.0",
"input.type": "log",
"log.offset": 553,
"related.ip": [
"70.29.80.15"
],
"service.type": "traefik",
"source.address": "70.29.80.15",
"source.as.number": 577,
@@ -104,9 +137,17 @@
},
{
"@timestamp": "2018-11-29T15:03:51.000Z",
"event.category": [
"web"
],
"event.dataset": "traefik.access",
"event.duration": 0,
"event.kind": "event",
"event.module": "traefik",
"event.outcome": "failure",
"event.type": [
"access"
],
"fileset.name": "access",
"http.request.method": "GET",
"http.request.referrer": "-",
@@ -115,6 +156,9 @@
"http.version": "1.1",
"input.type": "log",
"log.offset": 821,
"related.ip": [
"::1"
],
"service.type": "traefik",
"source.address": "::1",
"source.ip": "::1",
@@ -131,16 +175,27 @@
},
{
"@timestamp": "2018-01-19T10:01:02.000Z",
"event.category": [
"web"
],
"event.dataset": "traefik.access",
"event.duration": 13000000,
"event.kind": "event",
"event.module": "traefik",
"event.outcome": "success",
"event.type": [
"access"
],
"fileset.name": "access",
"http.request.method": "GET",
"http.response.body.bytes": 85,
"http.response.status_code": 200,
"http.version": "1.1",
"input.type": "log",
"log.offset": 931,
"related.ip": [
"94.254.131.115"
],
"service.type": "traefik",
"source.address": "94.254.131.115",
"source.as.number": 39603,
@@ -166,16 +221,27 @@
},
{
"@timestamp": "2018-01-19T10:01:02.000Z",
"event.category": [
"web"
],
"event.dataset": "traefik.access",
"event.duration": 8000000,
"event.kind": "event",
"event.module": "traefik",
"event.outcome": "success",
"event.type": [
"access"
],
"fileset.name": "access",
"http.request.method": "GET",
"http.response.body.bytes": 150,
"http.response.status_code": 200,
"http.version": "1.1",
"input.type": "log",
"log.offset": 1267,
"related.ip": [
"89.64.35.193"
],
"service.type": "traefik",
"source.address": "89.64.35.193",
"source.as.number": 6830,
@@ -201,15 +267,29 @@
},
{
"@timestamp": "2000-10-10T20:55:36.000Z",
"event.category": [
"web"
],
"event.dataset": "traefik.access",
"event.kind": "event",
"event.module": "traefik",
"event.outcome": "success",
"event.type": [
"access"
],
"fileset.name": "access",
"http.request.method": "GET",
"http.response.body.bytes": 2326,
"http.response.status_code": 200,
"http.version": "1.0",
"input.type": "log",
"log.offset": 1581,
"related.ip": [
"127.0.0.1"
],
"related.user": [
"frank"
],
"service.type": "traefik",
"source.address": "127.0.0.1",
"source.ip": "127.0.0.1",

0 comments on commit 72c5c29

Please sign in to comment.