pipeline.yml

description: Pipeline for parsing MySQL error logs
processors:
- grok:
    field: message
    patterns:
    - '%{MYSQLDATETIME}%{SPACE}(%{NUMBER:mysql.thread_id:long}%{SPACE})?(\[%{DATA:log.level}\]%{SPACE})?%{GREEDYMULTILINE:message}'
    - '%{GREEDYDATA:message}'
    ignore_missing: true
    pattern_definitions:
      LOCALDATETIME: (?:%{YEAR}-%{MONTHNUM}-%{MONTHDAY}|%{NUMBER})%{SPACE}%{TIME}
      MYSQLDATETIME: (?:%{LOCALDATETIME:_tmp.local_timestamp}|%{TIMESTAMP_ISO8601:_tmp.timestamp})
      GREEDYMULTILINE: |-
        (.|
        )+
- grok:
    field: message
    patterns:
    - '(\[%{DATA:event.code}\])%{SPACE}(\[%{DATA:event.provider}\])%{SPACE}%{GREEDYMULTILINE}'
    - '%{GREEDYDATA}'
    ignore_missing: true
    ignore_failure: true
    pattern_definitions:
      GREEDYMULTILINE: |-
        (.|
        )+
- rename:
    field: '@timestamp'
    target_field: event.created
- date:
    if: ctx._tmp?.local_timestamp != null && ctx.event?.timezone == null
    field: _tmp.local_timestamp
    formats:
    - yyMMdd H:m:s
    - yyMMdd  H:m:s
    - yyyy-MM-dd H:m:s
    - yyyy-MM-dd  H:m:s
- date:
    if: ctx._tmp?.local_timestamp != null && ctx.event?.timezone != null
    field: _tmp.local_timestamp
    timezone: '{{ event.timezone }}'
    formats:
    - yyMMdd H:m:s
    - yyMMdd  H:m:s
    - yyyy-MM-dd H:m:s
    - yyyy-MM-dd  H:m:s
- date:
    if: ctx._tmp?.timestamp != null
    field: _tmp.timestamp
    formats:
    - ISO8601
- remove:
    field: _tmp
    ignore_missing: true
- set:
    field: event.kind
    value: event
- append:
    field: event.category
    value: database
- append:
    field: event.type
    value: info
- append:
    field: event.type
    value: error
    if: "ctx?.log?.level != null && ctx.log.level.toLowerCase() == 'error'"
on_failure:
- set:
    field: error.message
    value: '{{ _ingest.on_failure_message }}'