Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fix zeek pipelines to include millisecond precision (elastic#16766)
Browse files Browse the repository at this point in the history
* Fix zeek pipelines to include millisecond precision

* Add changelog entry

(cherry picked from commit 2c310c0)
Andrew Stucki committed Mar 3, 2020

Unverified

This commit is not signed, but one or more authors requires that any commit attributed to them is signed.
1 parent 19ca8b2 commit 88414db
Showing 7 changed files with 72 additions and 30 deletions.
9 changes: 9 additions & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
@@ -44,6 +44,15 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d


*Filebeat*

- Ensure all zeek timestamps include millisecond precision. {issue}14599[14599] {pull}16766[16766]
- Fix s3 input hanging with GetObjectRequest API call by adding context_timeout config. {issue}15502[15502] {pull}15590[15590]
- Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656]
- Fix typos in zeek notice fileset config file. {issue}15764[15764] {pull}15765[15765]
- Fix mapping error when zeek weird logs do not contain IP addresses. {pull}15906[15906]
- Improve `elasticsearch/audit` fileset to handle timestamps correctly. {pull}15942[15942]
- Prevent Elasticsearch from spewing log warnings about redundant wildcards when setting up ingest pipelines for the `elasticsearch` module. {issue}15840[15840] {pull}15900[15900]
- Fix mapping error for cloudtrail additionalEventData field {pull}16088[16088]
- Fix a connection error in httpjson input. {pull}16123[16123]
- Fix mapping error for cloudtrail additionalEventData field {pull}16088[16088]
- Fix s3 input with cloudtrail fileset reading json file. {issue}16374[16374] {pull}16441[16441]
29 changes: 20 additions & 9 deletions x-pack/filebeat/module/zeek/files/ingest/pipeline.json
Original file line number Diff line number Diff line change
@@ -2,9 +2,20 @@
"description": "Pipeline for normalizing Zeek files.log",
"processors": [
{
"script": {
"lang": "painless",
"source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['files']['ts'] * 1000; ctx.zeek.files.remove('ts');"
"set": {
"field": "event.created",
"value": "{{_ingest.timestamp}}"
}
},
{
"date": {
"field": "zeek.files.ts",
"formats": ["UNIX"]
}
},
{
"remove": {
"field": "zeek.files.ts"
}
},
{
@@ -37,10 +48,10 @@
}
}
],
"on_failure" : [{
"set" : {
"field" : "error.message",
"value" : "{{ _ingest.on_failure_message }}"
}
"on_failure": [{
"set": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}]
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[
{
"@timestamp": 1547688796000,
"@timestamp": "2019-01-17T01:33:16.636Z",
"event.dataset": "zeek.files",
"event.id": "C8I0zn3r9EPbfLgta6",
"event.module": "zeek",
@@ -37,7 +37,7 @@
"zeek.session_id": "C8I0zn3r9EPbfLgta6"
},
{
"@timestamp": 1547688801000,
"@timestamp": "2019-01-17T01:33:21.566Z",
"event.dataset": "zeek.files",
"event.id": "C6sjVo23iNApLnlAt6",
"event.module": "zeek",
27 changes: 19 additions & 8 deletions x-pack/filebeat/module/zeek/http/ingest/pipeline.json
Original file line number Diff line number Diff line change
@@ -2,9 +2,20 @@
"description": "Pipeline for normalizing Zeek http.log",
"processors": [
{
"script": {
"lang": "painless",
"source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['http']['ts'] * 1000; ctx.zeek.http.remove('ts');"
"set": {
"field": "event.created",
"value": "{{_ingest.timestamp}}"
}
},
{
"date": {
"field": "zeek.http.ts",
"formats": ["UNIX"]
}
},
{
"remove": {
"field": "zeek.http.ts"
}
},
{
@@ -103,10 +114,10 @@
}
}
],
"on_failure" : [{
"set" : {
"field" : "error.message",
"value" : "{{ _ingest.on_failure_message }}"
}
"on_failure": [{
"set": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}]
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[
{
"@timestamp": 1547687130000,
"@timestamp": "2019-01-17T01:05:30.172Z",
"destination.address": "17.253.5.203",
"destination.as.number": 6185,
"destination.as.organization.name": "Apple Inc.",
27 changes: 19 additions & 8 deletions x-pack/filebeat/module/zeek/notice/ingest/pipeline.json
Original file line number Diff line number Diff line change
@@ -2,9 +2,20 @@
"description": "Pipeline for normalizing Zeek notice.log",
"processors": [
{
"script": {
"lang": "painless",
"source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['notice']['ts'] * 1000; ctx.zeek.notice.remove('ts');"
"set": {
"field": "event.created",
"value": "{{_ingest.timestamp}}"
}
},
{
"date": {
"field": "zeek.notice.ts",
"formats": ["UNIX"]
}
},
{
"remove": {
"field": "zeek.notice.ts"
}
},
{
@@ -95,10 +106,10 @@
}
}
],
"on_failure" : [{
"set" : {
"field" : "error.message",
"value" : "{{ _ingest.on_failure_message }}"
}
"on_failure": [{
"set": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}]
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[
{
"@timestamp": 1320435875000,
"@timestamp": "2011-11-04T19:44:35.879Z",
"event.dataset": "zeek.notice",
"event.module": "zeek",
"fileset.name": "notice",
@@ -20,7 +20,7 @@
"zeek.notice.suppress_for": 3600
},
{
"@timestamp": 1551393388000,
"@timestamp": "2019-02-28T22:36:28.426Z",
"destination.address": "207.154.238.205",
"destination.as.number": 14061,
"destination.as.organization.name": "DigitalOcean, LLC",

0 comments on commit 88414db

Please sign in to comment.