[Filebeat] Fix IPtables pipeline (#24928)

* #24878: Fix IPtables pipeline - fix dashboards - populate additional event.actions - move community_id processor to ingest node - set observer.name (cherry picked from commit ddcf8f1)
elastic · Apr 12, 2021 · 59918e1 · 59918e1
1 parent 71e2fe9
commit 59918e1
Show file tree

Hide file tree

Showing 10 changed files with 79 additions and 43 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -239,6 +239,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve PanOS parsing and ingest pipeline. {issue}22413[22413] {issue}22748[22748] {pull}24799[24799]
 - Fix S3 input validation for non amazonaws.com domains. {issue}24420[24420] {pull}24861[24861]
 - Fix google_workspace and okta modules pagination when next page template is empty. {pull}24967[24967]
+- Fix IPtables Pipeline and Ubiquiti dashboard. {issue}24878[24878] {pull}24928[24928]
 
 *Heartbeat*
 

diff --git a/...odule/iptables/_meta/kibana/7/dashboard/Filebeat-Iptables-Ubiquiti-Firewall-Overview.json b/...odule/iptables/_meta/kibana/7/dashboard/Filebeat-Iptables-Ubiquiti-Firewall-Overview.json
@@ -628,7 +628,7 @@
             "index": "filebeat-*",
             "query": {
               "language": "kuery",
-              "query": "iptables.ubiquiti.rule_set :* and event.outcome : \"deny\""
+              "query": "iptables.ubiquiti.rule_set :* and event.action : \"drop\""
             },
             "version": true
           }
@@ -659,7 +659,7 @@
             "index": "filebeat-*",
             "query": {
               "language": "kuery",
-              "query": "iptables.ubiquiti.rule_set :* and event.outcome : \"allow\""
+              "query": "iptables.ubiquiti.rule_set :* and event.action : \"accept\""
             },
             "version": true
           }

diff --git a/x-pack/filebeat/module/iptables/log/config/input.yml b/x-pack/filebeat/module/iptables/log/config/input.yml
@@ -19,38 +19,6 @@ tags: {{.tags}}
 
 processors:
   - add_locale: ~
-{{ if .community_id }}
-  - dissect:
-      tokenizer: "%{} SRC=%{source.ip} DST=%{destination.ip} "
-      field: "message"
-      target_prefix: ""
-  - dissect:
-      tokenizer: "%{} PROTO=%{network.transport} "
-      field: "message"
-      target_prefix: ""
-  - if:
-      or:
-        - equals.network.transport: TCP
-        - equals.network.transport: UDP
-        - equals.network.transport: SCTP
-    then:
-      dissect:
-        tokenizer: "%{} SPT=%{source.port} DPT=%{destination.port} "
-        field: "message"
-        target_prefix: ""
-    else:
-      dissect:
-        when:or:
-          - equals.network.transport: ICMP
-          - equals.network.transport: ICMPv6
-        tokenizer: "%{} TYPE=%{iptables.icmp.type} CODE=%{iptables.icmp.code} "
-        field: "message"
-        target_prefix: ""
-  - community_id:
-      fields:
-        icmp_type: iptables.icmp.type
-        icmp_code: iptables.icmp.code
-{{ end}}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml
@@ -6,11 +6,13 @@ processors:
 - grok:
     field: message
     patterns:
-    - '%{SYSLOGTIMESTAMP:iptables.raw_date}%{GREEDYDATA}\[%{UBIQUITI_LABEL}\]%{IPTABLES}%{SPACE}'
-    - '%{SYSLOGTIMESTAMP:iptables.raw_date}%{GREEDYDATA}%{IPTABLES}%{SPACE}'
+    - '%{SYSLOGTIMESTAMP:iptables.raw_date}%{SPACE}%{IPTABLES_HOSTNAME}%{GREEDYDATA}\[%{UBIQUITI_LABEL}\]%{IPTABLES}%{SPACE}'
+    - '%{SYSLOGTIMESTAMP:iptables.raw_date}%{SPACE}%{IPTABLES_ACTION}%{GREEDYDATA}%{IPTABLES}%{SPACE}'
     - '%{GREEDYDATA}\[%{UBIQUITI_LABEL}\]%{IPTABLES}%{SPACE}'
     - '%{GREEDYDATA}%{IPTABLES}%{SPACE}'
     pattern_definitions:
+      IPTABLES_HOSTNAME: '%{HOSTNAME:observer.name}%{SPACE}kernel:'
+      IPTABLES_ACTION: '(:?%{WORD:event.action}:|%{IPTABLES_HOSTNAME}%{SPACE}iptables%{SPACE}%{WORD:event.action}|%{IPTABLES_HOSTNAME})'
       UNSIGNED_INT: '[0-9]+'
       ETHTYPE: (?:[A-Fa-f0-9]{2}):(?:[A-Fa-f0-9]{2})
       ETHTYPE_DISCARD: (?::[A-Fa-f0-9]{2})*
@@ -59,7 +61,7 @@ processors:
     pattern_definitions:
       UBIQUITI_FIELD: '[^-]*'
 - date:
-    if: ctx.event.timezone == null
+    if: ctx?.iptables?.raw_date != null && ctx.event.timezone == null
     field: iptables.raw_date
     formats:
     - MMM  d HH:mm:ss
@@ -69,7 +71,7 @@ processors:
         field: error.message
         value: '{{ _ingest.on_failure_message }}'
 - date:
-    if: ctx.event.timezone != null
+    if: ctx?.iptables?.raw_date != null && ctx.event.timezone != null
     field: iptables.raw_date
     formats:
     - MMM  d HH:mm:ss
@@ -82,9 +84,22 @@ processors:
 - remove:
     field: iptables.raw_date
     ignore_missing: true
+- set:
+    field: observer.name
+    value: "{{hostname}}"
+    ignore_empty_value: true
+    if: ctx?.observer?.name == null
+- set:
+    field: observer.hostname
+    value: "{{hostname}}"
+    ignore_empty_value: true
+    if: ctx?.observer?.name == null
 - lowercase:
     field: network.transport
     ignore_missing: true
+- lowercase:
+    field: event.action
+    ignore_missing: true
 - geoip:
     field: source.ip
     target_field: source.geo
@@ -145,8 +160,8 @@ processors:
           object: event
           key: action
         map:
-          D: drop
-          A: accept
+          d: drop
+          a: accept
       - source:
           object: event
           key: action
@@ -156,6 +171,8 @@ processors:
         map:
           drop: denied
           accept: allowed
+          deny: denied
+          drop_input: denied
       - source:
           object: network
           key: transport
@@ -181,7 +198,10 @@ processors:
           }
         }
       }
-
+- community_id:
+    ignore_missing: true
+    icmp_type: iptables.icmp.type
+    icmp_code: iptables.icmp.code
 - script:
     lang: painless
     params:

diff --git a/x-pack/filebeat/module/iptables/log/manifest.yml b/x-pack/filebeat/module/iptables/log/manifest.yml
@@ -12,8 +12,6 @@ var:
     default: 9001
   - name: input
     default: syslog
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/input.yml

diff --git a/x-pack/filebeat/module/iptables/log/test/geo.log-expected.json b/x-pack/filebeat/module/iptables/log/test/geo.log-expected.json
@@ -44,6 +44,7 @@
         "network.type": "ipv4",
         "observer.egress.zone": "lan",
         "observer.ingress.zone": "wan",
+        "observer.name": "Hostname",
         "related.ip": [
             "158.109.0.1",
             "10.4.0.5"

diff --git a/x-pack/filebeat/module/iptables/log/test/icmp.log-expected.json b/x-pack/filebeat/module/iptables/log/test/icmp.log-expected.json
@@ -2,6 +2,7 @@
     {
         "destination.ip": "192.0.2.83",
         "destination.mac": "90:10:28:5f:62:24",
+        "event.action": "deny",
         "event.category": [
             "network"
         ],
@@ -10,6 +11,7 @@
         "event.module": "iptables",
         "event.timezone": "-02:00",
         "event.type": [
+            "denied",
             "connection"
         ],
         "fileset.name": "log",