sync with recent changes in zeek package (elastic#21155) (elastic#21181)

- always attempt community_id processor - dnp3 reorder pipeline to be same as package (cherry picked from commit 1932f9f)
faec · Sep 28, 2020 · 1c94c6d · 1c94c6d
1 parent 7d7ce2a
commit 1c94c6d
Show file tree

Hide file tree

Showing 59 changed files with 9 additions and 122 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -679,6 +679,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Improve Suricata Eve module with `x509` ECS mappings {pull}20973[20973]
 - Added new module for Zoom webhooks {pull}20414[20414]
 - Add type and sub_type to panw panos fileset {pull}20912[20912]
+- Always attempt community_id processor on zeek module {pull}21155[21155]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/zeek/connection/config/connection.yml b/x-pack/filebeat/module/zeek/connection/config/connection.yml
@@ -90,7 +90,6 @@ processors:
         kind: event
         category:
           - network
-{{ if .community_id }}
   - if:
       equals.network.transport: icmp
     then:
@@ -100,7 +99,6 @@ processors:
           icmp_code: zeek.connection.icmp.code
     else:
       community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/connection/manifest.yml b/x-pack/filebeat/module/zeek/connection/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/conn.log
   - name: tags
     default: [zeek.connection]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/connection.yml

diff --git a/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml b/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml
@@ -54,9 +54,7 @@ processors:
           - connection
           - protocol
           - info
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/dce_rpc/manifest.yml b/x-pack/filebeat/module/zeek/dce_rpc/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/dce_rpc.log
   - name: tags
     default: [zeek.dce_rpc]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/dce_rpc.yml
diff --git a/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml b/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml
@@ -116,9 +116,7 @@ processors:
           - connection
           - protocol
           - info
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/dhcp/manifest.yml b/x-pack/filebeat/module/zeek/dhcp/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/dhcp.log
   - name: tags
     default: [zeek.dhcp]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/dhcp.yml
diff --git a/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml b/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml
@@ -64,9 +64,7 @@ processors:
           - connection
           - protocol
           - info
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml
@@ -23,14 +23,6 @@ processors:
 - lowercase:
     field: event.action
     ignore_missing: true
-- append:
-    field: related.ip
-    value: '{{source.ip}}'
-    if: ctx?.source?.ip != null
-- append:
-    field: related.ip
-    value: '{{destination.ip}}'
-    if: ctx?.destination?.ip != null
 - geoip:
     field: destination.ip
     target_field: destination.geo
@@ -69,6 +61,14 @@ processors:
     field: destination.as.organization_name
     target_field: destination.as.organization.name
     ignore_missing: true
+- append:
+    field: related.ip
+    value: '{{source.ip}}'
+    if: ctx?.source?.ip != null
+- append:
+    field: related.ip
+    value: '{{destination.ip}}'
+    if: ctx?.destination?.ip != null
 on_failure:
 - set:
     field: error.message

diff --git a/x-pack/filebeat/module/zeek/dnp3/manifest.yml b/x-pack/filebeat/module/zeek/dnp3/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/dnp3.log
   - name: tags
     default: [zeek.dnp3]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/dnp3.yml
diff --git a/x-pack/filebeat/module/zeek/dns/config/dns.yml b/x-pack/filebeat/module/zeek/dns/config/dns.yml
@@ -193,9 +193,7 @@ processors:
           - connection
           - info
           - protocol
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - drop_fields:
       ignore_missing: true
       fields:

diff --git a/x-pack/filebeat/module/zeek/dns/manifest.yml b/x-pack/filebeat/module/zeek/dns/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/dns.log
   - name: tags
     default: [zeek.dns]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/dns.yml

diff --git a/x-pack/filebeat/module/zeek/dpd/config/dpd.yml b/x-pack/filebeat/module/zeek/dpd/config/dpd.yml
@@ -53,9 +53,7 @@ processors:
         type:
           - connection
           - info
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/dpd/manifest.yml b/x-pack/filebeat/module/zeek/dpd/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/dpd.log
   - name: tags
     default: [zeek.dpd]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/dpd.yml
diff --git a/x-pack/filebeat/module/zeek/ftp/config/ftp.yml b/x-pack/filebeat/module/zeek/ftp/config/ftp.yml
@@ -82,9 +82,7 @@ processors:
           - connection
           - info
           - protocol
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/ftp/manifest.yml b/x-pack/filebeat/module/zeek/ftp/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/ftp.log
   - name: tags
     default: [zeek.ftp]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/ftp.yml
diff --git a/x-pack/filebeat/module/zeek/http/config/http.yml b/x-pack/filebeat/module/zeek/http/config/http.yml
@@ -89,9 +89,7 @@ processors:
           - connection
           - info
           - protocol
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/http/manifest.yml b/x-pack/filebeat/module/zeek/http/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/http.log
   - name: tags
     default: [zeek.http]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/http.yml

diff --git a/x-pack/filebeat/module/zeek/intel/config/intel.yml b/x-pack/filebeat/module/zeek/intel/config/intel.yml
@@ -63,9 +63,7 @@ processors:
         kind: alert
         type:
           - info
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/intel/manifest.yml b/x-pack/filebeat/module/zeek/intel/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/intel.log
   - name: tags
     default: [zeek.intel]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/intel.yml

diff --git a/x-pack/filebeat/module/zeek/irc/config/irc.yml b/x-pack/filebeat/module/zeek/irc/config/irc.yml
@@ -68,9 +68,7 @@ processors:
           - connection
           - protocol
           - info
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/irc/manifest.yml b/x-pack/filebeat/module/zeek/irc/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/irc.log
   - name: tags
     default: [zeek.irc]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/irc.yml
diff --git a/x-pack/filebeat/module/zeek/kerberos/config/kerberos.yml b/x-pack/filebeat/module/zeek/kerberos/config/kerberos.yml
@@ -100,9 +100,7 @@ processors:
       tokenizer: "%{user.name}/%{user.domain}"
       field: zeek.kerberos.client
       target_prefix: ""
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/kerberos/manifest.yml b/x-pack/filebeat/module/zeek/kerberos/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/kerberos.log
   - name: tags
     default: [zeek.kerberos]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/kerberos.yml

diff --git a/x-pack/filebeat/module/zeek/modbus/config/modbus.yml b/x-pack/filebeat/module/zeek/modbus/config/modbus.yml
@@ -69,9 +69,7 @@ processors:
           target: event
           fields:
             outcome: success
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/modbus/manifest.yml b/x-pack/filebeat/module/zeek/modbus/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/modbus.log
   - name: tags
     default: [zeek.modbus]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/modbus.yml
diff --git a/x-pack/filebeat/module/zeek/mysql/config/mysql.yml b/x-pack/filebeat/module/zeek/mysql/config/mysql.yml
@@ -68,9 +68,7 @@ processors:
           target: event
           fields:
             outcome: failure
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/mysql/manifest.yml b/x-pack/filebeat/module/zeek/mysql/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/mysql.log
   - name: tags
     default: [zeek.mysql]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/mysql.yml
diff --git a/x-pack/filebeat/module/zeek/notice/config/notice.yml b/x-pack/filebeat/module/zeek/notice/config/notice.yml
@@ -100,9 +100,7 @@ processors:
           - intrusion_detection
         type:
           - info
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/notice/manifest.yml b/x-pack/filebeat/module/zeek/notice/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/notice.log
   - name: tags
     default: [zeek.notice]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/notice.yml

diff --git a/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml b/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml
@@ -82,9 +82,7 @@ processors:
           target: event
           fields:
             outcome: failure
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/ntlm/manifest.yml b/x-pack/filebeat/module/zeek/ntlm/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/ntlm.log
   - name: tags
     default: [zeek.ntlm]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/ntlm.yml
diff --git a/x-pack/filebeat/module/zeek/radius/config/radius.yml b/x-pack/filebeat/module/zeek/radius/config/radius.yml
@@ -54,9 +54,7 @@ processors:
         type:
           - info
           - connection
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/radius/manifest.yml b/x-pack/filebeat/module/zeek/radius/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/radius.log
   - name: tags
     default: [zeek.radius]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/radius.yml
diff --git a/x-pack/filebeat/module/zeek/rdp/config/rdp.yml b/x-pack/filebeat/module/zeek/rdp/config/rdp.yml
@@ -84,9 +84,7 @@ processors:
         type:
           - protocol
           - info
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/rdp/manifest.yml b/x-pack/filebeat/module/zeek/rdp/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/rdp.log
   - name: tags
     default: [zeek.rdp]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/rdp.yml
diff --git a/x-pack/filebeat/module/zeek/rfb/config/rfb.yml b/x-pack/filebeat/module/zeek/rfb/config/rfb.yml
@@ -69,9 +69,7 @@ processors:
         type:
           - connection
           - info
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/rfb/manifest.yml b/x-pack/filebeat/module/zeek/rfb/manifest.yml
@@ -10,8 +10,6 @@ var:
       - /usr/local/var/logs/current/rfb.log
   - name: tags
     default: [zeek.rfb]
-  - name: community_id
-    default: true
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/rfb.yml
diff --git a/x-pack/filebeat/module/zeek/sip/config/sip.yml b/x-pack/filebeat/module/zeek/sip/config/sip.yml
@@ -91,9 +91,7 @@ processors:
         type:
           - connection
           - protocol
-{{ if .community_id }}
   - community_id:
-{{ end }}
   - add_fields:
       target: ''
       fields: