Allow host.* fields to be disabled in Zeek module (elastic#19113)

If `forwarded` as configured as a tag (e.g. `var.tags: [forwarded]`) for the Zeek module then Filebeat will not add `host` fields to events. This is for use cases where Zeek is analyzing forwarded data (like from a network tap or mirror port). Relates: elastic#13920 (cherry picked from commit 018b17c)
andrewkroh · Jun 18, 2020 · 49cc2fe · 49cc2fe
1 parent 49836d5
commit 49cc2fe
Showing 38 changed files with 78 additions and 38 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -47,6 +47,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
   `forwarded` from the list. {issue}13920[13920]
 * Cisco {pull}18753[18753]
 * Checkpoint {pull}18754[18754]
+* Netflow {pull}19087[19087]
+* Zeek {pull}19113[19113] (`forwarded` tag is not included by default)
+* Suricata {pull}19107[19107] (`forwarded` tag is not included by default)
 - Preserve case of http.request.method.  ECS prior to 1.6 specified normalizing to lowercase, which lost information. Affects filesets: apache/access, elasticsearch/audit, iis/access, iis/error, nginx/access, nginx/ingress_controller, aws/elb, suricata/eve, zeek/http. {issue}18154[18154] {pull}18359[18359]
 - With the default configuration the cloud modules (aws, azure, googlecloud, o365, okta)
 will no longer send the `host` field that contains information about the host Filebeat is

diff --git a/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml b/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/connection/config/connection.yml b/x-pack/filebeat/module/zeek/connection/config/connection.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml b/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml b/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml b/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/dns/config/dns.yml b/x-pack/filebeat/module/zeek/dns/config/dns.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 processors:
   - rename:

diff --git a/x-pack/filebeat/module/zeek/dpd/config/dpd.yml b/x-pack/filebeat/module/zeek/dpd/config/dpd.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/files/config/files.yml b/x-pack/filebeat/module/zeek/files/config/files.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/ftp/config/ftp.yml b/x-pack/filebeat/module/zeek/ftp/config/ftp.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/http/config/http.yml b/x-pack/filebeat/module/zeek/http/config/http.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 
@@ -62,7 +63,7 @@ processors:
 
         - from: "zeek.http.password"
           to: "url.password"
-        
+
         - from: "zeek.http.user_agent"
           to: "user_agent.original"
 

diff --git a/x-pack/filebeat/module/zeek/intel/config/intel.yml b/x-pack/filebeat/module/zeek/intel/config/intel.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 processors:
   - rename:

diff --git a/x-pack/filebeat/module/zeek/irc/config/irc.yml b/x-pack/filebeat/module/zeek/irc/config/irc.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/kerberos/config/kerberos.yml b/x-pack/filebeat/module/zeek/kerberos/config/kerberos.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/modbus/config/modbus.yml b/x-pack/filebeat/module/zeek/modbus/config/modbus.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/mysql/config/mysql.yml b/x-pack/filebeat/module/zeek/mysql/config/mysql.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/notice/config/notice.yml b/x-pack/filebeat/module/zeek/notice/config/notice.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml b/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml b/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/pe/config/pe.yml b/x-pack/filebeat/module/zeek/pe/config/pe.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/radius/config/radius.yml b/x-pack/filebeat/module/zeek/radius/config/radius.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/rdp/config/rdp.yml b/x-pack/filebeat/module/zeek/rdp/config/rdp.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/rfb/config/rfb.yml b/x-pack/filebeat/module/zeek/rfb/config/rfb.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/sip/config/sip.yml b/x-pack/filebeat/module/zeek/sip/config/sip.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml b/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml b/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml b/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/smtp/config/smtp.yml b/x-pack/filebeat/module/zeek/smtp/config/smtp.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/snmp/config/snmp.yml b/x-pack/filebeat/module/zeek/snmp/config/snmp.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/socks/config/socks.yml b/x-pack/filebeat/module/zeek/socks/config/socks.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/ssh/config/ssh.yml b/x-pack/filebeat/module/zeek/ssh/config/ssh.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/ssl/config/ssl.yml b/x-pack/filebeat/module/zeek/ssl/config/ssl.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/stats/config/stats.yml b/x-pack/filebeat/module/zeek/stats/config/stats.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/syslog/config/syslog.yml b/x-pack/filebeat/module/zeek/syslog/config/syslog.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml b/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml b/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/weird/config/weird.yml b/x-pack/filebeat/module/zeek/weird/config/weird.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false
 

diff --git a/x-pack/filebeat/module/zeek/x509/config/x509.yml b/x-pack/filebeat/module/zeek/x509/config/x509.yml
@@ -4,7 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
-tags: {{.tags}}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 json.keys_under_root: false