Update Zeek DNS pipeline with ECS DNS fields

CHANGELOG.next.asciidoc

@@ -277,6 +277,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add S3 input to retrieve logs from AWS S3 buckets. {pull}12640[12640] {issue}12582[12582]
 - Add aws module s3access metricset. {pull}13170[13170] {issue}12880[12880]
 - Update PAN-OS fileset to use the ECS NAT fields. {issue}13320[13320] {pull}13330[13330]
+- Add fields to the Zeek DNS fileset for ECS DNS. {issue}13320[13320] {pull}NNNN[NNNN]
+- Add fields to the Zeek DNS fileset for ECS DNS. {issue}13320[13320] {pull}13324[13324]
 
 *Heartbeat*
 

libbeat/processors/script/javascript/module/include.go

@@ -20,6 +20,7 @@ package module
 import (
 	// Register javascript modules.
 	_ "github.com/elastic/beats/libbeat/processors/script/javascript/module/console"
+	_ "github.com/elastic/beats/libbeat/processors/script/javascript/module/net"
 	_ "github.com/elastic/beats/libbeat/processors/script/javascript/module/path"
 	_ "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor"
 	_ "github.com/elastic/beats/libbeat/processors/script/javascript/module/require"

libbeat/processors/script/javascript/module/net/net.go

@@ -0,0 +1,68 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package net
+
+import (
+	"net"
+
+	"github.com/dop251/goja"
+	"github.com/dop251/goja_nodejs/require"
+)
+
+// Require registers the net module that provides utilities for working with IP
+// addresses. It can be accessed using:
+//
+//    // javascript
+//    var net = require('net');
+//
+func Require(vm *goja.Runtime, module *goja.Object) {
+	o := module.Get("exports").(*goja.Object)
+	o.Set("isIP", isIP)
+	o.Set("isIPv4", isIPv4)
+	o.Set("isIPv6", isIPv6)
+}
+
+func isIP(input string) int32 {
+	ip := net.ParseIP(input)
+	if ip == nil {
+		return 0
+	}
+
+	if ip.To4() != nil {
+		return 4
+	}
+
+	return 6
+}
+
+func isIPv4(input string) bool {
+	return 4 == isIP(input)
+}
+
+func isIPv6(input string) bool {
+	return 6 == isIP(input)
+}
+
+// Enable adds net to the given runtime.
+func Enable(runtime *goja.Runtime) {
+	runtime.Set("net", require.Require(runtime, "net"))
+}
+
+func init() {
+	require.RegisterNativeModule("net", Require)
+}

libbeat/processors/script/javascript/module/net/net_test.go

@@ -0,0 +1,98 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package net_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/beats/libbeat/beat"
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/processors/script/javascript"
+
+	_ "github.com/elastic/beats/libbeat/processors/script/javascript/module/net"
+	_ "github.com/elastic/beats/libbeat/processors/script/javascript/module/require"
+)
+
+func TestNetIsIP(t *testing.T) {
+	const script = `
+var net = require('net');
+
+function process(evt) {
+    var ip = evt.Get("ip");
+    var ipType = net.isIP(ip);
+	switch (ipType) {
+    case 4:
+        evt.Put("network.type", "ipv4");
+        break
+    case 6:
+        evt.Put("network.type", "ipv6");
+        break
+    }
+}
+`
+
+	p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for ip, typ := range map[string]interface{}{
+		"192.168.0.1":        "ipv4",
+		"::ffff:192.168.0.1": "ipv4",
+		"2001:0db8:0000:0000:0000:ff00:0042:8329": "ipv6",
+		"2001:db8:0:0:0:ff00:42:8329":             "ipv6",
+		"2001:db8::ff00:42:8329":                  "ipv6",
+		"www.elastic.co":                          nil,
+	} {
+		evt, err := p.Run(&beat.Event{Fields: common.MapStr{"ip": ip}})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		fields := evt.Fields.Flatten()
+		assert.Equal(t, typ, fields["network.type"])
+	}
+}
+
+func TestNetIsIPvN(t *testing.T) {
+	const script = `
+var net = require('net');
+
+function process(evt) {
+   	if (net.isIPv4("192.168.0.1") !== true) {
+        throw "isIPv4 failed";
+    }
+
+   	if (net.isIPv6("2001:db8::ff00:42:8329") !== true) {
+        throw "isIPv6 failed";
+    }
+}
+`
+
+	p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = p.Run(&beat.Event{Fields: common.MapStr{}})
+	if err != nil {
+		t.Fatal(err)
+	}
+}

x-pack/filebeat/module/zeek/dns/config/dns.yml

@@ -6,39 +6,143 @@ paths:
 exclude_files: [".gz$"]
 tags: {{.tags}}
 
-json.keys_under_root: false
-
 processors:
-  - drop_fields:
-      fields: ["json.Z","json.auth","json.addl"]
   - rename:
       fields:
-        - from: "json"
-          to: "zeek.dns"
+        - {from: message, to: event.original}
+  - decode_json_fields:
+      fields: [event.original]
+      target: zeek.dns
+  - script:
+      lang: javascript
+      id: zeek_dns_flags
+      source: >
+        var net = require("net");
 
-        - from: "zeek.dns.id.orig_h"
-          to: "source.address"
+        function addDnsHeaderFlags(evt) {
+            var flag = evt.Get("zeek.dns.AA");
+            if (flag === true) {
+                evt.AppendTo("dns.header_flags", "AA");
+            }
+            flag = evt.Get("zeek.dns.TC");
+            if (flag === true) {
+                evt.AppendTo("dns.header_flags", "TC");
+            }
+            flag = evt.Get("zeek.dns.RD");
+            if (flag === true) {
+                evt.AppendTo("dns.header_flags", "RD");
+            }
+            flag = evt.Get("zeek.dns.RA");
+            if (flag === true) {
+                evt.AppendTo("dns.header_flags", "RA");
+            }
+        }
 
-        - from: "zeek.dns.id.orig_p"
-          to: "source.port"
+        function addDnsQuestionClass(evt) {
+            var qclass = evt.Get("zeek.dns.qclass");
+            if (!qclass) {
+                return;
+            }
+            switch (qclass) {
+                case 1:
+                    qclass = "IN";
+                    break;
+                case 3:
+                    qclass = "CH";
+                    break;
+                case 4:
+                    qclass = "HS";
+                    break;
+                case 254:
+                    qclass = "NONE";
+                    break;
+                case 255:
+                    qclass = "ANY";
+                    break;
+            }
+            evt.Put("dns.question.class", qclass);
+        }
 
-        - from: "zeek.dns.id.resp_h"
-          to: "destination.address"
+        function addDnsAnswers(evt) {
+            var answers = evt.Get("zeek.dns.answers");
+            var ttls = evt.Get("zeek.dns.TTLs");
+            if (!answers || !ttls || answers.length != ttls.length) {
+                return;
+            }
 
-        - from: "zeek.dns.id.resp_p"
-          to: "destination.port"
+            var resolvedIps = [];
+            var answersObjs = [];
+            for (var i = 0; i < answers.length; i++) {
+                var answer = answers[i];
+                answersObjs.push({
+                    data: answer,
+                    ttl: ttls[i],
+                })
+                if (net.isIP(answer)) {
+                    resolvedIps.push(answer);
+                }
+            }
+            evt.Put("dns.answers", answersObjs);
+            if (resolvedIps.length > 0) {
+                evt.Put("dns.resolved_ip", resolvedIps);
+            }
+        }
 
-        - from: "zeek.dns.proto"
-          to: "network.transport"
+        function addEventDuration(evt) {
+            var rttSec = evt.Get("zeek.dns.rtt");
+            if (!rttSec) {
+                return;
+            }
+            evt.Put("event.duration", rttSec * 1000000000);
+        }
 
-        - from: "zeek.dns.uid"
-          to: "zeek.session_id"
-
+        function process(evt) {
+            addDnsHeaderFlags(evt);
+            addDnsQuestionClass(evt);
+            addDnsAnswers(evt);
+            addEventDuration(evt);
+        }
+  - convert:
       ignore_missing: true
-      fail_on_error: false
-{{ if .community_id }}
-  - community_id:
+      ignore_failure: true
+      mode: rename
       fields:
-        source_ip: source.address
-        destination_ip: destination.address
+        - {from: zeek.dns.id.orig_h, to: source.address}
+        - {from: zeek.dns.id.orig_p, to: source.port, type: long}
+        - {from: zeek.dns.id.resp_h, to: destination.address}
+        - {from: zeek.dns.id.resp_p, to: destination.port, type: long}
+        - {from: zeek.dns.uid, to: zeek.session_id}
+        - {from: zeek.dns.proto, to: network.transport}
+  - convert:
+      ignore_missing: true
+      ignore_failure: true
+      mode: copy
+      fields:
+        - {from: source.address, to: source.ip, type: ip}
+        - {from: destination.address, to: destination.ip, type: ip}
+        - {from: zeek.session_id, to: event.id}
+        - {from: '@timestamp', to: event.created}
+        - {from: zeek.dns.trans_id, to: dns.id}
+        - {from: zeek.dns.query, to: dns.question.name}
+        - {from: zeek.dns.qtype_name, to: dns.question.type}
+        - {from: zeek.dns.rcode_name, to: dns.response_code}
+  - registered_domain:
+      ignore_missing: true
+      ignore_failure: true
+      field: dns.question.name
+      target_field: dns.question.registered_domain
+{{ if .community_id }}
+  - community_id: ~
 {{ end }}
+  - timestamp:
+      ignore_missing: true
+      field: zeek.dns.ts
+      layouts:
+        - UNIX
+  - drop_fields:
+      ignore_missing: true
+      fields:
+        - zeek.dns.Z
+        - zeek.dns.auth
+        - zeek.dns.addl
+        - zeek.dns.ts