convert latency field to float + new advanced example (#758)

* remove deprecated examples
* change latency to float
* add example

dnsutils/message.go

@@ -150,8 +150,7 @@ type DNSTap struct {
 	Timestamp        int64   `json:"-"`
 	TimeSec          int     `json:"-"`
 	TimeNsec         int     `json:"-"`
-	Latency          float64 `json:"-"`
-	LatencySec       string  `json:"latency"`
+	Latency          float64 `json:"latency"`
 	Payload          []byte  `json:"-"`
 	Extra            string  `json:"extra"`
 	PolicyRule       string  `json:"policy-rule"`
@@ -284,15 +283,15 @@ func (dm *DNSMessage) Init() {
 		Identity:         "-",
 		Version:          "-",
 		TimestampRFC3339: "-",
-		LatencySec:       "-",
-		Extra:            "-",
-		PolicyRule:       "-",
-		PolicyType:       "-",
-		PolicyMatch:      "-",
-		PolicyAction:     "-",
-		PolicyValue:      "-",
-		PeerName:         "-",
-		QueryZone:        "-",
+		//	LatencySec:       "-",
+		Extra:        "-",
+		PolicyRule:   "-",
+		PolicyType:   "-",
+		PolicyMatch:  "-",
+		PolicyAction: "-",
+		PolicyValue:  "-",
+		PeerName:     "-",
+		QueryZone:    "-",
 	}
 
 	dm.DNS = DNS{
@@ -734,7 +733,7 @@ func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBo
 		case directive == "qclass":
 			s.WriteString(dm.DNS.Qclass)
 		case directive == "latency":
-			s.WriteString(dm.DNSTap.LatencySec)
+			s.WriteString(fmt.Sprintf("%.9f", dm.DNSTap.Latency))
 		case directive == "malformed":
 			if dm.DNS.MalformedPacket {
 				s.WriteString("PKTERR")
@@ -1173,7 +1172,7 @@ func (dm *DNSMessage) Flatten() (map[string]interface{}, error) {
 		"dns.rcode":                  dm.DNS.Rcode,
 		"dns.questions-count":        dm.DNS.QuestionsCount,
 		"dnstap.identity":            dm.DNSTap.Identity,
-		"dnstap.latency":             dm.DNSTap.LatencySec,
+		"dnstap.latency":             dm.DNSTap.Latency,
 		"dnstap.operation":           dm.DNSTap.Operation,
 		"dnstap.timestamp-rfc3339ns": dm.DNSTap.TimestampRFC3339,
 		"dnstap.version":             dm.DNSTap.Version,
@@ -1406,8 +1405,6 @@ func (dm *DNSMessage) Matching(matching map[string]interface{}) (error, bool) {
 		}
 
 		expectedValue := reflect.ValueOf(value)
-		// fmt.Println(nestedKeys, realValue, realValue.Kind(), expectedValue.Kind())
-
 		switch expectedValue.Kind() {
 		// integer
 		case reflect.Int:
@@ -1461,7 +1458,15 @@ func matchUserMap(realValue, expectedValue reflect.Value) (bool, error) {
 		switch opName {
 		// Integer great than ?
 		case MatchingOpGreaterThan:
+			isFloat, isInt := false, false
+			if _, ok := opValue.Interface().(float64); ok {
+				isFloat = true
+			}
 			if _, ok := opValue.Interface().(int); !ok {
+				isInt = true
+			}
+
+			if !isFloat && !isInt {
 				return false, fmt.Errorf("integer is expected for greater-than operator")
 			}
 
@@ -1483,12 +1488,18 @@ func matchUserMap(realValue, expectedValue reflect.Value) (bool, error) {
 				return false, nil
 			}
 
-			if realValue.Kind() != reflect.Int {
-				return false, nil
+			if realValue.Kind() == reflect.Float64 {
+				if realValue.Interface().(float64) > opValue.Interface().(float64) {
+					return true, nil
+				}
 			}
-			if realValue.Interface().(int) > opValue.Interface().(int) {
-				return true, nil
+
+			if realValue.Kind() == reflect.Int {
+				if realValue.Interface().(int) > opValue.Interface().(int) {
+					return true, nil
+				}
 			}
+
 			return false, nil
 
 		// Integer lower than ?

dnsutils/message_test.go

@@ -242,7 +242,7 @@ func TestDnsMessage_Json_Reference(t *testing.T) {
 				  "identity": "-",
 				  "version": "-",
 				  "timestamp-rfc3339ns": "-",
-				  "latency": "-",
+				  "latency": 0,
 				  "extra": "-",
 				  "policy-type": "-",
 				  "policy-action": "-",
@@ -478,7 +478,7 @@ func TestDnsMessage_JsonFlatten_Reference(t *testing.T) {
 					"dns.resource-records.ar": "-",
 					"dns.resource-records.ns": "-",
 					"dnstap.identity": "-",
-					"dnstap.latency": "-",
+					"dnstap.latency": 0,
 					"dnstap.operation": "-",
 					"dnstap.timestamp-rfc3339ns": "-",
 					"dnstap.version": "-",
@@ -818,7 +818,7 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) {
 			format:    config.Global.TextFormat,
 			qname:     "dnscollector.fr",
 			identity:  "collector",
-			expected:  "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b dnscollector.fr A -",
+			expected:  "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b dnscollector.fr A 0.000000000",
 		},
 		{
 			name:      "custom_delimiter",
@@ -827,7 +827,7 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) {
 			format:    config.Global.TextFormat,
 			qname:     "dnscollector.fr",
 			identity:  "collector",
-			expected:  "-;collector;CLIENT_QUERY;NOERROR;1.2.3.4;1234;-;-;0b;dnscollector.fr;A;-",
+			expected:  "-;collector;CLIENT_QUERY;NOERROR;1.2.3.4;1234;-;-;0b;dnscollector.fr;A;0.000000000",
 		},
 		{
 			name:      "empty_delimiter",
@@ -836,7 +836,7 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) {
 			format:    config.Global.TextFormat,
 			qname:     "dnscollector.fr",
 			identity:  "collector",
-			expected:  "-collectorCLIENT_QUERYNOERROR1.2.3.41234--0bdnscollector.frA-",
+			expected:  "-collectorCLIENT_QUERYNOERROR1.2.3.41234--0bdnscollector.frA0.000000000",
 		},
 		{
 			name:      "qname_quote",
@@ -845,7 +845,7 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) {
 			format:    config.Global.TextFormat,
 			qname:     "dns collector.fr",
 			identity:  "collector",
-			expected:  "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b \"dns collector.fr\" A -",
+			expected:  "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b \"dns collector.fr\" A 0.000000000",
 		},
 		{
 			name:      "default_boundary",
@@ -854,7 +854,7 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) {
 			format:    config.Global.TextFormat,
 			qname:     "dns\"coll tor\".fr",
 			identity:  "collector",
-			expected:  "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b \"dns\\\"coll tor\\\".fr\" A -",
+			expected:  "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b \"dns\\\"coll tor\\\".fr\" A 0.000000000",
 		},
 		{
 			name:      "custom_boundary",
@@ -863,7 +863,7 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) {
 			format:    config.Global.TextFormat,
 			qname:     "dnscoll tor.fr",
 			identity:  "collector",
-			expected:  "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b !dnscoll tor.fr! A -",
+			expected:  "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b !dnscoll tor.fr! A 0.000000000",
 		},
 		{
 			name:      "custom_text",
@@ -939,8 +939,8 @@ func TestDnsMessage_TextFormat_DefaultDirectives(t *testing.T) {
 		},
 		{
 			format:   "latency",
-			dm:       DNSMessage{DNSTap: DNSTap{LatencySec: "0.00001"}},
-			expected: "0.00001",
+			dm:       DNSMessage{DNSTap: DNSTap{Latency: 0.00001}},
+			expected: "0.000010000",
 		},
 		{
 			format:   "qname qtype opcode",

docs/_examples/use-case-1.yml

@@ -1,18 +1,28 @@
 # This configuration sets up DNS traffic monitoring through DNStap on port 6000
 # and logging in both text and pcap formats.
+#
+# As prerequisites, we assume you have a DNS server which supports DNSTap (unbound, bind, powerdns, etc)
+# For more informations about dnstap, read the following page: https://dmachard.github.io/posts/0001-dnstap-testing/
+#
 
+# If turned on, debug messages are printed in the standard output
 global:
   trace:
     verbose: true 
 
 pipelines:
+  # Listen on tcp/6000 for incoming DNSTap protobuf messages from dns servers
   - name: tap
     dnstap:
       listen-ip: 0.0.0.0
       listen-port: 6000
     routing-policy:
       forward: [ text, pcap ]
 
+  # Write DNS logs to log file in text format and pcap
+  # with a maximum size of 100Mb for each files
+  # A rotation mechanism is implemented with 10 files maximum
+  # more detail about the text format: doc/configuration.md#custom-text-format
   - name: text
     logfile:
       file-path:  "/tmp/dnstap.log"

docs/_examples/use-case-10.yml

@@ -1,18 +1,28 @@
 # This configuration sets up DNS traffic monitoring through DNStap on port 6000;
 # and applies tranformation to reduce qname to lowercase
+#
+# As prerequisites, we assume you have 
+# - a DNS server which supports DNSTap (unbound, bind, powerdns, etc) for more informations about dnstap, 
+#   read the following page: https://dmachard.github.io/posts/0001-dnstap-testing/
 
+# If turned on, debug messages are printed in the standard output
 global:
   trace:
     verbose: true
 
 pipelines:
+  # Listen on tcp/6000 for incoming DNSTap protobuf messages from dns servers
   - name: tap
     dnstap:
       listen-ip: 0.0.0.0
       listen-port: 6000
+    # Routes DNS messages from the tap collector to standard output
     routing-policy:
       forward: [ console ]
 
+  # Print DNS messages on standard output with TEXT format
+  # with on tranformation to reduce qname to lowercase
+  # For example: Wwww.GooGlE.com will be equal to www.google.com
   - name: console
     stdout:
       mode: text

docs/_examples/use-case-11.yml

@@ -1,21 +1,31 @@
 # This configuration sets up DNS traffic monitoring through DNStap on port 6000;
 # and add geographical metadata with GeoIP database
+#
+# As prerequisites, we assume you have 
+# - a DNS server which supports DNSTap (unbound, bind, powerdns, etc) for more informations about dnstap, 
+#   read the following page: https://dmachard.github.io/posts/0001-dnstap-testing/
 
+# If turned on, debug messages are printed in the standard output
 global:
   trace:
     verbose: true
 
 pipelines:
+  # Listen on tcp/6000 for incoming DNSTap protobuf messages from dns servers
+  # and try to add country name in metadata
   - name: tap
     dnstap:
       listen-ip: 0.0.0.0
       listen-port: 6000
     transforms:
       geoip:
         mmdb-country-file: "./tests/testsdata/GeoLite2-Country.mmdb"
+    # Routes DNS messages from the tap collector to standard output
     routing-policy:
       forward: [ console ]
 
+  # Print DNS messages on standard output with TEXT format
+  # Configure a custom text format to display the country name
   - name: console
     stdout:
       mode: text