add rewrite transform + quote some text directives (#756)

* add rewrite transform & escape dnstap version directive
* quote peer name and identity
* update docs
* ignore other types

README.md

@@ -1,9 +1,9 @@
 <p align="center">
 <img src="https://goreportcard.com/badge/github.com/dmachard/go-dns-collector" alt="Go Report"/>
 <img src="https://img.shields.io/badge/go%20version-min%201.21-green" alt="Go version"/>
-<img src="https://img.shields.io/badge/go%20tests-439-green" alt="Go tests"/>
+<img src="https://img.shields.io/badge/go%20tests-447-green" alt="Go tests"/>
 <img src="https://img.shields.io/badge/go%20bench-21-green" alt="Go bench"/>
-<img src="https://img.shields.io/badge/go%20lines-30949-green" alt="Go lines"/>
+<img src="https://img.shields.io/badge/go%20lines-31309-green" alt="Go lines"/>
 </p>
 
 <p align="center">
@@ -68,8 +68,8 @@
 
 - **[Transformers](./docs/transformers.md)**
 
-  - Custom [Relabeling](docs/transformers/transform_relabeling.md) for JSON structure
-  - Add additionnal [Tags](docs/transformers/transform_atags.md)
+  - [Rewrite](docs/transformers/transform_rewrite.md) DNS messages or custom [Relabeling](docs/transformers/transform_relabeling.md) for JSON output
+  - Add additionnal [Tags](docs/transformers/transform_atags.md) in DNS messages
   - Traffic [Filtering](docs/transformers/transform_trafficfiltering.md) and [Reducer](docs/transformers/transform_trafficreducer.md)
   - Latency [Computing](docs/transformers/transform_latency.md)
   - Apply [User Privacy](docs/transformers/transform_userprivacy.md)

dnsutils/message.go

@@ -673,26 +673,26 @@ func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBo
 			if len(qname) == 0 {
 				s.WriteString(".")
 			} else {
-				if len(fieldDelimiter) > 0 {
-					if strings.Contains(qname, fieldDelimiter) {
-						qnameEscaped := qname
-						if strings.Contains(qname, fieldBoundary) {
-							qnameEscaped = strings.ReplaceAll(qnameEscaped, fieldBoundary, "\\"+fieldBoundary)
-						}
-						s.WriteString(fmt.Sprintf(fieldBoundary+"%s"+fieldBoundary, qnameEscaped))
-					} else {
-						s.WriteString(qname)
-					}
-				} else {
-					s.WriteString(qname)
-				}
+				quoteStringAndWrite(&s, qname, fieldDelimiter, fieldBoundary)
 			}
 		case directive == "identity":
-			s.WriteString(dm.DNSTap.Identity)
+			if len(dm.DNSTap.Identity) == 0 {
+				s.WriteString("-")
+			} else {
+				quoteStringAndWrite(&s, dm.DNSTap.Identity, fieldDelimiter, fieldBoundary)
+			}
 		case directive == "peer-name":
-			s.WriteString(dm.DNSTap.PeerName)
+			if len(dm.DNSTap.PeerName) == 0 {
+				s.WriteString("-")
+			} else {
+				quoteStringAndWrite(&s, dm.DNSTap.PeerName, fieldDelimiter, fieldBoundary)
+			}
 		case directive == "version":
-			s.WriteString(dm.DNSTap.Version)
+			if len(dm.DNSTap.Version) == 0 {
+				s.WriteString("-")
+			} else {
+				quoteStringAndWrite(&s, dm.DNSTap.Version, fieldDelimiter, fieldBoundary)
+			}
 		case directive == "extra":
 			s.WriteString(dm.DNSTap.Extra)
 		case directive == "policy-rule":
@@ -1171,6 +1171,7 @@ func (dm *DNSMessage) Flatten() (map[string]interface{}, error) {
 		"dns.qtype":                  dm.DNS.Qtype,
 		"dns.qclass":                 dm.DNS.Qclass,
 		"dns.rcode":                  dm.DNS.Rcode,
+		"dns.questions-count":        dm.DNS.QuestionsCount,
 		"dnstap.identity":            dm.DNSTap.Identity,
 		"dnstap.latency":             dm.DNSTap.LatencySec,
 		"dnstap.operation":           dm.DNSTap.Operation,
@@ -1858,7 +1859,9 @@ func GetFakeDNSMessage() DNSMessage {
 	dm := DNSMessage{}
 	dm.Init()
 	dm.DNSTap.Identity = "collector"
+	dm.DNSTap.Version = "dnscollector 1.0.0"
 	dm.DNSTap.Operation = "CLIENT_QUERY"
+	dm.DNSTap.PeerName = "localhost (127.0.0.1)"
 	dm.DNS.Type = DNSQuery
 	dm.DNS.Qname = pkgconfig.ProgQname
 	dm.NetworkInfo.QueryIP = "1.2.3.4"
@@ -1913,3 +1916,19 @@ func convertToString(value interface{}) string {
 		return fmt.Sprintf("%v", v)
 	}
 }
+
+func quoteStringAndWrite(s *strings.Builder, fieldString, fieldDelimiter, fieldBoundary string) {
+	if len(fieldDelimiter) > 0 {
+		if strings.Contains(fieldString, fieldDelimiter) {
+			fieldEscaped := fieldString
+			if strings.Contains(fieldString, fieldBoundary) {
+				fieldEscaped = strings.ReplaceAll(fieldEscaped, fieldBoundary, "\\"+fieldBoundary)
+			}
+			s.WriteString(fmt.Sprintf(fieldBoundary+"%s"+fieldBoundary, fieldEscaped))
+		} else {
+			s.WriteString(fieldString)
+		}
+	} else {
+		s.WriteString(fieldString)
+	}
+}

dnsutils/message_test.go

@@ -469,6 +469,7 @@ func TestDnsMessage_JsonFlatten_Reference(t *testing.T) {
 					"dns.qtype": "-",
 					"dns.rcode": "-",
 					"dns.qclass": "-",
+					"dns.questions-count": 0,
 					"dns.resource-records.an.0.name": "google.nl",
 					"dns.resource-records.an.0.rdata": "142.251.39.99",
 					"dns.resource-records.an.0.rdatatype": "A",
@@ -807,6 +808,7 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) {
 		boundary  string
 		format    string
 		qname     string
+		identity  string
 		expected  string
 	}{
 		{
@@ -815,64 +817,98 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) {
 			boundary:  config.Global.TextFormatBoundary,
 			format:    config.Global.TextFormat,
 			qname:     "dnscollector.fr",
-			expected:  "- - - - - - - - 0b dnscollector.fr - -",
+			identity:  "collector",
+			expected:  "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b dnscollector.fr A -",
 		},
 		{
 			name:      "custom_delimiter",
 			delimiter: ";",
 			boundary:  config.Global.TextFormatBoundary,
 			format:    config.Global.TextFormat,
 			qname:     "dnscollector.fr",
-			expected:  "-;-;-;-;-;-;-;-;0b;dnscollector.fr;-;-",
+			identity:  "collector",
+			expected:  "-;collector;CLIENT_QUERY;NOERROR;1.2.3.4;1234;-;-;0b;dnscollector.fr;A;-",
 		},
 		{
 			name:      "empty_delimiter",
 			delimiter: "",
 			boundary:  config.Global.TextFormatBoundary,
 			format:    config.Global.TextFormat,
 			qname:     "dnscollector.fr",
-			expected:  "--------0bdnscollector.fr--",
+			identity:  "collector",
+			expected:  "-collectorCLIENT_QUERYNOERROR1.2.3.41234--0bdnscollector.frA-",
 		},
 		{
 			name:      "qname_quote",
 			delimiter: config.Global.TextFormatDelimiter,
 			boundary:  config.Global.TextFormatBoundary,
 			format:    config.Global.TextFormat,
 			qname:     "dns collector.fr",
-			expected:  "- - - - - - - - 0b \"dns collector.fr\" - -",
+			identity:  "collector",
+			expected:  "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b \"dns collector.fr\" A -",
 		},
 		{
 			name:      "default_boundary",
 			delimiter: config.Global.TextFormatDelimiter,
 			boundary:  config.Global.TextFormatBoundary,
 			format:    config.Global.TextFormat,
 			qname:     "dns\"coll tor\".fr",
-			expected:  "- - - - - - - - 0b \"dns\\\"coll tor\\\".fr\" - -",
+			identity:  "collector",
+			expected:  "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b \"dns\\\"coll tor\\\".fr\" A -",
 		},
 		{
 			name:      "custom_boundary",
 			delimiter: config.Global.TextFormatDelimiter,
 			boundary:  "!",
 			format:    config.Global.TextFormat,
 			qname:     "dnscoll tor.fr",
-			expected:  "- - - - - - - - 0b !dnscoll tor.fr! - -",
+			identity:  "collector",
+			expected:  "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b !dnscoll tor.fr! A -",
 		},
 		{
 			name:      "custom_text",
 			delimiter: config.Global.TextFormatDelimiter,
 			boundary:  config.Global.TextFormatBoundary,
 			format:    "qname {IN} qtype",
 			qname:     "dnscollector.fr",
-			expected:  "dnscollector.fr IN -",
+			identity:  "",
+			expected:  "dnscollector.fr IN A",
+		},
+		{
+			name:      "quote_dnstap_version",
+			delimiter: config.Global.TextFormatDelimiter,
+			boundary:  config.Global.TextFormatBoundary,
+			format:    "identity version qname",
+			qname:     "dnscollector.fr",
+			identity:  "collector",
+			expected:  "collector \"dnscollector 1.0.0\" dnscollector.fr",
+		},
+		{
+			name:      "quote_dnstap_identity",
+			delimiter: config.Global.TextFormatDelimiter,
+			boundary:  config.Global.TextFormatBoundary,
+			format:    "identity qname",
+			qname:     "dnscollector.fr",
+			identity:  "dns collector",
+			expected:  "\"dns collector\" dnscollector.fr",
+		},
+		{
+			name:      "quote_dnstap_peername",
+			delimiter: config.Global.TextFormatDelimiter,
+			boundary:  config.Global.TextFormatBoundary,
+			format:    "peer-name qname",
+			qname:     "dnscollector.fr",
+			identity:  "",
+			expected:  "\"localhost (127.0.0.1)\" dnscollector.fr",
 		},
 	}
 
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {
-			dm := DNSMessage{}
-			dm.Init()
+			dm := GetFakeDNSMessage()
 
 			dm.DNS.Qname = tc.qname
+			dm.DNSTap.Identity = tc.identity
 
 			line := dm.String(strings.Fields(tc.format), tc.delimiter, tc.boundary)
 			if line != tc.expected {

docs/development.md

@@ -2,7 +2,7 @@
 # DNS-collector - Development
 
 To compile DNS-collector, we assume you have a working Go setup.
-First, make sure your golang version is `1.20` or higher
+First, make sure your golang version is `1.21` or higher
 
 How to userguides:
 

docs/dnsjson.md

@@ -29,6 +29,7 @@ Example:
     "qtype": "A",
     "id": 23455,
     "qclass": "IN",
+    "questions-count": 0,
     "flags": {
       "qr": true,
       "tc": false,
@@ -116,6 +117,7 @@ Here's a flat JSON output formatted using `jq`:
   "dns.qtype": "A",
   "dns.rcode": "NOERROR",
   "dns.qclass": "IN",
+  "dns.questions-count": 0,
   "dns.resource-records.an.0.name": "google.nl",
   "dns.resource-records.an.0.rdata": "142.251.39.99",
   "dns.resource-records.an.0.rdatatype": "A",

docs/transformers.md

@@ -27,3 +27,4 @@ Transformers processing is currently in this order :
 | [Traffic Prediction](transformers/transform_trafficprediction.md) | Features to train machine learning models              |
 | [Additionnal Tags](transformers/transform_atags.md) | Add additionnal tags |
 | [JSON relabeling](transformers/transform_relabeling.md) | JSON relabeling to rename or remove keys |
+| [DNS message rewrite](transformers/transform_rewrite.md) | Rewrite value for DNS messages structure |

docs/transformers/transform_relabeling.md

@@ -1,7 +1,7 @@
 # Transformer: Relabeling
 
 Use this transformer to remove or rename some JSON keys.
-This transformation is only applied to the [`flat-json`](../dnsjson.md) output format.
+This transformation is only applied to the [`flat-json`](../dnsjson.md#flat-json-format-recommended) output format.
 
 Options:
 
@@ -14,28 +14,27 @@ Options:
 Configuration example
 
 ```yaml
-  loggers:
-    - name: console
-      stdout:
-        mode: flat-json
-      transforms:
-        relabeling:
-          rename:
-            - regex: "dnstap\\.timestamp-rfc3339ns"
-              replacement: "timestamp"
-            - regex: "dns\\.qname"
-              replacement: "query"
-            - regex: "network\\.query-ip"
-              replacement: "client"
-            - regex: "network\\.response-ip"
-              replacement: "server"
-            - regex: "dnstap\\.identity"
-              replacement: "client_id"
-            - regex: "^dns\\.resource-records\\.an\\..*\\.rdata$"
-              replacement: "answers_rdata"
-          remove:
-            - regex: "dns"
-            - regex: "network"
+- name: console
+  stdout:
+    mode: flat-json
+  transforms:
+    relabeling:
+      rename:
+        - regex: "dnstap\\.timestamp-rfc3339ns"
+          replacement: "timestamp"
+        - regex: "dns\\.qname"
+          replacement: "query"
+        - regex: "network\\.query-ip"
+          replacement: "client"
+        - regex: "network\\.response-ip"
+          replacement: "server"
+        - regex: "dnstap\\.identity"
+          replacement: "client_id"
+        - regex: "^dns\\.resource-records\\.an\\..*\\.rdata$"
+          replacement: "answers_rdata"
+      remove:
+        - regex: "dns"
+        - regex: "network"
 ```
 
 This config produces the following flat-json ouput:

docs/transformers/transform_rewrite.md

@@ -0,0 +1,25 @@
+# Transformer: Rewrite
+
+Use this transformer to rewrite the content of DNS messages according to the [structure](../dnsjson.md#dns-collector---json-encoding).
+For more details, see the feature request [here](https://github.com/dmachard/go-dnscollector/issues/527).
+
+> Only fields with int and string types are supported.
+
+Options:
+
+* `identifiers` (map)
+  > Expect a key/value where the key is the namf of the field to rewrite (Please refer  to the [`flat-json`](../dnsjson.md#flat-json-format-recommended) output to see all identifiers keys ) and the value is the new one.
+
+Config example to remove the DNStap version and update the identity name.
+
+```yaml
+- name: tap
+dnstap:
+    listen-ip: 0.0.0.0
+    listen-port: 6000
+transforms:
+    rewrite:
+    identifiers:
+        dnstap.version: ""
+        dnstap.identity: "foo"
+```

pkgconfig/transformers.go

@@ -90,6 +90,10 @@ type ConfigTransformers struct {
 		Rename []RelabelingConfig `yaml:"rename,flow"`
 		Remove []RelabelingConfig `yaml:"remove,flow"`
 	} `yaml:"relabeling"`
+	Rewrite struct {
+		Enable      bool                   `yaml:"enable" default:"false"`
+		Identifiers map[string]interface{} `yaml:"identifiers,flow"`
+	} `yaml:"rewrite"`
 }
 
 func (c *ConfigTransformers) SetDefault() {