update normalize transform to replace non printable characters

Makefile

@@ -4,7 +4,7 @@ GO_VERSION := $(shell go env GOVERSION | sed -n 's/go\([0-9]\+\.[0-9]\+\).*/\1/p
 
 GO_LOGGER := 1.0.0
 GO_POWERDNS_PROTOBUF := 1.1.1
-GO_DNSTAP_PROTOBUF := 1.0.1
+GO_DNSTAP_PROTOBUF := 1.0.2
 GO_FRAMESTREAM := 0.10.0
 GO_CLIENTSYSLOG := 0.4.0
 GO_TOPMAP := 1.0.0

config.yml

@@ -41,13 +41,14 @@ pipelines:
     transforms:
       normalize:
         qname-lowercase: true
+        qname-replace-nonprintable: true
     routing-policy:
       forward: [ console ]
       dropped: [ ]
 
   - name: console
     stdout:
-      mode: jinja
+      mode: text
 
 ################################################
 # DEPRECATED - multiplexer configuration

docs/transformers/transform_normalize.md

@@ -6,28 +6,34 @@ This transformer can be used:
 - to add top level domain. For example for `books.amazon.co.uk`, the `TLD`
 is `co.uk` and the `TLD+1` is `amazon.co.uk`.
 - to use small text form. For example: `CLIENT_QUERY` will be replaced by `CQ`
+- to replace or remove non-printable characters
 
 Options:
 
-*`qname-lowercase` (boolean)
+* `qname-lowercase` (boolean)
   > enable or disable lowercase
 
-*`rr-lowercase` (boolean)
+* `rr-lowercase` (boolean)
   > enable or disable lowercase for all resources records
 
-*`add-tld` (boolean)
+* `qname-replace-nonprintable` (boolean)
+  > replace non printable characters with decimal value
+  > the domain `"invalid\tinvalid . com"` will be `invalid\009invalid\032.\032com`
+
+* `add-tld` (boolean)
   > add top level domain
 
-*`add-tld-plus-one` (boolean)
+* `add-tld-plus-one` (boolean)
   > add top level domain plus one label
 
-*`quiet-text` (boolean)
+* `quiet-text` (boolean)
   > Quiet text mode to reduce the size of the logs
 
 ```yaml
 transforms:
   normalize:
-    qname-lowercase: true
+    qname-lowercase: false
+    qname-replace-nonprintable: false
     rr-lowercase: false
     add-tld: false
     add-tld-plus-one: false
@@ -68,6 +74,6 @@ Example:
 
 Specific directives added for text format:
 
-*`publicsuffix-tld`: [Public Suffix](https://publicsuffix.org/) of the DNS QNAME
-*`publicsuffix-etld+1`: [Public Suffix](https://publicsuffix.org/) plus one label of the DNS QNAME
-*`publicsuffix-managed-icann`: [Public Suffix](https://publicsuffix.org/) flag for managed icann domains
+* `publicsuffix-tld`: [Public Suffix](https://publicsuffix.org/) of the DNS QNAME
+* `publicsuffix-etld+1`: [Public Suffix](https://publicsuffix.org/) plus one label of the DNS QNAME
+* `publicsuffix-managed-icann`: [Public Suffix](https://publicsuffix.org/) flag for managed icann domains

pkgconfig/transformers.go

@@ -23,12 +23,13 @@ type ConfigTransformers struct {
 		HashIPAlgo        string `yaml:"hash-ip-algo" default:"sha1"`
 	} `yaml:"user-privacy"`
 	Normalize struct {
-		Enable         bool `yaml:"enable" default:"false"`
-		QnameLowerCase bool `yaml:"qname-lowercase" default:"false"`
-		RRLowerCase    bool `yaml:"rr-lowercase" default:"false"`
-		QuietText      bool `yaml:"quiet-text" default:"false"`
-		AddTld         bool `yaml:"add-tld" default:"false"`
-		AddTldPlusOne  bool `yaml:"add-tld-plus-one" default:"false"`
+		Enable              bool `yaml:"enable" default:"false"`
+		QnameLowerCase      bool `yaml:"qname-lowercase" default:"false"`
+		RRLowerCase         bool `yaml:"rr-lowercase" default:"false"`
+		QuietText           bool `yaml:"quiet-text" default:"false"`
+		AddTld              bool `yaml:"add-tld" default:"false"`
+		AddTldPlusOne       bool `yaml:"add-tld-plus-one" default:"false"`
+		ReplaceNonPrintable bool `yaml:"qname-replace-nonprintable" default:"false"`
 	} `yaml:"normalize"`
 	Latency struct {
 		Enable            bool `yaml:"enable" default:"false"`

transformers/normalize.go

@@ -1,7 +1,9 @@
 package transformers
 
 import (
+	"fmt"
 	"strings"
+	"unicode"
 
 	"github.com/dmachard/go-dnscollector/dnsutils"
 	"github.com/dmachard/go-dnscollector/pkgconfig"
@@ -56,20 +58,22 @@ func NewNormalizeTransform(config *pkgconfig.ConfigTransformers, logger *logger.
 
 func (t *NormalizeTransform) GetTransforms() ([]Subtransform, error) {
 	subprocessors := []Subtransform{}
-	if t.config.Normalize.RRLowerCase {
+	if t.config.Normalize.Enable && t.config.Normalize.ReplaceNonPrintable {
+		subprocessors = append(subprocessors, Subtransform{name: "normalize:qname-replace-nonprintable", processFunc: t.ReplaceNonprintable})
+	}
+	if t.config.Normalize.Enable && t.config.Normalize.RRLowerCase {
 		subprocessors = append(subprocessors, Subtransform{name: "normalize:rr-lowercase", processFunc: t.RRLowercase})
 	}
-	if t.config.Normalize.QnameLowerCase {
+	if t.config.Normalize.Enable && t.config.Normalize.QnameLowerCase {
 		subprocessors = append(subprocessors, Subtransform{name: "normalize:qname-lowercase", processFunc: t.QnameLowercase})
 	}
-	if t.config.Normalize.QuietText {
+	if t.config.Normalize.Enable && t.config.Normalize.QuietText {
 		subprocessors = append(subprocessors, Subtransform{name: "normalize:quiet", processFunc: t.QuietText})
 	}
-
-	if t.config.Normalize.AddTld {
+	if t.config.Normalize.Enable && t.config.Normalize.AddTld {
 		subprocessors = append(subprocessors, Subtransform{name: "normalize:add-etld", processFunc: t.GetEffectiveTld})
 	}
-	if t.config.Normalize.AddTldPlusOne {
+	if t.config.Normalize.Enable && t.config.Normalize.AddTldPlusOne {
 		subprocessors = append(subprocessors, Subtransform{name: "normalize:add-etld+1", processFunc: t.GetEffectiveTldPlusOne})
 	}
 	return subprocessors, nil
@@ -87,6 +91,26 @@ func (t *NormalizeTransform) RRLowercase(dm *dnsutils.DNSMessage) (int, error) {
 	return ReturnKeep, nil
 }
 
+func (t *NormalizeTransform) ReplaceNonprintable(dm *dnsutils.DNSMessage) (int, error) {
+
+	var builder strings.Builder
+	qname := dm.DNS.Qname
+	for _, r := range qname {
+		if unicode.IsPrint(r) {
+			if unicode.IsSpace(r) {
+				builder.WriteString(fmt.Sprintf("\\%03d", r))
+			} else {
+				builder.WriteRune(r)
+			}
+		} else {
+			builder.WriteString(fmt.Sprintf("\\%03d", r))
+		}
+	}
+	dm.DNS.Qname = builder.String()
+
+	return ReturnKeep, nil
+}
+
 func (t *NormalizeTransform) QuietText(dm *dnsutils.DNSMessage) (int, error) {
 	if v, found := DnstapMessage[dm.DNSTap.Operation]; found {
 		dm.DNSTap.Operation = v