From f1d1168a7189dfcb683662312378fa286b12d99d Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Tue, 25 Jun 2024 08:23:58 +0200 Subject: [PATCH 1/2] update normalize transform to replace non printable characters --- Makefile | 2 +- config.yml | 3 +- docs/transformers/transform_normalize.md | 7 ++++- pkgconfig/transformers.go | 13 +++++---- transformers/normalize.go | 36 ++++++++++++++++++++---- 5 files changed, 46 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 1ffafb69..b523ccc9 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ GO_VERSION := $(shell go env GOVERSION | sed -n 's/go\([0-9]\+\.[0-9]\+\).*/\1/p GO_LOGGER := 1.0.0 GO_POWERDNS_PROTOBUF := 1.1.1 -GO_DNSTAP_PROTOBUF := 1.0.1 +GO_DNSTAP_PROTOBUF := 1.0.2 GO_FRAMESTREAM := 0.10.0 GO_CLIENTSYSLOG := 0.4.0 GO_TOPMAP := 1.0.0 diff --git a/config.yml b/config.yml index 31074530..f625124f 100644 --- a/config.yml +++ b/config.yml @@ -41,13 +41,14 @@ pipelines: transforms: normalize: qname-lowercase: true + qname-replace-nonprintable: true routing-policy: forward: [ console ] dropped: [ ] - name: console stdout: - mode: jinja + mode: text ################################################ # DEPRECATED - multiplexer configuration diff --git a/docs/transformers/transform_normalize.md b/docs/transformers/transform_normalize.md index f83c82a0..233025e4 100644 --- a/docs/transformers/transform_normalize.md +++ b/docs/transformers/transform_normalize.md @@ -15,6 +15,10 @@ Options: *`rr-lowercase` (boolean) > enable or disable lowercase for all resources records +*`qname-replace-nonprintable` (boolean) + > replace non printable characters with decimal value + > the domain `"invalid\tinvalid . com"` will be `invalid\009invalid\032.\032com` + *`add-tld` (boolean) > add top level domain @@ -27,7 +31,8 @@ Options: ```yaml transforms: normalize: - qname-lowercase: true + qname-lowercase: false + qname-replace-nonprintable: false rr-lowercase: false add-tld: false add-tld-plus-one: false diff --git a/pkgconfig/transformers.go b/pkgconfig/transformers.go index 1d82119b..37be6ff9 100644 --- a/pkgconfig/transformers.go +++ b/pkgconfig/transformers.go @@ -23,12 +23,13 @@ type ConfigTransformers struct { HashIPAlgo string `yaml:"hash-ip-algo" default:"sha1"` } `yaml:"user-privacy"` Normalize struct { - Enable bool `yaml:"enable" default:"false"` - QnameLowerCase bool `yaml:"qname-lowercase" default:"false"` - RRLowerCase bool `yaml:"rr-lowercase" default:"false"` - QuietText bool `yaml:"quiet-text" default:"false"` - AddTld bool `yaml:"add-tld" default:"false"` - AddTldPlusOne bool `yaml:"add-tld-plus-one" default:"false"` + Enable bool `yaml:"enable" default:"false"` + QnameLowerCase bool `yaml:"qname-lowercase" default:"false"` + RRLowerCase bool `yaml:"rr-lowercase" default:"false"` + QuietText bool `yaml:"quiet-text" default:"false"` + AddTld bool `yaml:"add-tld" default:"false"` + AddTldPlusOne bool `yaml:"add-tld-plus-one" default:"false"` + ReplaceNonPrintable bool `yaml:"qname-replace-nonprintable" default:"false"` } `yaml:"normalize"` Latency struct { Enable bool `yaml:"enable" default:"false"` diff --git a/transformers/normalize.go b/transformers/normalize.go index 895873ab..98220631 100644 --- a/transformers/normalize.go +++ b/transformers/normalize.go @@ -1,7 +1,9 @@ package transformers import ( + "fmt" "strings" + "unicode" "github.com/dmachard/go-dnscollector/dnsutils" "github.com/dmachard/go-dnscollector/pkgconfig" @@ -56,20 +58,22 @@ func NewNormalizeTransform(config *pkgconfig.ConfigTransformers, logger *logger. func (t *NormalizeTransform) GetTransforms() ([]Subtransform, error) { subprocessors := []Subtransform{} - if t.config.Normalize.RRLowerCase { + if t.config.Normalize.Enable && t.config.Normalize.ReplaceNonPrintable { + subprocessors = append(subprocessors, Subtransform{name: "normalize:qname-replace-nonprintable", processFunc: t.ReplaceNonprintable}) + } + if t.config.Normalize.Enable && t.config.Normalize.RRLowerCase { subprocessors = append(subprocessors, Subtransform{name: "normalize:rr-lowercase", processFunc: t.RRLowercase}) } - if t.config.Normalize.QnameLowerCase { + if t.config.Normalize.Enable && t.config.Normalize.QnameLowerCase { subprocessors = append(subprocessors, Subtransform{name: "normalize:qname-lowercase", processFunc: t.QnameLowercase}) } - if t.config.Normalize.QuietText { + if t.config.Normalize.Enable && t.config.Normalize.QuietText { subprocessors = append(subprocessors, Subtransform{name: "normalize:quiet", processFunc: t.QuietText}) } - - if t.config.Normalize.AddTld { + if t.config.Normalize.Enable && t.config.Normalize.AddTld { subprocessors = append(subprocessors, Subtransform{name: "normalize:add-etld", processFunc: t.GetEffectiveTld}) } - if t.config.Normalize.AddTldPlusOne { + if t.config.Normalize.Enable && t.config.Normalize.AddTldPlusOne { subprocessors = append(subprocessors, Subtransform{name: "normalize:add-etld+1", processFunc: t.GetEffectiveTldPlusOne}) } return subprocessors, nil @@ -87,6 +91,26 @@ func (t *NormalizeTransform) RRLowercase(dm *dnsutils.DNSMessage) (int, error) { return ReturnKeep, nil } +func (t *NormalizeTransform) ReplaceNonprintable(dm *dnsutils.DNSMessage) (int, error) { + + var builder strings.Builder + qname := dm.DNS.Qname + for _, r := range qname { + if unicode.IsPrint(r) { + if unicode.IsSpace(r) { + builder.WriteString(fmt.Sprintf("\\%03d", r)) + } else { + builder.WriteRune(r) + } + } else { + builder.WriteString(fmt.Sprintf("\\%03d", r)) + } + } + dm.DNS.Qname = builder.String() + + return ReturnKeep, nil +} + func (t *NormalizeTransform) QuietText(dm *dnsutils.DNSMessage) (int, error) { if v, found := DnstapMessage[dm.DNSTap.Operation]; found { dm.DNSTap.Operation = v From 81bc64b8dc864085c4452bda900fe4f0f8f0409f Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Tue, 25 Jun 2024 08:29:42 +0200 Subject: [PATCH 2/2] fix doc --- docs/transformers/transform_normalize.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/transformers/transform_normalize.md b/docs/transformers/transform_normalize.md index 233025e4..82a5e8d3 100644 --- a/docs/transformers/transform_normalize.md +++ b/docs/transformers/transform_normalize.md @@ -6,26 +6,27 @@ This transformer can be used: - to add top level domain. For example for `books.amazon.co.uk`, the `TLD` is `co.uk` and the `TLD+1` is `amazon.co.uk`. - to use small text form. For example: `CLIENT_QUERY` will be replaced by `CQ` +- to replace or remove non-printable characters Options: -*`qname-lowercase` (boolean) +* `qname-lowercase` (boolean) > enable or disable lowercase -*`rr-lowercase` (boolean) +* `rr-lowercase` (boolean) > enable or disable lowercase for all resources records -*`qname-replace-nonprintable` (boolean) +* `qname-replace-nonprintable` (boolean) > replace non printable characters with decimal value > the domain `"invalid\tinvalid . com"` will be `invalid\009invalid\032.\032com` -*`add-tld` (boolean) +* `add-tld` (boolean) > add top level domain -*`add-tld-plus-one` (boolean) +* `add-tld-plus-one` (boolean) > add top level domain plus one label -*`quiet-text` (boolean) +* `quiet-text` (boolean) > Quiet text mode to reduce the size of the logs ```yaml @@ -73,6 +74,6 @@ Example: Specific directives added for text format: -*`publicsuffix-tld`: [Public Suffix](https://publicsuffix.org/) of the DNS QNAME -*`publicsuffix-etld+1`: [Public Suffix](https://publicsuffix.org/) plus one label of the DNS QNAME -*`publicsuffix-managed-icann`: [Public Suffix](https://publicsuffix.org/) flag for managed icann domains +* `publicsuffix-tld`: [Public Suffix](https://publicsuffix.org/) of the DNS QNAME +* `publicsuffix-etld+1`: [Public Suffix](https://publicsuffix.org/) plus one label of the DNS QNAME +* `publicsuffix-managed-icann`: [Public Suffix](https://publicsuffix.org/) flag for managed icann domains