From 0ef99aa8c74202947fc2591b7eb9059fc964fb3d Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Fri, 21 Jun 2024 20:42:09 +0200 Subject: [PATCH] add custom text format with jinja templating --- README.md | 2 +- config.yml | 5 +-- dnsutils/message.go | 19 +++++++++++ dnsutils/message_test.go | 60 +++++++++++++++++++++++++++++++++++ docs/configuration.md | 25 ++++++++++++--- docs/loggers/logger_file.md | 4 +-- docs/loggers/logger_stdout.md | 4 +-- docs/performance.md | 16 ++++++---- go.mod | 1 + go.sum | 2 ++ pkgconfig/constants.go | 1 + pkgconfig/global.go | 2 +- workers/dnsprocessor.go | 3 ++ workers/dnstapserver.go | 3 ++ workers/logfile.go | 11 +++++++ workers/stdout.go | 9 ++++++ 16 files changed, 148 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 967de097..25bd2b03 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Go Report Go version Go tests -Go bench +Go bench Go lines

diff --git a/config.yml b/config.yml index 8dda8063..31074530 100644 --- a/config.yml +++ b/config.yml @@ -6,10 +6,11 @@ global: trace: verbose: true server-identity: "dns-collector" + pid-file: "" text-format: "timestamp-rfc3339ns identity operation rcode queryip queryport family protocol length-unit qname qtype latency" text-format-delimiter: " " text-format-boundary: "\"" - pid-file: "" + text-jinja: "" worker: interval-monitor: 10 buffer-size: 4096 @@ -46,7 +47,7 @@ pipelines: - name: console stdout: - mode: text + mode: jinja ################################################ # DEPRECATED - multiplexer configuration diff --git a/dnsutils/message.go b/dnsutils/message.go index d26b8432..0ce65015 100644 --- a/dnsutils/message.go +++ b/dnsutils/message.go @@ -19,6 +19,7 @@ import ( "github.com/dmachard/go-dnscollector/pkgconfig" "github.com/dmachard/go-dnstap-protobuf" "github.com/dmachard/go-netutils" + "github.com/flosch/pongo2/v6" "github.com/google/gopacket" "github.com/google/gopacket/layers" "github.com/miekg/dns" @@ -118,6 +119,8 @@ type DNS struct { Qname string `json:"qname"` Qclass string `json:"qclass"` + QuestionsCount int `json:"questions-count"` + Qtype string `json:"qtype"` Flags DNSFlags `json:"flags"` DNSRRs DNSRRs `json:"resource-records"` @@ -872,6 +875,22 @@ func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBo return []byte(s.String()), nil } +func (dm *DNSMessage) ToTextTemplate(template string) (string, error) { + context := pongo2.Context{"dm": dm} + + // Parse and execute the template + tmpl, err := pongo2.FromString(template) + if err != nil { + return "", err + } + + result, err := tmpl.Execute(context) + if err != nil { + return "", err + } + return result, nil +} + func (dm *DNSMessage) ToJSON() string { buffer := new(bytes.Buffer) json.NewEncoder(buffer).Encode(dm) diff --git a/dnsutils/message_test.go b/dnsutils/message_test.go index 44d170ae..e3683d87 100644 --- a/dnsutils/message_test.go +++ b/dnsutils/message_test.go @@ -213,6 +213,7 @@ func TestDnsMessage_Json_Reference(t *testing.T) { "qname": "-", "qtype": "-", "qclass": "-", + "questions-count": 0, "flags": { "qr": false, "tc": false, @@ -1556,3 +1557,62 @@ func BenchmarkDnsMessage_ToFlatten(b *testing.B) { } } } + +// To jinja templating +func TestDnsMessage_ToJinjaFormat(t *testing.T) { + dm := DNSMessage{} + dm.Init() + + dm.DNS.Qname = "qname_for_test" + + template := ` +;; Got {% if dm.DNS.Type == "QUERY" %}query{% else %}answer{% endif %} from {{ dm.NetworkInfo.QueryIP }}#{{ dm.NetworkInfo.QueryPort }}: +;; ->>HEADER<<- opcode: {{ dm.DNS.Opcode }}, status: {{ dm.DNS.Rcode }}, id: {{ dm.DNS.ID }} +;; flags: {{ dm.DNS.Flags.QR | yesno:"qr ," }}{{ dm.DNS.Flags.RD | yesno:"rd ," }}{{ dm.DNS.Flags.RA | yesno:"ra ," }}; QUERY: {{ dm.DNS.QuestionsCount }}, ANSWER: {{ dm.DNS.DNSRRs.Answers | length }}, AUTHORITY: {{ dm.DNS.DNSRRs.Nameservers | length }}, ADDITIONAL: {{ dm.DNS.DNSRRs.Records | length }} + +;; QUESTION SECTION: +;{{ dm.DNS.Qname }} {{ dm.DNS.Qclass }} {{ dm.DNS.Qtype }} + +;; ANSWER SECTION: {% for rr in dm.DNS.DNSRRs.Answers %} +{{ rr.Name }} {{ rr.TTL }} {{ rr.Class }} {{ rr.Rdatatype }} {{ rr.Rdata }}{% endfor %} + +;; WHEN: {{ dm.DNSTap.Timestamp }} +;; MSG SIZE rcvd: {{ dm.DNS.Length }}` + + text, err := dm.ToTextTemplate(template) + if err != nil { + t.Errorf("Want no error, got: %s", err) + } + + if !strings.Contains(text, dm.DNS.Qname) { + t.Errorf("Want qname in template, got: %s", text) + } +} + +func BenchmarkDnsMessage_ToJinjaFormat(b *testing.B) { + dm := DNSMessage{} + dm.Init() + dm.InitTransforms() + + template := ` +;; Got {% if dm.DNS.Type == "QUERY" %}query{% else %}answer{% endif %} from {{ dm.NetworkInfo.QueryIP }}#{{ dm.NetworkInfo.QueryPort }}: +;; ->>HEADER<<- opcode: {{ dm.DNS.Opcode }}, status: {{ dm.DNS.Rcode }}, id: {{ dm.DNS.ID }} +;; flags: {{ dm.DNS.Flags.QR | yesno:"qr ," }}{{ dm.DNS.Flags.RD | yesno:"rd ," }}{{ dm.DNS.Flags.RA | yesno:"ra ," }}; QUERY: {{ dm.DNS.QuestionsCount }}, ANSWER: {{ dm.DNS.DNSRRs.Answers | length }}, AUTHORITY: {{ dm.DNS.DNSRRs.Nameservers | length }}, ADDITIONAL: {{ dm.DNS.DNSRRs.Records | length }} + +;; QUESTION SECTION: +;{{ dm.DNS.Qname }} {{ dm.DNS.Qclass }} {{ dm.DNS.Qtype }} + +;; ANSWER SECTION: {% for rr in dm.DNS.DNSRRs.Answers %} +{{ rr.Name }} {{ rr.TTL }} {{ rr.Class }} {{ rr.Rdatatype }} {{ rr.Rdata }}{% endfor %} + +;; WHEN: {{ dm.DNSTap.Timestamp }} +;; MSG SIZE rcvd: {{ dm.DNS.Length }}` + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := dm.ToTextTemplate(template) + if err != nil { + b.Fatalf("could not encode to template: %v\n", err) + } + } +} diff --git a/docs/configuration.md b/docs/configuration.md index 8bafcad6..eb8bc826 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -123,13 +123,11 @@ global: text-format-delimiter: " " text-format-splitter: " " text-format-boundary: "\"" - + text-jinja: "" ``` If you require a output format like CSV, the delimiter can be configured with the `text-format-delimiter` option. -The default separator is [space]. - -text-format can contain raw text enclosed by curly braces, eg +The default separator is [space]. text-format can contain raw text enclosed by curly braces, eg ```yaml text-format: "timestamp-rfc3339ns identity operation rcode queryip queryport qname qtype {DNSTAP}" @@ -145,6 +143,25 @@ Output example: ``` +If you want a more flexible format, you can use the `text-jinja` setting +Example to enable output similiar to dig style: + +``` +text-jinja: |+ + ;; Got {% if dm.DNS.Type == "QUERY" %}query{% else %}answer{% endif %} from {{ dm.NetworkInfo.QueryIP }}#{{ dm.NetworkInfo.QueryPort }}: + ;; ->>HEADER<<- opcode: {{ dm.DNS.Opcode }}, status: {{ dm.DNS.Rcode }}, id: {{ dm.DNS.ID }} + ;; flags: {{ dm.DNS.Flags.QR | yesno:"qr ," }}{{ dm.DNS.Flags.RD | yesno:"rd ," }}{{ dm.DNS.Flags.RA | yesno:"ra ," }}; QUERY: {{ dm.DNS.QuestionsCount }}, ANSWER: {{ dm.DNS.DNSRRs.Answers | length }}, AUTHORITY: {{ dm.DNS.DNSRRs.Nameservers | length }}, ADDITIONAL: {{ dm.DNS.DNSRRs.Records | length }} + + ;; QUESTION SECTION: + ;{{ dm.DNS.Qname }} {{ dm.DNS.Qclass }} {{ dm.DNS.Qtype }} + + ;; ANSWER SECTION: {% for rr in dm.DNS.DNSRRs.Answers %} + {{ rr.Name }} {{ rr.TTL }} {{ rr.Class }} {{ rr.Rdatatype }} {{ rr.Rdata }}{% endfor %} + + ;; WHEN: {{ dm.DNSTap.Timestamp }} + ;; MSG SIZE rcvd: {{ dm.DNS.Length }} +``` + ## Pid file Set path to create DNS-collector PID. diff --git a/docs/loggers/logger_file.md b/docs/loggers/logger_file.md index 0e2ff42d..28a49119 100644 --- a/docs/loggers/logger_file.md +++ b/docs/loggers/logger_file.md @@ -3,7 +3,7 @@ Enable this logger if you want to log your DNS traffic to a file in plain text mode or binary mode. * with rotation file support -* supported format: `text`, `json` and `flat json`, `pcap` or `dnstap` +* supported format: `text`, `jinja`, `json` and `flat json`, `pcap` or `dnstap` * gzip compression * execute external command after each rotation * custom text format @@ -39,7 +39,7 @@ Options: > run external script after file compress step * `mode` (string) - > output format: text, json, flat-json, pcap or dnstap + > output format: text, jinja, json, flat-json, pcap or dnstap * `text-format` (string) > output text format, please refer to the default text format to see all diff --git a/docs/loggers/logger_stdout.md b/docs/loggers/logger_stdout.md index d1ec7809..9b66ff50 100644 --- a/docs/loggers/logger_stdout.md +++ b/docs/loggers/logger_stdout.md @@ -3,13 +3,13 @@ Print to your standard output, all DNS logs received * in text or json format -* custom text format +* custom text format (with jinja templating support) * binary mode (pcap) Options: * `mode` (string) - > output format: `text`, `json`, `flat-json` or `pcap` + > output format: `text`, `jinja`, `json`, `flat-json` or `pcap` * `text-format` (string) > output text format, please refer to the default text format to see all available [directives](../configuration.md#custom-text-format), use this parameter if you want a specific format diff --git a/docs/performance.md b/docs/performance.md index 1833a98c..ab80ad13 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -18,13 +18,15 @@ goos: linux goarch: amd64 pkg: github.com/dmachard/go-dnscollector/dnsutils cpu: Intel(R) Core(TM) i5-7200U CPU @ 2.50GHz -BenchmarkDnsMessage_ToTextFormat-4 2555529 450.2 ns/op 80 B/op 4 allocs/op -BenchmarkDnsMessage_ToPacketLayer-4 1138892 952.0 ns/op 1144 B/op 12 allocs/op -BenchmarkDnsMessage_ToDNSTap-4 1036468 1136 ns/op 592 B/op 18 allocs/op -BenchmarkDnsMessage_ToExtendedDNSTap-4 612438 1970 ns/op 1056 B/op 25 allocs/op -BenchmarkDnsMessage_ToJSON-4 188379 6724 ns/op 3632 B/op 3 allocs/op -BenchmarkDnsMessage_ToFlatten-4 121525 10151 ns/op 8215 B/op 29 allocs/op -BenchmarkDnsMessage_ToFlatJSON-4 20704 58365 ns/op 22104 B/op 220 allocs/op +BenchmarkDnsMessage_ToTextFormat-4 2262946 518.8 ns/op 80 B/op 4 allocs/op +BenchmarkDnsMessage_ToPacketLayer-4 1241736 926.9 ns/op 1144 B/op 12 allocs/op +BenchmarkDnsMessage_ToDNSTap-4 894579 1464 ns/op 592 B/op 18 allocs/op +BenchmarkDnsMessage_ToExtendedDNSTap-4 608203 2342 ns/op 1056 B/op 25 allocs/op +BenchmarkDnsMessage_ToJSON-4 130080 7749 ns/op 3632 B/op 3 allocs/op +BenchmarkDnsMessage_ToFlatten-4 117115 9227 ns/op 8369 B/op 29 allocs/op +BenchmarkDnsMessage_ToFlatJSON-4 21238 54535 ns/op 20106 B/op 219 allocs/op +BenchmarkDnsMessage_ToFlatten_Relabelling-4 35614 32544 ns/op 8454 B/op 30 allocs/op +BenchmarkDnsMessage_ToJinjaFormat-4 9840 120301 ns/op 50093 B/op 959 allocs/op ``` ## Memory usage diff --git a/go.mod b/go.mod index 56c4b99e..93252120 100644 --- a/go.mod +++ b/go.mod @@ -72,6 +72,7 @@ require ( github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect github.com/fatih/color v1.15.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/flosch/pongo2/v6 v6.0.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.1 // indirect diff --git a/go.sum b/go.sum index 7ab1880c..4f6da7cc 100644 --- a/go.sum +++ b/go.sum @@ -204,6 +204,8 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flosch/pongo2/v6 v6.0.0 h1:lsGru8IAzHgIAw6H2m4PCyleO58I40ow6apih0WprMU= +github.com/flosch/pongo2/v6 v6.0.0/go.mod h1:CuDpFm47R0uGGE7z13/tTlt1Y6zdxvr2RLT5LJhsHEU= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= diff --git a/pkgconfig/constants.go b/pkgconfig/constants.go index 9c976027..b097679a 100644 --- a/pkgconfig/constants.go +++ b/pkgconfig/constants.go @@ -18,6 +18,7 @@ const ( BadVeryLongDomain = "ultramegaverytoolonglabel.dnscollector" + badLongLabel + badLongLabel + badLongLabel + badLongLabel + badLongLabel + ".dev." + ModeJinja = "jinja" ModeText = "text" ModeJSON = "json" ModeFlatJSON = "flat-json" diff --git a/pkgconfig/global.go b/pkgconfig/global.go index 30ba19f9..e6df7010 100644 --- a/pkgconfig/global.go +++ b/pkgconfig/global.go @@ -9,8 +9,8 @@ import ( type ConfigGlobal struct { TextFormat string `yaml:"text-format" default:"timestamp identity operation rcode queryip queryport family protocol length-unit qname qtype latency"` TextFormatDelimiter string `yaml:"text-format-delimiter" default:" "` - TextFormatSplitter string `yaml:"text-format-splitter" default:" "` TextFormatBoundary string `yaml:"text-format-boundary" default:"\""` + TextJinja string `yaml:"text-jinja" default:""` Trace struct { Verbose bool `yaml:"verbose" default:"false"` LogMalformed bool `yaml:"log-malformed" default:"false"` diff --git a/workers/dnsprocessor.go b/workers/dnsprocessor.go index a5ff1bad..3d5deca0 100644 --- a/workers/dnsprocessor.go +++ b/workers/dnsprocessor.go @@ -61,6 +61,9 @@ func (w *DNSProcessor) StartCollect() { w.LogError("dns parser malformed packet: %s - %v+", err, dm) } + // get number of questions + dm.DNS.QuestionsCount = dnsHeader.Qdcount + // dns reply ? if dnsHeader.Qr == 1 { dm.DNSTap.Operation = "CLIENT_RESPONSE" diff --git a/workers/dnstapserver.go b/workers/dnstapserver.go index cb7c6ab9..11506e87 100644 --- a/workers/dnstapserver.go +++ b/workers/dnstapserver.go @@ -512,6 +512,9 @@ func (w *DNSTapProcessor) StartCollect() { } } + // get number of questions + dm.DNS.QuestionsCount = dnsHeader.Qdcount + if err = dnsutils.DecodePayload(&dm, &dnsHeader, w.GetConfig()); err != nil { dm.DNS.MalformedPacket = true w.LogInfo("dns payload parser stopped: %s", err) diff --git a/workers/logfile.go b/workers/logfile.go index c2e8b849..d9bd1477 100644 --- a/workers/logfile.go +++ b/workers/logfile.go @@ -34,6 +34,7 @@ const ( func IsValid(mode string) bool { switch mode { case + pkgconfig.ModeJinja, pkgconfig.ModeText, pkgconfig.ModeJSON, pkgconfig.ModeFlatJSON, @@ -506,11 +507,21 @@ func (w *LogFile) StartLogging() { delimiter.WriteString("\n") w.WriteToPlain(delimiter.Bytes()) + // with custom text mode + case pkgconfig.ModeJinja: + textLine, err := dm.ToTextTemplate(w.GetConfig().Global.TextJinja) + if err != nil { + w.LogError("jinja template: %s", err) + continue + } + w.WriteToPlain([]byte(textLine)) + // with json mode case pkgconfig.ModeFlatJSON: flat, err := dm.Flatten() if err != nil { w.LogError("flattening DNS message failed: %e", err) + continue } json.NewEncoder(buffer).Encode(flat) w.WriteToPlain(buffer.Bytes()) diff --git a/workers/stdout.go b/workers/stdout.go index 92f44c84..02562853 100644 --- a/workers/stdout.go +++ b/workers/stdout.go @@ -20,6 +20,7 @@ import ( func IsStdoutValidMode(mode string) bool { switch mode { case + pkgconfig.ModeJinja, pkgconfig.ModeText, pkgconfig.ModeJSON, pkgconfig.ModeFlatJSON, @@ -186,6 +187,14 @@ func (w *StdOut) StartLogging() { case pkgconfig.ModeText: w.writerText.Print(dm.String(w.textFormat, w.GetConfig().Global.TextFormatDelimiter, w.GetConfig().Global.TextFormatBoundary)) + case pkgconfig.ModeJinja: + textLine, err := dm.ToTextTemplate(w.GetConfig().Global.TextJinja) + if err != nil { + w.LogError("process: unable to update template: %s", err) + continue + } + w.writerText.Print(textLine) + case pkgconfig.ModeJSON: json.NewEncoder(buffer).Encode(dm) w.writerText.Print(buffer.String())