diff --git a/.golangci.yml b/.golangci.yml index e8958ea5..64902ef7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -25,4 +25,4 @@ linters: - gocritic # list of linters to use in the future: - #- gosec + #- gosec \ No newline at end of file diff --git a/README.md b/README.md index 4a58dc16..65fdebbd 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@
- - + +
diff --git a/dnsutils/message.go b/dnsutils/message.go index 5448df0d..73a764e1 100644 --- a/dnsutils/message.go +++ b/dnsutils/message.go @@ -21,7 +21,6 @@ import ( "github.com/google/gopacket" "github.com/google/gopacket/layers" "github.com/miekg/dns" - "github.com/nqd/flat" "google.golang.org/protobuf/proto" ) @@ -303,11 +302,11 @@ func (dm *DNSMessage) InitTransforms() { dm.Geo = &TransformDNSGeo{} } -func (dm *DNSMessage) handleGeoIPDirectives(directives []string, s *strings.Builder) error { +func (dm *DNSMessage) handleGeoIPDirectives(directive string, s *strings.Builder) error { if dm.Geo == nil { s.WriteString("-") } else { - switch directive := directives[0]; { + switch { case directive == "geoip-continent": s.WriteString(dm.Geo.Continent) case directive == "geoip-country": @@ -325,10 +324,17 @@ func (dm *DNSMessage) handleGeoIPDirectives(directives []string, s *strings.Buil return nil } -func (dm *DNSMessage) handlePdnsDirectives(directives []string, s *strings.Builder) error { +func (dm *DNSMessage) handlePdnsDirectives(directive string, s *strings.Builder) error { if dm.PowerDNS == nil { s.WriteString("-") } else { + var directives []string + if i := strings.IndexByte(directive, ':'); i == -1 { + directives = append(directives, directive) + } else { + directives = []string{directive[:i], directive[i+1:]} + } + switch directive := directives[0]; { case directive == "powerdns-tags": if dm.PowerDNS.Tags == nil { @@ -425,11 +431,11 @@ func (dm *DNSMessage) handlePdnsDirectives(directives []string, s *strings.Build return nil } -func (dm *DNSMessage) handleSuspiciousDirectives(directives []string, s *strings.Builder) error { +func (dm *DNSMessage) handleSuspiciousDirectives(directive string, s *strings.Builder) error { if dm.Suspicious == nil { s.WriteString("-") } else { - switch directive := directives[0]; { + switch { case directive == "suspicious-score": s.WriteString(strconv.Itoa(int(dm.Suspicious.Score))) default: @@ -439,11 +445,11 @@ func (dm *DNSMessage) handleSuspiciousDirectives(directives []string, s *strings return nil } -func (dm *DNSMessage) handlePublicSuffixDirectives(directives []string, s *strings.Builder) error { +func (dm *DNSMessage) handlePublicSuffixDirectives(directive string, s *strings.Builder) error { if dm.PublicSuffix == nil { s.WriteString("-") } else { - switch directive := directives[0]; { + switch { case directive == "publixsuffix-tld": s.WriteString(dm.PublicSuffix.QnamePublicSuffix) case directive == "publixsuffix-etld+1": @@ -455,12 +461,12 @@ func (dm *DNSMessage) handlePublicSuffixDirectives(directives []string, s *strin return nil } -func (dm *DNSMessage) handleExtractedDirectives(directives []string, s *strings.Builder) error { +func (dm *DNSMessage) handleExtractedDirectives(directive string, s *strings.Builder) error { if dm.Extracted == nil { s.WriteString("-") return nil } - switch directive := directives[0]; { + switch { case directive == "extracted-dns-payload": if len(dm.DNS.Payload) > 0 { dst := make([]byte, base64.StdEncoding.EncodedLen(len(dm.DNS.Payload))) @@ -475,11 +481,11 @@ func (dm *DNSMessage) handleExtractedDirectives(directives []string, s *strings. return nil } -func (dm *DNSMessage) handleFilteringDirectives(directives []string, s *strings.Builder) error { +func (dm *DNSMessage) handleFilteringDirectives(directive string, s *strings.Builder) error { if dm.Filtering == nil { s.WriteString("-") } else { - switch directive := directives[0]; { + switch { case directive == "filtering-sample-rate": s.WriteString(strconv.Itoa(dm.Filtering.SampleRate)) default: @@ -489,11 +495,11 @@ func (dm *DNSMessage) handleFilteringDirectives(directives []string, s *strings. return nil } -func (dm *DNSMessage) handleReducerDirectives(directives []string, s *strings.Builder) error { +func (dm *DNSMessage) handleReducerDirectives(directive string, s *strings.Builder) error { if dm.Reducer == nil { s.WriteString("-") } else { - switch directive := directives[0]; { + switch { case directive == "reducer-occurrences": s.WriteString(strconv.Itoa(dm.Reducer.Occurrences)) case directive == "reducer-cumulative-length": @@ -505,11 +511,11 @@ func (dm *DNSMessage) handleReducerDirectives(directives []string, s *strings.Bu return nil } -func (dm *DNSMessage) handleMachineLearningDirectives(directives []string, s *strings.Builder) error { +func (dm *DNSMessage) handleMachineLearningDirectives(directive string, s *strings.Builder) error { if dm.MachineLearning == nil { s.WriteString("-") } else { - switch directive := directives[0]; { + switch { case directive == "ml-entropy": s.WriteString(strconv.FormatFloat(dm.MachineLearning.Entropy, 'f', -1, 64)) case directive == "ml-length": @@ -570,37 +576,12 @@ func (dm *DNSMessage) String(format []string, fieldDelimiter string, fieldBounda func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBoundary string) ([]byte, error) { var s strings.Builder - for i, word := range format { - directives := strings.SplitN(word, ":", 2) - switch directive := directives[0]; { - // default directives - case directive == "ttl": - if len(dm.DNS.DNSRRs.Answers) > 0 { - s.WriteString(strconv.Itoa(dm.DNS.DNSRRs.Answers[0].TTL)) - } else { - s.WriteByte('-') - } - case directive == "answer": - if len(dm.DNS.DNSRRs.Answers) > 0 { - s.WriteString(dm.DNS.DNSRRs.Answers[0].Rdata) - } else { - s.WriteByte('-') - } - case directive == "edns-csubnet": - if len(dm.EDNS.Options) > 0 { - for _, opt := range dm.EDNS.Options { - if opt.Name == "CSUBNET" { - s.WriteString(opt.Data) - break - } - } - } else { - s.WriteByte('-') - } - case directive == "answercount": - s.WriteString(strconv.Itoa(len(dm.DNS.DNSRRs.Answers))) - case directive == "id": - s.WriteString(strconv.Itoa(dm.DNS.ID)) + answers := dm.DNS.DNSRRs.Answers + qname := dm.DNS.Qname + flags := dm.DNS.Flags + + for i, directive := range format { + switch { case directive == "timestamp-rfc3339ns", directive == "timestamp": s.WriteString(dm.DNSTap.TimestampRFC3339) case directive == "timestamp-unixms": @@ -612,6 +593,20 @@ func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBo case directive == "localtime": ts := time.Unix(int64(dm.DNSTap.TimeSec), int64(dm.DNSTap.TimeNsec)) s.WriteString(ts.Format("2006-01-02 15:04:05.999999999")) + case directive == "qname": + if len(qname) == 0 { + s.WriteString(".") + } else { + if strings.Contains(qname, fieldDelimiter) { + qnameEscaped := qname + if strings.Contains(qname, fieldBoundary) { + qnameEscaped = strings.ReplaceAll(qnameEscaped, fieldBoundary, "\\"+fieldBoundary) + } + s.WriteString(fmt.Sprintf(fieldBoundary+"%s"+fieldBoundary, qnameEscaped)) + } else { + s.WriteString(qname) + } + } case directive == "identity": s.WriteString(dm.DNSTap.Identity) case directive == "peer-name": @@ -636,6 +631,9 @@ func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBo s.WriteString(dm.DNSTap.Operation) case directive == "rcode": s.WriteString(dm.DNS.Rcode) + + case directive == "id": + s.WriteString(strconv.Itoa(dm.DNS.ID)) case directive == "queryip": s.WriteString(dm.NetworkInfo.QueryIP) case directive == "queryport": @@ -652,20 +650,6 @@ func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBo s.WriteString(strconv.Itoa(dm.DNS.Length) + "b") case directive == "length": s.WriteString(strconv.Itoa(dm.DNS.Length)) - case directive == "qname": - if len(dm.DNS.Qname) == 0 { - s.WriteString(".") - } else { - if strings.Contains(dm.DNS.Qname, fieldDelimiter) { - qname := dm.DNS.Qname - if strings.Contains(qname, fieldBoundary) { - qname = strings.ReplaceAll(qname, fieldBoundary, "\\"+fieldBoundary) - } - s.WriteString(fmt.Sprintf(fieldBoundary+"%s"+fieldBoundary, qname)) - } else { - s.WriteString(dm.DNS.Qname) - } - } case directive == "qtype": s.WriteString(dm.DNS.Qtype) case directive == "latency": @@ -693,83 +677,109 @@ func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBo s.WriteByte('-') } case directive == "tc": - if dm.DNS.Flags.TC { + if flags.TC { s.WriteString("TC") } else { s.WriteByte('-') } case directive == "aa": - if dm.DNS.Flags.AA { + if flags.AA { s.WriteString("AA") } else { s.WriteByte('-') } case directive == "ra": - if dm.DNS.Flags.RA { + if flags.RA { s.WriteString("RA") } else { s.WriteByte('-') } case directive == "ad": - if dm.DNS.Flags.AD { + if flags.AD { s.WriteString("AD") } else { s.WriteByte('-') } + case directive == "ttl": + if len(answers) > 0 { + s.WriteString(strconv.Itoa(answers[0].TTL)) + } else { + s.WriteByte('-') + } + case directive == "answer": + if len(answers) > 0 { + s.WriteString(answers[0].Rdata) + } else { + s.WriteByte('-') + } + case directive == "answercount": + s.WriteString(strconv.Itoa(len(answers))) + + case directive == "edns-csubnet": + if len(dm.EDNS.Options) > 0 { + for _, opt := range dm.EDNS.Options { + if opt.Name == "CSUBNET" { + s.WriteString(opt.Data) + break + } + } + } else { + s.WriteByte('-') + } + // more directives from collectors case PdnsDirectives.MatchString(directive): - err := dm.handlePdnsDirectives(directives, &s) + err := dm.handlePdnsDirectives(directive, &s) if err != nil { return nil, err } // more directives from transformers case ReducerDirectives.MatchString(directive): - err := dm.handleReducerDirectives(directives, &s) + err := dm.handleReducerDirectives(directive, &s) if err != nil { return nil, err } case GeoIPDirectives.MatchString(directive): - err := dm.handleGeoIPDirectives(directives, &s) + err := dm.handleGeoIPDirectives(directive, &s) if err != nil { return nil, err } case SuspiciousDirectives.MatchString(directive): - err := dm.handleSuspiciousDirectives(directives, &s) + err := dm.handleSuspiciousDirectives(directive, &s) if err != nil { return nil, err } case PublicSuffixDirectives.MatchString(directive): - err := dm.handlePublicSuffixDirectives(directives, &s) + err := dm.handlePublicSuffixDirectives(directive, &s) if err != nil { return nil, err } case ExtractedDirectives.MatchString(directive): - err := dm.handleExtractedDirectives(directives, &s) + err := dm.handleExtractedDirectives(directive, &s) if err != nil { return nil, err } case MachineLearningDirectives.MatchString(directive): - err := dm.handleMachineLearningDirectives(directives, &s) + err := dm.handleMachineLearningDirectives(directive, &s) if err != nil { return nil, err } case FilteringDirectives.MatchString(directive): - err := dm.handleFilteringDirectives(directives, &s) + err := dm.handleFilteringDirectives(directive, &s) if err != nil { return nil, err } - // error unsupport directive for text format + // handle invalid directive default: - return nil, errors.New(ErrorUnexpectedDirective + word) + return nil, errors.New(ErrorUnexpectedDirective + directive) } if i < len(format)-1 { s.WriteString(fieldDelimiter) } } - return []byte(s.String()), nil } @@ -1036,15 +1046,193 @@ func (dm *DNSMessage) ToPacketLayer() ([]gopacket.SerializableLayer, error) { return pkt, nil } -func (dm *DNSMessage) Flatten() (ret map[string]interface{}, err error) { - // TODO perhaps panic when flattening fails, as it should always work. - var tmp []byte - if tmp, err = json.Marshal(dm); err != nil { - return +func (dm *DNSMessage) Flatten() (map[string]interface{}, error) { + dnsFields := map[string]interface{}{ + "dns.flags.aa": dm.DNS.Flags.AA, + "dns.flags.ad": dm.DNS.Flags.AD, + "dns.flags.qr": dm.DNS.Flags.QR, + "dns.flags.ra": dm.DNS.Flags.RA, + "dns.flags.tc": dm.DNS.Flags.TC, + "dns.flags.rd": dm.DNS.Flags.RD, + "dns.flags.cd": dm.DNS.Flags.CD, + "dns.length": dm.DNS.Length, + "dns.malformed-packet": dm.DNS.MalformedPacket, + "dns.id": dm.DNS.ID, + "dns.opcode": dm.DNS.Opcode, + "dns.qname": dm.DNS.Qname, + "dns.qtype": dm.DNS.Qtype, + "dns.rcode": dm.DNS.Rcode, + "dnstap.identity": dm.DNSTap.Identity, + "dnstap.latency": dm.DNSTap.LatencySec, + "dnstap.operation": dm.DNSTap.Operation, + "dnstap.timestamp-rfc3339ns": dm.DNSTap.TimestampRFC3339, + "dnstap.version": dm.DNSTap.Version, + "dnstap.extra": dm.DNSTap.Extra, + "dnstap.policy-rule": dm.DNSTap.PolicyRule, + "dnstap.policy-type": dm.DNSTap.PolicyType, + "dnstap.policy-action": dm.DNSTap.PolicyAction, + "dnstap.policy-match": dm.DNSTap.PolicyMatch, + "dnstap.policy-value": dm.DNSTap.PolicyValue, + "dnstap.peer-name": dm.DNSTap.PeerName, + "dnstap.query-zone": dm.DNSTap.QueryZone, + "edns.dnssec-ok": dm.EDNS.Do, + "edns.rcode": dm.EDNS.ExtendedRcode, + "edns.udp-size": dm.EDNS.UDPSize, + "edns.version": dm.EDNS.Version, + "network.family": dm.NetworkInfo.Family, + "network.ip-defragmented": dm.NetworkInfo.IPDefragmented, + "network.protocol": dm.NetworkInfo.Protocol, + "network.query-ip": dm.NetworkInfo.QueryIP, + "network.query-port": dm.NetworkInfo.QueryPort, + "network.response-ip": dm.NetworkInfo.ResponseIP, + "network.response-port": dm.NetworkInfo.ResponsePort, + "network.tcp-reassembled": dm.NetworkInfo.TCPReassembled, } - json.Unmarshal(tmp, &ret) - ret, err = flat.Flatten(ret, nil) - return + + // Add empty slices + if len(dm.DNS.DNSRRs.Answers) == 0 { + dnsFields["dns.resource-records.an"] = "-" + } + if len(dm.DNS.DNSRRs.Records) == 0 { + dnsFields["dns.resource-records.ar"] = "-" + } + if len(dm.DNS.DNSRRs.Nameservers) == 0 { + dnsFields["dns.resource-records.ns"] = "-" + } + if len(dm.EDNS.Options) == 0 { + dnsFields["edns.options"] = "-" + } + + // Add DNSAnswer fields: "dns.resource-records.an.0.name": "google.nl" + // nolint: goconst + for i, an := range dm.DNS.DNSRRs.Answers { + prefixAn := "dns.resource-records.an." + strconv.Itoa(i) + dnsFields[prefixAn+".name"] = an.Name + dnsFields[prefixAn+".rdata"] = an.Rdata + dnsFields[prefixAn+".rdatatype"] = an.Rdatatype + dnsFields[prefixAn+".ttl"] = an.TTL + } + for i, ns := range dm.DNS.DNSRRs.Nameservers { + prefixNs := "dns.resource-records.ns." + strconv.Itoa(i) + dnsFields[prefixNs+".name"] = ns.Name + dnsFields[prefixNs+".rdata"] = ns.Rdata + dnsFields[prefixNs+".rdatatype"] = ns.Rdatatype + dnsFields[prefixNs+".ttl"] = ns.TTL + } + for i, ar := range dm.DNS.DNSRRs.Records { + prefixAr := "dns.resource-records.ar." + strconv.Itoa(i) + dnsFields[prefixAr+".name"] = ar.Name + dnsFields[prefixAr+".rdata"] = ar.Rdata + dnsFields[prefixAr+".rdatatype"] = ar.Rdatatype + dnsFields[prefixAr+".ttl"] = ar.TTL + } + + // Add EDNSoptions fields: "edns.options.0.code": 10, + for i, opt := range dm.EDNS.Options { + prefixOpt := "edns.options." + strconv.Itoa(i) + dnsFields[prefixOpt+".code"] = opt.Code + dnsFields[prefixOpt+".data"] = opt.Data + dnsFields[prefixOpt+".name"] = opt.Name + } + + // Add TransformDNSGeo fields + if dm.Geo != nil { + dnsFields["geoip.city"] = dm.Geo.City + dnsFields["geoip.continent"] = dm.Geo.Continent + dnsFields["geoip.country-isocode"] = dm.Geo.CountryIsoCode + dnsFields["geoip.as-number"] = dm.Geo.AutonomousSystemNumber + dnsFields["geoip.as-owner"] = dm.Geo.AutonomousSystemOrg + } + + // Add TransformSuspicious fields + if dm.Suspicious != nil { + dnsFields["suspicious.score"] = dm.Suspicious.Score + dnsFields["suspicious.malformed-pkt"] = dm.Suspicious.MalformedPacket + dnsFields["suspicious.large-pkt"] = dm.Suspicious.LargePacket + dnsFields["suspicious.long-domain"] = dm.Suspicious.LongDomain + dnsFields["suspicious.slow-domain"] = dm.Suspicious.SlowDomain + dnsFields["suspicious.unallowed-chars"] = dm.Suspicious.UnallowedChars + dnsFields["suspicious.uncommon-qtypes"] = dm.Suspicious.UncommonQtypes + dnsFields["suspicious.excessive-number-labels"] = dm.Suspicious.ExcessiveNumberLabels + dnsFields["suspicious.domain"] = dm.Suspicious.Domain + } + + // Add TransformPublicSuffix fields + if dm.PublicSuffix != nil { + dnsFields["publicsuffix.tld"] = dm.PublicSuffix.QnamePublicSuffix + dnsFields["publicsuffix.etld+1"] = dm.PublicSuffix.QnameEffectiveTLDPlusOne + } + + // Add TransformExtracted fields + if dm.Extracted != nil { + dnsFields["extracted.dns_payload"] = dm.Extracted.Base64Payload + } + + // Add TransformReducer fields + if dm.Reducer != nil { + dnsFields["reducer.occurrences"] = dm.Reducer.Occurrences + dnsFields["reducer.cumulative-length"] = dm.Reducer.CumulativeLength + } + + // Add TransformFiltering fields + if dm.Filtering != nil { + dnsFields["filtering.sample-rate"] = dm.Filtering.SampleRate + } + + // Add TransformML fields + if dm.MachineLearning != nil { + dnsFields["ml.entropy"] = dm.MachineLearning.Entropy + dnsFields["ml.length"] = dm.MachineLearning.Length + dnsFields["ml.labels"] = dm.MachineLearning.Labels + dnsFields["ml.digits"] = dm.MachineLearning.Digits + dnsFields["ml.lowers"] = dm.MachineLearning.Lowers + dnsFields["ml.uppers"] = dm.MachineLearning.Uppers + dnsFields["ml.specials"] = dm.MachineLearning.Specials + dnsFields["ml.others"] = dm.MachineLearning.Others + dnsFields["ml.ratio-digits"] = dm.MachineLearning.RatioDigits + dnsFields["ml.ratio-letters"] = dm.MachineLearning.RatioLetters + dnsFields["ml.ratio-specials"] = dm.MachineLearning.RatioSpecials + dnsFields["ml.ratio-others"] = dm.MachineLearning.RatioOthers + dnsFields["ml.consecutive-chars"] = dm.MachineLearning.ConsecutiveChars + dnsFields["ml.consecutive-vowels"] = dm.MachineLearning.ConsecutiveVowels + dnsFields["ml.consecutive-digits"] = dm.MachineLearning.ConsecutiveDigits + dnsFields["ml.consecutive-consonants"] = dm.MachineLearning.ConsecutiveConsonants + dnsFields["ml.size"] = dm.MachineLearning.Size + dnsFields["ml.occurrences"] = dm.MachineLearning.Occurrences + dnsFields["ml.uncommon-qtypes"] = dm.MachineLearning.UncommonQtypes + } + + // Add TransformATags fields + if dm.ATags != nil { + if len(dm.ATags.Tags) == 0 { + dnsFields["atags.tags"] = "-" + } + for i, tag := range dm.ATags.Tags { + dnsFields["atags.tags."+strconv.Itoa(i)] = tag + } + } + + // Add PowerDNS collectors fields + if dm.PowerDNS != nil { + if len(dm.PowerDNS.Tags) == 0 { + dnsFields["powerdns.tags"] = "-" + } + for i, tag := range dm.PowerDNS.Tags { + dnsFields["powerdns.tags."+strconv.Itoa(i)] = tag + } + dnsFields["powerdns.original-request-subnet"] = dm.PowerDNS.OriginalRequestSubnet + dnsFields["powerdns.applied-policy"] = dm.PowerDNS.AppliedPolicy + dnsFields["powerdns.applied-policy-hit"] = dm.PowerDNS.AppliedPolicyHit + dnsFields["powerdns.applied-policy-kind"] = dm.PowerDNS.AppliedPolicyKind + dnsFields["powerdns.applied-policy-trigger"] = dm.PowerDNS.AppliedPolicyTrigger + dnsFields["powerdns.applied-policy-type"] = dm.PowerDNS.AppliedPolicyType + for mk, mv := range dm.PowerDNS.Metadata { + dnsFields["powerdns.metadata."+mk] = mv + } + dnsFields["powerdns.http-version"] = dm.PowerDNS.HTTPVersion + } + + return dnsFields, nil } func (dm *DNSMessage) Matching(matching map[string]interface{}) (error, bool) { diff --git a/dnsutils/message_test.go b/dnsutils/message_test.go index b9d22bb7..778e8ec9 100644 --- a/dnsutils/message_test.go +++ b/dnsutils/message_test.go @@ -6,8 +6,10 @@ import ( "strings" "testing" + "github.com/dmachard/go-dnscollector/netlib" "github.com/dmachard/go-dnscollector/pkgconfig" "github.com/dmachard/go-dnstap-protobuf" + "github.com/miekg/dns" "google.golang.org/protobuf/proto" ) @@ -35,6 +37,47 @@ func encodeToDNSTap(dm DNSMessage, t *testing.T) *ExtendedDnstap { return edt } +func TestDnsMessage_ToDNSTap(t *testing.T) { + dm := GetFakeDNSMessageWithPayload() + dm.DNSTap.Extra = "extra:value" + + // encode to dnstap + tapMsg, err := dm.ToDNSTap(false) + if err != nil { + t.Fatalf("could not encode to dnstap: %v\n", err) + } + + // decode dnstap message + dt := &dnstap.Dnstap{} + err = proto.Unmarshal(tapMsg, dt) + if err != nil { + t.Fatalf("error to decode dnstap: %v", err) + } + + if string(dt.GetIdentity()) != dm.DNSTap.Identity { + t.Errorf("identify field should be equal got=%s", string(dt.GetIdentity())) + } + + if string(dt.GetExtra()) != dm.DNSTap.Extra { + t.Errorf("extra field should be equal got=%s", string(dt.GetExtra())) + } +} + +func BenchmarkDnsMessage_ToDNSTap(b *testing.B) { + dm := DNSMessage{} + dm.Init() + dm.InitTransforms() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := dm.ToDNSTap(false) + if err != nil { + b.Fatalf("could not encode to dnstap: %v\n", err) + } + } +} + +// Tests for Extended DNSTap format func TestDnsMessage_ToExtendedDNSTap_GetOriginalDnstapExtra(t *testing.T) { dm := GetFakeDNSMessageWithPayload() dm.DNSTap.Extra = "tag0:value0" @@ -120,29 +163,17 @@ func TestDnsMessage_ToExtendedDNSTap_TransformGeo(t *testing.T) { } } -func TestDnsMessage_ToDNSTap(t *testing.T) { - dm := GetFakeDNSMessageWithPayload() - dm.DNSTap.Extra = "extra:value" - - // encode to dnstap - tapMsg, err := dm.ToDNSTap(false) - if err != nil { - t.Fatalf("could not encode to dnstap: %v\n", err) - } - - // decode dnstap message - dt := &dnstap.Dnstap{} - err = proto.Unmarshal(tapMsg, dt) - if err != nil { - t.Fatalf("error to decode dnstap: %v", err) - } - - if string(dt.GetIdentity()) != dm.DNSTap.Identity { - t.Errorf("identify field should be equal got=%s", string(dt.GetIdentity())) - } - - if string(dt.GetExtra()) != dm.DNSTap.Extra { - t.Errorf("extra field should be equal got=%s", string(dt.GetExtra())) +func BenchmarkDnsMessage_ToExtendedDNSTap(b *testing.B) { + dm := DNSMessage{} + dm.Init() + dm.InitTransforms() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := dm.ToDNSTap(true) + if err != nil { + b.Fatalf("could not encode to extended dnstap: %v\n", err) + } } } @@ -228,74 +259,6 @@ func TestDnsMessage_Json_Reference(t *testing.T) { } } -func TestDnsMessage_JsonFlatten_Reference(t *testing.T) { - dm := DNSMessage{} - dm.Init() - - refJSON := ` - { - "dns.flags.aa": false, - "dns.flags.ad": false, - "dns.flags.qr": false, - "dns.flags.ra": false, - "dns.flags.tc": false, - "dns.flags.rd": false, - "dns.flags.cd": false, - "dns.length": 0, - "dns.malformed-packet": false, - "dns.id": 0, - "dns.opcode": 0, - "dns.qname": "-", - "dns.qtype": "-", - "dns.rcode": "-", - "dns.resource-records.an": [], - "dns.resource-records.ar": [], - "dns.resource-records.ns": [], - "dnstap.identity": "-", - "dnstap.latency": "-", - "dnstap.operation": "-", - "dnstap.timestamp-rfc3339ns": "-", - "dnstap.version": "-", - "dnstap.extra": "-", - "dnstap.policy-rule": "-", - "dnstap.policy-type": "-", - "dnstap.policy-action": "-", - "dnstap.policy-match": "-", - "dnstap.policy-value": "-", - "dnstap.peer-name": "-", - "dnstap.query-zone": "-", - "edns.dnssec-ok": 0, - "edns.options": [], - "edns.rcode": 0, - "edns.udp-size": 0, - "edns.version": 0, - "network.family": "-", - "network.ip-defragmented": false, - "network.protocol": "-", - "network.query-ip": "-", - "network.query-port": "-", - "network.response-ip": "-", - "network.response-port": "-", - "network.tcp-reassembled": false - } - ` - - dmFlat, err := dm.Flatten() - if err != nil { - t.Fatalf("could not flat json: %s\n", err) - } - - var refMap map[string]interface{} - err = json.Unmarshal([]byte(refJSON), &refMap) - if err != nil { - t.Fatalf("could not unmarshal ref json: %s\n", err) - } - - if !reflect.DeepEqual(dmFlat, refMap) { - t.Errorf("flatten json format different from reference") - } -} - func TestDnsMessage_Json_Collectors_Reference(t *testing.T) { testcases := []struct { collector string @@ -445,6 +408,368 @@ func TestDnsMessage_Json_Transforms_Reference(t *testing.T) { } } +func BenchmarkDnsMessage_ToJSON(b *testing.B) { + dm := DNSMessage{} + dm.Init() + dm.InitTransforms() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + dm.ToJSON() + } +} + +// Tests for Flat JSON format +func TestDnsMessage_JsonFlatten_Reference(t *testing.T) { + dm := DNSMessage{} + dm.Init() + + // add some items in slices field + dm.DNS.DNSRRs.Answers = append(dm.DNS.DNSRRs.Answers, DNSAnswer{Name: "google.nl", Rdata: "142.251.39.99", Rdatatype: "A", TTL: 300}) + dm.EDNS.Options = append(dm.EDNS.Options, DNSOption{Code: 10, Data: "aaaabbbbcccc", Name: "COOKIE"}) + + refJSON := ` + { + "dns.flags.aa": false, + "dns.flags.ad": false, + "dns.flags.qr": false, + "dns.flags.ra": false, + "dns.flags.tc": false, + "dns.flags.rd": false, + "dns.flags.cd": false, + "dns.length": 0, + "dns.malformed-packet": false, + "dns.id": 0, + "dns.opcode": 0, + "dns.qname": "-", + "dns.qtype": "-", + "dns.rcode": "-", + "dns.resource-records.an.0.name": "google.nl", + "dns.resource-records.an.0.rdata": "142.251.39.99", + "dns.resource-records.an.0.rdatatype": "A", + "dns.resource-records.an.0.ttl": 300, + "dns.resource-records.ar": "-", + "dns.resource-records.ns": "-", + "dnstap.identity": "-", + "dnstap.latency": "-", + "dnstap.operation": "-", + "dnstap.timestamp-rfc3339ns": "-", + "dnstap.version": "-", + "dnstap.extra": "-", + "dnstap.policy-rule": "-", + "dnstap.policy-type": "-", + "dnstap.policy-action": "-", + "dnstap.policy-match": "-", + "dnstap.policy-value": "-", + "dnstap.peer-name": "-", + "dnstap.query-zone": "-", + "edns.dnssec-ok": 0, + "edns.options.0.code": 10, + "edns.options.0.data": "aaaabbbbcccc", + "edns.options.0.name": "COOKIE", + "edns.rcode": 0, + "edns.udp-size": 0, + "edns.version": 0, + "network.family": "-", + "network.ip-defragmented": false, + "network.protocol": "-", + "network.query-ip": "-", + "network.query-port": "-", + "network.response-ip": "-", + "network.response-port": "-", + "network.tcp-reassembled": false + } + ` + + var dmFlat map[string]interface{} + dmJSON, err := dm.ToFlatJSON() + if err != nil { + t.Fatalf("could not convert dm to flat json: %s\n", err) + } + err = json.Unmarshal([]byte(dmJSON), &dmFlat) + if err != nil { + t.Fatalf("could not unmarshal dm json: %s\n", err) + } + + var refMap map[string]interface{} + err = json.Unmarshal([]byte(refJSON), &refMap) + if err != nil { + t.Fatalf("could not unmarshal ref json: %s\n", err) + } + + for k, vRef := range refMap { + vFlat, ok := dmFlat[k] + if !ok { + t.Fatalf("Missing key %s in flatten message according to reference", k) + } + if vRef != vFlat { + t.Errorf("Invalid value for key=%s get=%v expected=%v", k, vFlat, vRef) + } + } + + for k := range dmFlat { + _, ok := refMap[k] + if !ok { + t.Errorf("This key %s should not be in the flat message", k) + } + } +} + +func TestDnsMessage_JsonFlatten_Transforms_Reference(t *testing.T) { + + testcases := []struct { + transform string + dm DNSMessage + jsonRef string + }{ + { + transform: "filtering", + dm: DNSMessage{Filtering: &TransformFiltering{SampleRate: 22}}, + jsonRef: `{ + "filtering.sample-rate": 22 + }`, + }, + { + transform: "reducer", + dm: DNSMessage{Reducer: &TransformReducer{Occurrences: 10, CumulativeLength: 47}}, + jsonRef: `{ + "reducer.occurrences": 10, + "reducer.cumulative-length": 47 + }`, + }, + { + transform: "publixsuffix", + dm: DNSMessage{ + PublicSuffix: &TransformPublicSuffix{ + QnamePublicSuffix: "com", + QnameEffectiveTLDPlusOne: "hello.com", + }, + }, + jsonRef: `{ + "publicsuffix.tld": "com", + "publicsuffix.etld+1": "hello.com" + }`, + }, + { + transform: "geoip", + dm: DNSMessage{ + Geo: &TransformDNSGeo{ + City: "Paris", + Continent: "Europe", + CountryIsoCode: "FR", + AutonomousSystemNumber: "1234", + AutonomousSystemOrg: "Internet", + }, + }, + jsonRef: `{ + "geoip.city": "Paris", + "geoip.continent": "Europe", + "geoip.country-isocode": "FR", + "geoip.as-number": "1234", + "geoip.as-owner": "Internet" + }`, + }, + { + transform: "suspicious", + dm: DNSMessage{Suspicious: &TransformSuspicious{Score: 1.0, + MalformedPacket: false, + LargePacket: true, + LongDomain: true, + SlowDomain: false, + UnallowedChars: true, + UncommonQtypes: false, + ExcessiveNumberLabels: true, + Domain: "gogle.co", + }}, + jsonRef: `{ + "suspicious.score": 1.0, + "suspicious.malformed-pkt": false, + "suspicious.large-pkt": true, + "suspicious.long-domain": true, + "suspicious.slow-domain": false, + "suspicious.unallowed-chars": true, + "suspicious.uncommon-qtypes": false, + "suspicious.excessive-number-labels": true, + "suspicious.domain": "gogle.co" + }`, + }, + { + transform: "extracted", + dm: DNSMessage{Extracted: &TransformExtracted{Base64Payload: []byte{}}}, + jsonRef: `{ + "extracted.dns_payload": "" + }`, + }, + { + transform: "machinelearning", + dm: DNSMessage{MachineLearning: &TransformML{ + Entropy: 10.0, + Length: 2, + Labels: 2, + Digits: 1, + Lowers: 35, + Uppers: 23, + Specials: 2, + Others: 1, + RatioDigits: 1.0, + RatioLetters: 1.0, + RatioSpecials: 1.0, + RatioOthers: 1.0, + ConsecutiveChars: 10, + ConsecutiveVowels: 10, + ConsecutiveDigits: 10, + ConsecutiveConsonants: 10, + Size: 11, + Occurrences: 10, + UncommonQtypes: 1, + }}, + jsonRef: `{ + "ml.entropy": 10.0, + "ml.length": 2, + "ml.labels": 2, + "ml.digits": 1, + "ml.lowers": 35, + "ml.uppers": 23, + "ml.specials": 2, + "ml.others": 1, + "ml.ratio-digits": 1.0, + "ml.ratio-letters": 1.0, + "ml.ratio-specials": 1.0, + "ml.ratio-others": 1.0, + "ml.consecutive-chars": 10, + "ml.consecutive-vowels": 10, + "ml.consecutive-digits": 10, + "ml.consecutive-consonants": 10, + "ml.size": 11, + "ml.occurrences": 10, + "ml.uncommon-qtypes": 1 + }`, + }, + { + transform: "atags", + dm: DNSMessage{ATags: &TransformATags{Tags: []string{"test0", "test1"}}}, + jsonRef: `{ + "atags.tags.0": "test0", + "atags.tags.1": "test1" + }`, + }, + } + + for _, tc := range testcases { + t.Run(tc.transform, func(t *testing.T) { + + tc.dm.Init() + + var dmFlat map[string]interface{} + dmJSON, err := tc.dm.ToFlatJSON() + if err != nil { + t.Fatalf("could not convert dm to flat json: %s\n", err) + } + err = json.Unmarshal([]byte(dmJSON), &dmFlat) + if err != nil { + t.Fatalf("could not unmarshal dm json: %s\n", err) + } + + var refMap map[string]interface{} + err = json.Unmarshal([]byte(tc.jsonRef), &refMap) + if err != nil { + t.Fatalf("could not unmarshal ref json: %s\n", err) + } + + for k, vRef := range refMap { + vFlat, ok := dmFlat[k] + if !ok { + t.Fatalf("Missing key %s in flatten message according to reference", k) + } + if vRef != vFlat { + t.Errorf("Invalid value for key=%s get=%v expected=%v", k, vFlat, vRef) + } + } + }) + } +} + +func TestDnsMessage_JsonFlatten_Collectors_Reference(t *testing.T) { + testcases := []struct { + collector string + dm DNSMessage + jsonRef string + }{ + { + collector: "powerdns", + dm: DNSMessage{PowerDNS: &PowerDNS{ + OriginalRequestSubnet: "subnet", + AppliedPolicy: "basicrpz", + AppliedPolicyHit: "hit", + AppliedPolicyKind: "kind", + AppliedPolicyTrigger: "trigger", + AppliedPolicyType: "type", + Tags: []string{"tag1"}, + Metadata: map[string]string{"stream_id": "collector"}, + HTTPVersion: "http3", + }}, + + jsonRef: `{ + "powerdns.original-request-subnet": "subnet", + "powerdns.applied-policy": "basicrpz", + "powerdns.applied-policy-hit": "hit", + "powerdns.applied-policy-kind": "kind", + "powerdns.applied-policy-trigger": "trigger", + "powerdns.applied-policy-type": "type", + "powerdns.tags.0": "tag1", + "powerdns.metadata.stream_id": "collector", + "powerdns.http-version": "http3" + }`, + }, + } + for _, tc := range testcases { + t.Run(tc.collector, func(t *testing.T) { + + tc.dm.Init() + + var dmFlat map[string]interface{} + dmJSON, err := tc.dm.ToFlatJSON() + if err != nil { + t.Fatalf("could not convert dm to flat json: %s\n", err) + } + err = json.Unmarshal([]byte(dmJSON), &dmFlat) + if err != nil { + t.Fatalf("could not unmarshal dm json: %s\n", err) + } + + var refMap map[string]interface{} + err = json.Unmarshal([]byte(tc.jsonRef), &refMap) + if err != nil { + t.Fatalf("could not unmarshal ref json: %s\n", err) + } + + for k, vRef := range refMap { + vFlat, ok := dmFlat[k] + if !ok { + t.Fatalf("Missing key %s in flatten message according to reference", k) + } + if vRef != vFlat { + t.Errorf("Invalid value for key=%s get=%v expected=%v", k, vFlat, vRef) + } + } + }) + } +} + +func BenchmarkDnsMessage_ToFlatJSON(b *testing.B) { + dm := DNSMessage{} + dm.Init() + dm.InitTransforms() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := dm.ToFlatJSON() + if err != nil { + b.Fatalf("could not encode to flat json: %v\n", err) + } + } +} + // Tests for TEXT format func TestDnsMessage_TextFormat_ToString(t *testing.T) { @@ -1016,3 +1341,59 @@ func TestDnsMessage_TextFormat_Directives_Filtering(t *testing.T) { }) } } + +func BenchmarkDnsMessage_ToTextFormat(b *testing.B) { + dm := DNSMessage{} + dm.Init() + dm.InitTransforms() + + textFormat := []string{"timestamp-rfc3339ns", "identity", + "operation", "rcode", "queryip", "queryport", "family", + "protocol", "length-unit", "qname", "qtype", "latency"} + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := dm.ToTextLine(textFormat, " ", "\"") + if err != nil { + b.Fatalf("could not encode to text format: %v\n", err) + } + } +} + +// Tests for PCAP serialization +func BenchmarkDnsMessage_ToPacketLayer(b *testing.B) { + dm := DNSMessage{} + dm.Init() + dm.InitTransforms() + + dnsmsg := new(dns.Msg) + dnsmsg.SetQuestion("dnscollector.dev.", dns.TypeAAAA) + dnsquestion, _ := dnsmsg.Pack() + + dm.NetworkInfo.Family = netlib.ProtoIPv4 + dm.NetworkInfo.Protocol = netlib.ProtoUDP + dm.DNS.Payload = dnsquestion + dm.DNS.Length = len(dnsquestion) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := dm.ToPacketLayer() + if err != nil { + b.Fatalf("could not encode to pcap: %v\n", err) + } + } +} + +// Others tests +func BenchmarkDnsMessage_ToFlatten(b *testing.B) { + dm := DNSMessage{} + dm.Init() + dm.InitTransforms() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := dm.Flatten() + if err != nil { + b.Fatalf("could not flat: %v\n", err) + } + } +} diff --git a/docs/development.md b/docs/development.md index ce867c12..0b4389e1 100644 --- a/docs/development.md +++ b/docs/development.md @@ -63,6 +63,13 @@ Execute a test for one specific testcase in a package go test -timeout 10s -cover -v ./loggers -run Test_SyslogRun ``` +Run bench + +```bash +cd dnsutils/ +go test -run=^$ -bench=. +``` + ## Update Golang version and package dependencies Update package dependencies diff --git a/docs/dnsjson.md b/docs/dnsjson.md index 550a027a..e8f0193e 100644 --- a/docs/dnsjson.md +++ b/docs/dnsjson.md @@ -1,14 +1,14 @@ -# DNS-collector - DNS JSON encoding +# DNS-collector - JSON encoding -The dns collector enable to transform dns queries or replies in JSON format. -A JSON format contains dns message with additionnal metadata added by transformers or collectors. +The `DNS-collector` enables the transformation of DNS queries or replies into `JSON` format. +The JSON format contains DNS messages with additionnal metadata added by transformers or collectors. -Default JSON payload:: +Main default JSON payload parts: -- `network`: query/response ip and port, the protocol and family used -- `dnstap`: message type, arrival packet time, latency. -- `dns`: dns fields -- `edns`: extended dns options +- `network`: Query/response IP and port, the protocol, and family used. +- `dnstap`: Message type, arrival packet time, latency. +- `dns`: DNS fields. +- `edns`: Extended DNS options. Example: @@ -87,10 +87,13 @@ Example: } ``` -## Flat JSON export format +## Flat JSON format (recommended) -Sometimes, a single level key-value output in JSON is easier to ingest than multi-level JSON. -Using flat-json requires more processing on the host running go-dnscollector but delivers every output field as its own key/value pair. Here's a flat-json output as formatted by `jq`: +At times, a single level key-value output in JSON is easier to ingest than multi-level JSON structures. +Utilizing `flat-json` delivers every output field as its own key/value pair but requires more processing +on the host running DNS-collector. + +Here's a flat JSON output formatted using `jq`: ```json { @@ -112,8 +115,8 @@ Using flat-json requires more processing on the host running go-dnscollector but "dns.resource-records.an.0.rdata": "142.251.39.99", "dns.resource-records.an.0.rdatatype": "A", "dns.resource-records.an.0.ttl": 300, - "dns.resource-records.ar": [], - "dns.resource-records.ns": [], + "dns.resource-records.ar": "-", + "dns.resource-records.ns": "-", "dnstap.identity": "foo", "dnstap.peer-name": "172.16.0.2", "dnstap.latency": "0.000000",