diff --git a/dnsutils/dnsmessage.go b/dnsutils/dnsmessage.go index 4b72495f..95eced01 100644 --- a/dnsutils/dnsmessage.go +++ b/dnsutils/dnsmessage.go @@ -1,27 +1,7 @@ package dnsutils import ( - "bytes" - "encoding/base64" - "encoding/binary" - "encoding/json" - "errors" - "fmt" - "log" - "math" - "net" - "reflect" "regexp" - "strconv" - "strings" - "time" - - "github.com/dmachard/go-dnstap-protobuf" - "github.com/dmachard/go-netutils" - "github.com/flosch/pongo2/v6" - "github.com/google/gopacket" - "github.com/google/gopacket/layers" - "google.golang.org/protobuf/proto" ) var ( @@ -290,1151 +270,3 @@ func (dm *DNSMessage) InitTransforms() { dm.Geo = &TransformDNSGeo{} dm.Relabeling = &TransformRelabeling{} } - -func (dm *DNSMessage) handleGeoIPDirectives(directive string, s *strings.Builder) error { - if dm.Geo == nil { - s.WriteString("-") - } else { - switch { - case directive == "geoip-continent": - s.WriteString(dm.Geo.Continent) - case directive == "geoip-country": - s.WriteString(dm.Geo.CountryIsoCode) - case directive == "geoip-city": - s.WriteString(dm.Geo.City) - case directive == "geoip-as-number": - s.WriteString(dm.Geo.AutonomousSystemNumber) - case directive == "geoip-as-owner": - s.WriteString(dm.Geo.AutonomousSystemOrg) - default: - return errors.New(ErrorUnexpectedDirective + directive) - } - } - return nil -} - -func (dm *DNSMessage) handlePdnsDirectives(directive string, s *strings.Builder) error { - if dm.PowerDNS == nil { - s.WriteString("-") - } else { - var directives []string - if i := strings.IndexByte(directive, ':'); i == -1 { - directives = append(directives, directive) - } else { - directives = []string{directive[:i], directive[i+1:]} - } - - switch directive := directives[0]; { - case directive == "powerdns-tags": - if dm.PowerDNS.Tags == nil { - s.WriteString("-") - } else { - if len(dm.PowerDNS.Tags) > 0 { - if len(directives) == 2 { - tagIndex, err := strconv.Atoi(directives[1]) - if err != nil { - log.Fatalf("unsupport tag index provided (integer expected): %s", directives[1]) - } - if tagIndex >= len(dm.PowerDNS.Tags) { - s.WriteString("-") - } else { - s.WriteString(dm.PowerDNS.Tags[tagIndex]) - } - } else { - for i, tag := range dm.PowerDNS.Tags { - s.WriteString(tag) - // add separator - if i+1 < len(dm.PowerDNS.Tags) { - s.WriteString(",") - } - } - } - } else { - s.WriteString("-") - } - } - case directive == "powerdns-applied-policy": - if len(dm.PowerDNS.AppliedPolicy) > 0 { - s.WriteString(dm.PowerDNS.AppliedPolicy) - } else { - s.WriteString("-") - } - case directive == "powerdns-applied-policy-hit": - if len(dm.PowerDNS.AppliedPolicyHit) > 0 { - s.WriteString(dm.PowerDNS.AppliedPolicyHit) - } else { - s.WriteString("-") - } - case directive == "powerdns-applied-policy-kind": - if len(dm.PowerDNS.AppliedPolicyKind) > 0 { - s.WriteString(dm.PowerDNS.AppliedPolicyKind) - } else { - s.WriteString("-") - } - case directive == "powerdns-applied-policy-trigger": - if len(dm.PowerDNS.AppliedPolicyTrigger) > 0 { - s.WriteString(dm.PowerDNS.AppliedPolicyTrigger) - } else { - s.WriteString("-") - } - case directive == "powerdns-applied-policy-type": - if len(dm.PowerDNS.AppliedPolicyType) > 0 { - s.WriteString(dm.PowerDNS.AppliedPolicyType) - } else { - s.WriteString("-") - } - case directive == "powerdns-original-request-subnet": - if len(dm.PowerDNS.OriginalRequestSubnet) > 0 { - s.WriteString(dm.PowerDNS.OriginalRequestSubnet) - } else { - s.WriteString("-") - } - case directive == "powerdns-metadata": - if dm.PowerDNS.Metadata == nil { - s.WriteString("-") - } else { - if len(dm.PowerDNS.Metadata) > 0 && len(directives) == 2 { - if metaValue, ok := dm.PowerDNS.Metadata[directives[1]]; ok { - if len(metaValue) > 0 { - s.WriteString(strings.ReplaceAll(metaValue, " ", "_")) - } else { - s.WriteString("-") - } - } else { - s.WriteString("-") - } - } else { - s.WriteString("-") - } - } - case directive == "powerdns-http-version": - if len(dm.PowerDNS.HTTPVersion) > 0 { - s.WriteString(dm.PowerDNS.HTTPVersion) - } else { - s.WriteString("-") - } - default: - return errors.New(ErrorUnexpectedDirective + directive) - } - } - return nil -} - -func (dm *DNSMessage) handleATagsDirectives(directive string, s *strings.Builder) error { - if dm.ATags == nil { - s.WriteString("-") - } else { - var directives []string - if i := strings.IndexByte(directive, ':'); i == -1 { - directives = append(directives, directive) - } else { - directives = []string{directive[:i], directive[i+1:]} - } - - switch directive := directives[0]; { - case directive == "atags": - if len(dm.ATags.Tags) > 0 { - if len(directives) == 2 { - tagIndex, err := strconv.Atoi(directives[1]) - if err != nil { - log.Fatalf("unsupport tag index provided (integer expected): %s", directives[1]) - } - if tagIndex >= len(dm.ATags.Tags) { - s.WriteString("-") - } else { - s.WriteString(dm.ATags.Tags[tagIndex]) - } - } else { - for i, tag := range dm.ATags.Tags { - s.WriteString(tag) - // add separator - if i+1 < len(dm.ATags.Tags) { - s.WriteString(",") - } - } - } - } else { - s.WriteString("-") - } - default: - return errors.New(ErrorUnexpectedDirective + directive) - } - } - return nil -} - -func (dm *DNSMessage) handleSuspiciousDirectives(directive string, s *strings.Builder) error { - if dm.Suspicious == nil { - s.WriteString("-") - } else { - switch { - case directive == "suspicious-score": - s.WriteString(strconv.Itoa(int(dm.Suspicious.Score))) - default: - return errors.New(ErrorUnexpectedDirective + directive) - } - } - return nil -} - -func (dm *DNSMessage) handlePublicSuffixDirectives(directive string, s *strings.Builder) error { - if dm.PublicSuffix == nil { - s.WriteString("-") - } else { - switch { - case directive == "publixsuffix-tld": - s.WriteString(dm.PublicSuffix.QnamePublicSuffix) - case directive == "publixsuffix-etld+1": - s.WriteString(dm.PublicSuffix.QnameEffectiveTLDPlusOne) - case directive == "publixsuffix-managed-icann": - if dm.PublicSuffix.ManagedByICANN { - s.WriteString("managed") - } else { - s.WriteString("private") - } - default: - return errors.New(ErrorUnexpectedDirective + directive) - } - } - return nil -} - -func (dm *DNSMessage) handleExtractedDirectives(directive string, s *strings.Builder) error { - if dm.Extracted == nil { - s.WriteString("-") - return nil - } - switch { - case directive == "extracted-dns-payload": - if len(dm.DNS.Payload) > 0 { - dst := make([]byte, base64.StdEncoding.EncodedLen(len(dm.DNS.Payload))) - base64.StdEncoding.Encode(dst, dm.DNS.Payload) - s.Write(dst) - } else { - s.WriteString("-") - } - default: - return errors.New(ErrorUnexpectedDirective + directive) - } - return nil -} - -func (dm *DNSMessage) handleFilteringDirectives(directive string, s *strings.Builder) error { - if dm.Filtering == nil { - s.WriteString("-") - } else { - switch { - case directive == "filtering-sample-rate": - s.WriteString(strconv.Itoa(dm.Filtering.SampleRate)) - default: - return errors.New(ErrorUnexpectedDirective + directive) - } - } - return nil -} - -func (dm *DNSMessage) handleReducerDirectives(directive string, s *strings.Builder) error { - if dm.Reducer == nil { - s.WriteString("-") - } else { - switch { - case directive == "reducer-occurrences": - s.WriteString(strconv.Itoa(dm.Reducer.Occurrences)) - case directive == "reducer-cumulative-length": - s.WriteString(strconv.Itoa(dm.Reducer.CumulativeLength)) - default: - return errors.New(ErrorUnexpectedDirective + directive) - } - } - return nil -} - -func (dm *DNSMessage) handleMachineLearningDirectives(directive string, s *strings.Builder) error { - if dm.MachineLearning == nil { - s.WriteString("-") - } else { - switch { - case directive == "ml-entropy": - s.WriteString(strconv.FormatFloat(dm.MachineLearning.Entropy, 'f', -1, 64)) - case directive == "ml-length": - s.WriteString(strconv.Itoa(dm.MachineLearning.Length)) - case directive == "ml-digits": - s.WriteString(strconv.Itoa(dm.MachineLearning.Digits)) - case directive == "ml-lowers": - s.WriteString(strconv.Itoa(dm.MachineLearning.Lowers)) - case directive == "ml-uppers": - s.WriteString(strconv.Itoa(dm.MachineLearning.Uppers)) - case directive == "ml-specials": - s.WriteString(strconv.Itoa(dm.MachineLearning.Specials)) - case directive == "ml-others": - s.WriteString(strconv.Itoa(dm.MachineLearning.Others)) - case directive == "ml-labels": - s.WriteString(strconv.Itoa(dm.MachineLearning.Labels)) - case directive == "ml-ratio-digits": - s.WriteString(strconv.FormatFloat(dm.MachineLearning.RatioDigits, 'f', 3, 64)) - case directive == "ml-ratio-letters": - s.WriteString(strconv.FormatFloat(dm.MachineLearning.RatioLetters, 'f', 3, 64)) - case directive == "ml-ratio-specials": - s.WriteString(strconv.FormatFloat(dm.MachineLearning.RatioSpecials, 'f', 3, 64)) - case directive == "ml-ratio-others": - s.WriteString(strconv.FormatFloat(dm.MachineLearning.RatioOthers, 'f', 3, 64)) - case directive == "ml-consecutive-chars": - s.WriteString(strconv.Itoa(dm.MachineLearning.ConsecutiveChars)) - case directive == "ml-consecutive-vowels": - s.WriteString(strconv.Itoa(dm.MachineLearning.ConsecutiveVowels)) - case directive == "ml-consecutive-digits": - s.WriteString(strconv.Itoa(dm.MachineLearning.ConsecutiveDigits)) - case directive == "ml-consecutive-consonants": - s.WriteString(strconv.Itoa(dm.MachineLearning.ConsecutiveConsonants)) - case directive == "ml-size": - s.WriteString(strconv.Itoa(dm.MachineLearning.Size)) - case directive == "ml-occurrences": - s.WriteString(strconv.Itoa(dm.MachineLearning.Occurrences)) - case directive == "ml-uncommon-qtypes": - s.WriteString(strconv.Itoa(dm.MachineLearning.UncommonQtypes)) - default: - return errors.New(ErrorUnexpectedDirective + directive) - } - } - return nil -} - -func (dm *DNSMessage) Bytes(format []string, fieldDelimiter string, fieldBoundary string) []byte { - line, err := dm.ToTextLine(format, fieldDelimiter, fieldBoundary) - if err != nil { - log.Fatalf("unsupport directive for text format: %s", err) - } - return line -} - -func (dm *DNSMessage) String(format []string, fieldDelimiter string, fieldBoundary string) string { - return string(dm.Bytes(format, fieldDelimiter, fieldBoundary)) -} - -func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBoundary string) ([]byte, error) { - var s strings.Builder - - an := dm.DNS.DNSRRs.Answers - qname := dm.DNS.Qname - flags := dm.DNS.Flags - - for i, directive := range format { - switch { - case directive == "timestamp-rfc3339ns", directive == "timestamp": - s.WriteString(dm.DNSTap.TimestampRFC3339) - case directive == "timestamp-unixms": - s.WriteString(fmt.Sprintf("%d", dm.DNSTap.Timestamp/1000000)) - case directive == "timestamp-unixus": - s.WriteString(fmt.Sprintf("%d", dm.DNSTap.Timestamp/1000)) - case directive == "timestamp-unixns": - s.WriteString(fmt.Sprintf("%d", dm.DNSTap.Timestamp)) - case directive == "localtime": - ts := time.Unix(int64(dm.DNSTap.TimeSec), int64(dm.DNSTap.TimeNsec)) - s.WriteString(ts.Format("2006-01-02 15:04:05.999999999")) - case directive == "qname": - if len(qname) == 0 { - s.WriteString(".") - } else { - QuoteStringAndWrite(&s, qname, fieldDelimiter, fieldBoundary) - } - case directive == "identity": - if len(dm.DNSTap.Identity) == 0 { - s.WriteString("-") - } else { - QuoteStringAndWrite(&s, dm.DNSTap.Identity, fieldDelimiter, fieldBoundary) - } - case directive == "peer-name": - if len(dm.DNSTap.PeerName) == 0 { - s.WriteString("-") - } else { - QuoteStringAndWrite(&s, dm.DNSTap.PeerName, fieldDelimiter, fieldBoundary) - } - case directive == "version": - if len(dm.DNSTap.Version) == 0 { - s.WriteString("-") - } else { - QuoteStringAndWrite(&s, dm.DNSTap.Version, fieldDelimiter, fieldBoundary) - } - case directive == "extra": - s.WriteString(dm.DNSTap.Extra) - case directive == "policy-rule": - s.WriteString(dm.DNSTap.PolicyRule) - case directive == "policy-type": - s.WriteString(dm.DNSTap.PolicyType) - case directive == "policy-action": - s.WriteString(dm.DNSTap.PolicyAction) - case directive == "policy-match": - s.WriteString(dm.DNSTap.PolicyMatch) - case directive == "policy-value": - s.WriteString(dm.DNSTap.PolicyValue) - case directive == "query-zone": - s.WriteString(dm.DNSTap.QueryZone) - case directive == "operation": - s.WriteString(dm.DNSTap.Operation) - case directive == "rcode": - s.WriteString(dm.DNS.Rcode) - case directive == "id": - s.WriteString(strconv.Itoa(dm.DNS.ID)) - case directive == "queryip": - s.WriteString(dm.NetworkInfo.QueryIP) - case directive == "queryport": - s.WriteString(dm.NetworkInfo.QueryPort) - case directive == "responseip": - s.WriteString(dm.NetworkInfo.ResponseIP) - case directive == "responseport": - s.WriteString(dm.NetworkInfo.ResponsePort) - case directive == "family": - s.WriteString(dm.NetworkInfo.Family) - case directive == "protocol": - s.WriteString(dm.NetworkInfo.Protocol) - case directive == "length-unit": - s.WriteString(strconv.Itoa(dm.DNS.Length) + "b") - case directive == "length": - s.WriteString(strconv.Itoa(dm.DNS.Length)) - case directive == "qtype": - s.WriteString(dm.DNS.Qtype) - case directive == "qclass": - s.WriteString(dm.DNS.Qclass) - case directive == "latency": - s.WriteString(fmt.Sprintf("%.9f", dm.DNSTap.Latency)) - case directive == "malformed": - if dm.DNS.MalformedPacket { - s.WriteString("PKTERR") - } else { - s.WriteByte('-') - } - case directive == "qr": - s.WriteString(dm.DNS.Type) - case directive == "opcode": - s.WriteString(strconv.Itoa(dm.DNS.Opcode)) - case directive == "tr": - if dm.NetworkInfo.TCPReassembled { - s.WriteString("TR") - } else { - s.WriteByte('-') - } - case directive == "df": - if dm.NetworkInfo.IPDefragmented { - s.WriteString("DF") - } else { - s.WriteByte('-') - } - case directive == "tc": - if flags.TC { - s.WriteString("TC") - } else { - s.WriteByte('-') - } - case directive == "aa": - if flags.AA { - s.WriteString("AA") - } else { - s.WriteByte('-') - } - case directive == "ra": - if flags.RA { - s.WriteString("RA") - } else { - s.WriteByte('-') - } - case directive == "ad": - if flags.AD { - s.WriteString("AD") - } else { - s.WriteByte('-') - } - case directive == "ttl": - if len(an) > 0 { - s.WriteString(strconv.Itoa(an[0].TTL)) - } else { - s.WriteByte('-') - } - case directive == "answer": - if len(an) > 0 { - s.WriteString(an[0].Rdata) - } else { - s.WriteByte('-') - } - - case directive == "questionscount" || directive == "qdcount": - s.WriteString(strconv.Itoa(dm.DNS.QdCount)) - case directive == "answercount" || directive == "ancount": - s.WriteString(strconv.Itoa(dm.DNS.AnCount)) - case directive == "nscount": - s.WriteString(strconv.Itoa(dm.DNS.NsCount)) - case directive == "arcount": - s.WriteString(strconv.Itoa(dm.DNS.ArCount)) - - case directive == "edns-csubnet": - if len(dm.EDNS.Options) > 0 { - for _, opt := range dm.EDNS.Options { - if opt.Name == "CSUBNET" { - s.WriteString(opt.Data) - break - } - } - } else { - s.WriteByte('-') - } - - // more directives from collectors - case PdnsDirectives.MatchString(directive): - err := dm.handlePdnsDirectives(directive, &s) - if err != nil { - return nil, err - } - - // more directives from transformers - case ReducerDirectives.MatchString(directive): - err := dm.handleReducerDirectives(directive, &s) - if err != nil { - return nil, err - } - case GeoIPDirectives.MatchString(directive): - err := dm.handleGeoIPDirectives(directive, &s) - if err != nil { - return nil, err - } - case SuspiciousDirectives.MatchString(directive): - err := dm.handleSuspiciousDirectives(directive, &s) - if err != nil { - return nil, err - } - case PublicSuffixDirectives.MatchString(directive): - err := dm.handlePublicSuffixDirectives(directive, &s) - if err != nil { - return nil, err - } - case ExtractedDirectives.MatchString(directive): - err := dm.handleExtractedDirectives(directive, &s) - if err != nil { - return nil, err - } - case MachineLearningDirectives.MatchString(directive): - err := dm.handleMachineLearningDirectives(directive, &s) - if err != nil { - return nil, err - } - case FilteringDirectives.MatchString(directive): - err := dm.handleFilteringDirectives(directive, &s) - if err != nil { - return nil, err - } - case ATagsDirectives.MatchString(directive): - err := dm.handleATagsDirectives(directive, &s) - if err != nil { - return nil, err - } - case RawTextDirective.MatchString(directive): - directive = strings.ReplaceAll(directive, "{", "") - directive = strings.ReplaceAll(directive, "}", "") - s.WriteString(directive) - - // handle invalid directive - default: - return nil, errors.New(ErrorUnexpectedDirective + directive) - } - - if i < len(format)-1 { - if len(fieldDelimiter) > 0 { - s.WriteString(fieldDelimiter) - } - } - } - return []byte(s.String()), nil -} - -func (dm *DNSMessage) ToTextTemplate(template string) (string, error) { - context := pongo2.Context{"dm": dm} - - // Parse and execute the template - tmpl, err := pongo2.FromString(template) - if err != nil { - return "", err - } - - result, err := tmpl.Execute(context) - if err != nil { - return "", err - } - return result, nil -} - -func (dm *DNSMessage) ToJSON() string { - buffer := new(bytes.Buffer) - json.NewEncoder(buffer).Encode(dm) - return buffer.String() -} - -func (dm *DNSMessage) ToFlatJSON() (string, error) { - buffer := new(bytes.Buffer) - flat, err := dm.Flatten() - if err != nil { - return "", err - } - json.NewEncoder(buffer).Encode(flat) - return buffer.String(), nil -} - -func (dm *DNSMessage) ToDNSTap(extended bool) ([]byte, error) { - if len(dm.DNSTap.Payload) > 0 { - return dm.DNSTap.Payload, nil - } - - dt := &dnstap.Dnstap{} - t := dnstap.Dnstap_MESSAGE - dt.Identity = []byte(dm.DNSTap.Identity) - dt.Version = []byte("-") - dt.Type = &t - - mt := dnstap.Message_Type(dnstap.Message_Type_value[dm.DNSTap.Operation]) - - var sf dnstap.SocketFamily - if ipNet, valid := netutils.IPToInet[dm.NetworkInfo.Family]; valid { - sf = dnstap.SocketFamily(dnstap.SocketFamily_value[ipNet]) - } - sp := dnstap.SocketProtocol(dnstap.SocketProtocol_value[dm.NetworkInfo.Protocol]) - tsec := uint64(dm.DNSTap.TimeSec) - tnsec := uint32(dm.DNSTap.TimeNsec) - - var rport uint32 - var qport uint32 - if dm.NetworkInfo.ResponsePort != "-" { - if port, err := strconv.Atoi(dm.NetworkInfo.ResponsePort); err != nil { - return nil, err - } else if port < 0 || port > 65535 { - return nil, errors.New("invalid response port value") - } else { - rport = uint32(port) - } - } - - if dm.NetworkInfo.QueryPort != "-" { - if port, err := strconv.Atoi(dm.NetworkInfo.QueryPort); err != nil { - return nil, err - } else if port < 0 || port > 65535 { - return nil, errors.New("invalid query port value") - } else { - qport = uint32(port) - } - } - - msg := &dnstap.Message{Type: &mt} - - msg.SocketFamily = &sf - msg.SocketProtocol = &sp - - reqIP := net.ParseIP(dm.NetworkInfo.QueryIP) - if dm.NetworkInfo.Family == netutils.ProtoIPv4 { - msg.QueryAddress = reqIP.To4() - } else { - msg.QueryAddress = reqIP.To16() - } - msg.QueryPort = &qport - - rspIP := net.ParseIP(dm.NetworkInfo.ResponseIP) - if dm.NetworkInfo.Family == netutils.ProtoIPv4 { - msg.ResponseAddress = rspIP.To4() - } else { - msg.ResponseAddress = rspIP.To16() - } - msg.ResponsePort = &rport - - if dm.DNS.Type == DNSQuery { - msg.QueryMessage = dm.DNS.Payload - msg.QueryTimeSec = &tsec - msg.QueryTimeNsec = &tnsec - } else { - msg.ResponseTimeSec = &tsec - msg.ResponseTimeNsec = &tnsec - msg.ResponseMessage = dm.DNS.Payload - } - - dt.Message = msg - - // add dnstap extra - if len(dm.DNSTap.Extra) > 0 { - dt.Extra = []byte(dm.DNSTap.Extra) - } - - // contruct new dnstap field with all tranformations - // the original extra field is kept if exist - if extended { - ednstap := &ExtendedDnstap{} - - // add original dnstap value if exist - if len(dm.DNSTap.Extra) > 0 { - ednstap.OriginalDnstapExtra = []byte(dm.DNSTap.Extra) - } - - // add additionnals tags ? - if dm.ATags != nil { - ednstap.Atags = &ExtendedATags{ - Tags: dm.ATags.Tags, - } - } - - // add public suffix - if dm.PublicSuffix != nil { - ednstap.Normalize = &ExtendedNormalize{ - Tld: dm.PublicSuffix.QnamePublicSuffix, - EtldPlusOne: dm.PublicSuffix.QnameEffectiveTLDPlusOne, - } - } - - // add filtering - if dm.Filtering != nil { - ednstap.Filtering = &ExtendedFiltering{ - SampleRate: uint32(dm.Filtering.SampleRate), - } - } - - // add geo - if dm.Geo != nil { - ednstap.Geo = &ExtendedGeo{ - City: dm.Geo.City, - Continent: dm.Geo.Continent, - Isocode: dm.Geo.CountryIsoCode, - AsNumber: dm.Geo.AutonomousSystemNumber, - AsOrg: dm.Geo.AutonomousSystemOrg, - } - } - - extendedData, err := proto.Marshal(ednstap) - if err != nil { - return nil, err - } - dt.Extra = extendedData - } - - data, err := proto.Marshal(dt) - if err != nil { - return nil, err - } - return data, nil -} - -func (dm *DNSMessage) ToPacketLayer() ([]gopacket.SerializableLayer, error) { - if len(dm.DNS.Payload) == 0 { - return nil, errors.New("payload is empty") - } - - eth := &layers.Ethernet{ - SrcMAC: net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, - DstMAC: net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}} - ip4 := &layers.IPv4{Version: 4, TTL: 64} - ip6 := &layers.IPv6{Version: 6} - udp := &layers.UDP{} - tcp := &layers.TCP{} - - // prepare ip - srcIP, srcPort, dstIP, dstPort := GetIPPort(dm) - if srcPort < 0 || srcPort > math.MaxUint16 { - return nil, errors.New("invalid source port value") - } - if dstPort < 0 || dstPort > math.MaxUint16 { - return nil, errors.New("invalid destination port value") - } - - // packet layer array - pkt := []gopacket.SerializableLayer{} - - // set source and destination IP - switch dm.NetworkInfo.Family { - case netutils.ProtoIPv4: - eth.EthernetType = layers.EthernetTypeIPv4 - ip4.SrcIP = net.ParseIP(srcIP) - ip4.DstIP = net.ParseIP(dstIP) - case netutils.ProtoIPv6: - eth.EthernetType = layers.EthernetTypeIPv6 - ip6.SrcIP = net.ParseIP(srcIP) - ip6.DstIP = net.ParseIP(dstIP) - default: - return nil, errors.New("family (" + dm.NetworkInfo.Family + ") not yet implemented") - } - - // set transport - switch dm.NetworkInfo.Protocol { - - // DNS over UDP - case netutils.ProtoUDP: - udp.SrcPort = layers.UDPPort(srcPort) - udp.DstPort = layers.UDPPort(dstPort) - - // update iplayer - switch dm.NetworkInfo.Family { - case netutils.ProtoIPv4: - ip4.Protocol = layers.IPProtocolUDP - udp.SetNetworkLayerForChecksum(ip4) - pkt = append(pkt, gopacket.Payload(dm.DNS.Payload), udp, ip4) - case netutils.ProtoIPv6: - ip6.NextHeader = layers.IPProtocolUDP - udp.SetNetworkLayerForChecksum(ip6) - pkt = append(pkt, gopacket.Payload(dm.DNS.Payload), udp, ip6) - } - - // DNS over TCP - case netutils.ProtoTCP: - tcp.SrcPort = layers.TCPPort(srcPort) - tcp.DstPort = layers.TCPPort(dstPort) - tcp.PSH = true - tcp.Window = 65535 - - // dns length - dnsLengthField := make([]byte, 2) - binary.BigEndian.PutUint16(dnsLengthField[0:], uint16(dm.DNS.Length)) - - // update iplayer - switch dm.NetworkInfo.Family { - case netutils.ProtoIPv4: - ip4.Protocol = layers.IPProtocolTCP - tcp.SetNetworkLayerForChecksum(ip4) - pkt = append(pkt, gopacket.Payload(append(dnsLengthField, dm.DNS.Payload...)), tcp, ip4) - case netutils.ProtoIPv6: - ip6.NextHeader = layers.IPProtocolTCP - tcp.SetNetworkLayerForChecksum(ip6) - pkt = append(pkt, gopacket.Payload(append(dnsLengthField, dm.DNS.Payload...)), tcp, ip6) - } - - // DNS over HTTPS and DNS over TLS - // These protocols are translated to DNS over UDP - case ProtoDoH, ProtoDoT: - udp.SrcPort = layers.UDPPort(srcPort) - udp.DstPort = layers.UDPPort(dstPort) - - // update iplayer - switch dm.NetworkInfo.Family { - case netutils.ProtoIPv4: - ip4.Protocol = layers.IPProtocolUDP - udp.SetNetworkLayerForChecksum(ip4) - pkt = append(pkt, gopacket.Payload(dm.DNS.Payload), udp, ip4) - case netutils.ProtoIPv6: - ip6.NextHeader = layers.IPProtocolUDP - udp.SetNetworkLayerForChecksum(ip6) - pkt = append(pkt, gopacket.Payload(dm.DNS.Payload), udp, ip6) - } - - default: - return nil, errors.New("protocol " + dm.NetworkInfo.Protocol + " not yet implemented") - } - - pkt = append(pkt, eth) - - return pkt, nil -} - -func (dm *DNSMessage) Flatten() (map[string]interface{}, error) { - dnsFields := map[string]interface{}{ - "dns.flags.aa": dm.DNS.Flags.AA, - "dns.flags.ad": dm.DNS.Flags.AD, - "dns.flags.qr": dm.DNS.Flags.QR, - "dns.flags.ra": dm.DNS.Flags.RA, - "dns.flags.tc": dm.DNS.Flags.TC, - "dns.flags.rd": dm.DNS.Flags.RD, - "dns.flags.cd": dm.DNS.Flags.CD, - "dns.length": dm.DNS.Length, - "dns.malformed-packet": dm.DNS.MalformedPacket, - "dns.id": dm.DNS.ID, - "dns.opcode": dm.DNS.Opcode, - "dns.qname": dm.DNS.Qname, - "dns.qtype": dm.DNS.Qtype, - "dns.qclass": dm.DNS.Qclass, - "dns.rcode": dm.DNS.Rcode, - "dns.qdcount": dm.DNS.QdCount, - "dns.ancount": dm.DNS.AnCount, - "dns.arcount": dm.DNS.ArCount, - "dns.nscount": dm.DNS.NsCount, - "dnstap.identity": dm.DNSTap.Identity, - "dnstap.latency": dm.DNSTap.Latency, - "dnstap.operation": dm.DNSTap.Operation, - "dnstap.timestamp-rfc3339ns": dm.DNSTap.TimestampRFC3339, - "dnstap.version": dm.DNSTap.Version, - "dnstap.extra": dm.DNSTap.Extra, - "dnstap.policy-rule": dm.DNSTap.PolicyRule, - "dnstap.policy-type": dm.DNSTap.PolicyType, - "dnstap.policy-action": dm.DNSTap.PolicyAction, - "dnstap.policy-match": dm.DNSTap.PolicyMatch, - "dnstap.policy-value": dm.DNSTap.PolicyValue, - "dnstap.peer-name": dm.DNSTap.PeerName, - "dnstap.query-zone": dm.DNSTap.QueryZone, - "edns.dnssec-ok": dm.EDNS.Do, - "edns.rcode": dm.EDNS.ExtendedRcode, - "edns.udp-size": dm.EDNS.UDPSize, - "edns.version": dm.EDNS.Version, - "network.family": dm.NetworkInfo.Family, - "network.ip-defragmented": dm.NetworkInfo.IPDefragmented, - "network.protocol": dm.NetworkInfo.Protocol, - "network.query-ip": dm.NetworkInfo.QueryIP, - "network.query-port": dm.NetworkInfo.QueryPort, - "network.response-ip": dm.NetworkInfo.ResponseIP, - "network.response-port": dm.NetworkInfo.ResponsePort, - "network.tcp-reassembled": dm.NetworkInfo.TCPReassembled, - } - - // Add empty slices - if len(dm.DNS.DNSRRs.Answers) == 0 { - dnsFields["dns.resource-records.an"] = "-" - } - if len(dm.DNS.DNSRRs.Records) == 0 { - dnsFields["dns.resource-records.ar"] = "-" - } - if len(dm.DNS.DNSRRs.Nameservers) == 0 { - dnsFields["dns.resource-records.ns"] = "-" - } - if len(dm.EDNS.Options) == 0 { - dnsFields["edns.options"] = "-" - } - - // Add DNSAnswer fields: "dns.resource-records.an.0.name": "google.nl" - // nolint: goconst - for i, an := range dm.DNS.DNSRRs.Answers { - prefixAn := "dns.resource-records.an." + strconv.Itoa(i) - dnsFields[prefixAn+".name"] = an.Name - dnsFields[prefixAn+".rdata"] = an.Rdata - dnsFields[prefixAn+".rdatatype"] = an.Rdatatype - dnsFields[prefixAn+".ttl"] = an.TTL - dnsFields[prefixAn+".class"] = an.Class - } - for i, ns := range dm.DNS.DNSRRs.Nameservers { - prefixNs := "dns.resource-records.ns." + strconv.Itoa(i) - dnsFields[prefixNs+".name"] = ns.Name - dnsFields[prefixNs+".rdata"] = ns.Rdata - dnsFields[prefixNs+".rdatatype"] = ns.Rdatatype - dnsFields[prefixNs+".ttl"] = ns.TTL - dnsFields[prefixNs+".class"] = ns.Class - } - for i, ar := range dm.DNS.DNSRRs.Records { - prefixAr := "dns.resource-records.ar." + strconv.Itoa(i) - dnsFields[prefixAr+".name"] = ar.Name - dnsFields[prefixAr+".rdata"] = ar.Rdata - dnsFields[prefixAr+".rdatatype"] = ar.Rdatatype - dnsFields[prefixAr+".ttl"] = ar.TTL - dnsFields[prefixAr+".class"] = ar.Class - } - - // Add EDNSoptions fields: "edns.options.0.code": 10, - for i, opt := range dm.EDNS.Options { - prefixOpt := "edns.options." + strconv.Itoa(i) - dnsFields[prefixOpt+".code"] = opt.Code - dnsFields[prefixOpt+".data"] = opt.Data - dnsFields[prefixOpt+".name"] = opt.Name - } - - // Add TransformDNSGeo fields - if dm.Geo != nil { - dnsFields["geoip.city"] = dm.Geo.City - dnsFields["geoip.continent"] = dm.Geo.Continent - dnsFields["geoip.country-isocode"] = dm.Geo.CountryIsoCode - dnsFields["geoip.as-number"] = dm.Geo.AutonomousSystemNumber - dnsFields["geoip.as-owner"] = dm.Geo.AutonomousSystemOrg - } - - // Add TransformSuspicious fields - if dm.Suspicious != nil { - dnsFields["suspicious.score"] = dm.Suspicious.Score - dnsFields["suspicious.malformed-pkt"] = dm.Suspicious.MalformedPacket - dnsFields["suspicious.large-pkt"] = dm.Suspicious.LargePacket - dnsFields["suspicious.long-domain"] = dm.Suspicious.LongDomain - dnsFields["suspicious.slow-domain"] = dm.Suspicious.SlowDomain - dnsFields["suspicious.unallowed-chars"] = dm.Suspicious.UnallowedChars - dnsFields["suspicious.uncommon-qtypes"] = dm.Suspicious.UncommonQtypes - dnsFields["suspicious.excessive-number-labels"] = dm.Suspicious.ExcessiveNumberLabels - dnsFields["suspicious.domain"] = dm.Suspicious.Domain - } - - // Add TransformPublicSuffix fields - if dm.PublicSuffix != nil { - dnsFields["publicsuffix.tld"] = dm.PublicSuffix.QnamePublicSuffix - dnsFields["publicsuffix.etld+1"] = dm.PublicSuffix.QnameEffectiveTLDPlusOne - dnsFields["publicsuffix.managed-icann"] = dm.PublicSuffix.ManagedByICANN - } - - // Add TransformExtracted fields - if dm.Extracted != nil { - dnsFields["extracted.dns_payload"] = dm.Extracted.Base64Payload - } - - // Add TransformReducer fields - if dm.Reducer != nil { - dnsFields["reducer.occurrences"] = dm.Reducer.Occurrences - dnsFields["reducer.cumulative-length"] = dm.Reducer.CumulativeLength - } - - // Add TransformFiltering fields - if dm.Filtering != nil { - dnsFields["filtering.sample-rate"] = dm.Filtering.SampleRate - } - - // Add TransformML fields - if dm.MachineLearning != nil { - dnsFields["ml.entropy"] = dm.MachineLearning.Entropy - dnsFields["ml.length"] = dm.MachineLearning.Length - dnsFields["ml.labels"] = dm.MachineLearning.Labels - dnsFields["ml.digits"] = dm.MachineLearning.Digits - dnsFields["ml.lowers"] = dm.MachineLearning.Lowers - dnsFields["ml.uppers"] = dm.MachineLearning.Uppers - dnsFields["ml.specials"] = dm.MachineLearning.Specials - dnsFields["ml.others"] = dm.MachineLearning.Others - dnsFields["ml.ratio-digits"] = dm.MachineLearning.RatioDigits - dnsFields["ml.ratio-letters"] = dm.MachineLearning.RatioLetters - dnsFields["ml.ratio-specials"] = dm.MachineLearning.RatioSpecials - dnsFields["ml.ratio-others"] = dm.MachineLearning.RatioOthers - dnsFields["ml.consecutive-chars"] = dm.MachineLearning.ConsecutiveChars - dnsFields["ml.consecutive-vowels"] = dm.MachineLearning.ConsecutiveVowels - dnsFields["ml.consecutive-digits"] = dm.MachineLearning.ConsecutiveDigits - dnsFields["ml.consecutive-consonants"] = dm.MachineLearning.ConsecutiveConsonants - dnsFields["ml.size"] = dm.MachineLearning.Size - dnsFields["ml.occurrences"] = dm.MachineLearning.Occurrences - dnsFields["ml.uncommon-qtypes"] = dm.MachineLearning.UncommonQtypes - } - - // Add TransformATags fields - if dm.ATags != nil { - if len(dm.ATags.Tags) == 0 { - dnsFields["atags.tags"] = "-" - } - for i, tag := range dm.ATags.Tags { - dnsFields["atags.tags."+strconv.Itoa(i)] = tag - } - } - - // Add PowerDNS collectors fields - if dm.PowerDNS != nil { - if len(dm.PowerDNS.Tags) == 0 { - dnsFields["powerdns.tags"] = "-" - } - for i, tag := range dm.PowerDNS.Tags { - dnsFields["powerdns.tags."+strconv.Itoa(i)] = tag - } - dnsFields["powerdns.original-request-subnet"] = dm.PowerDNS.OriginalRequestSubnet - dnsFields["powerdns.applied-policy"] = dm.PowerDNS.AppliedPolicy - dnsFields["powerdns.applied-policy-hit"] = dm.PowerDNS.AppliedPolicyHit - dnsFields["powerdns.applied-policy-kind"] = dm.PowerDNS.AppliedPolicyKind - dnsFields["powerdns.applied-policy-trigger"] = dm.PowerDNS.AppliedPolicyTrigger - dnsFields["powerdns.applied-policy-type"] = dm.PowerDNS.AppliedPolicyType - for mk, mv := range dm.PowerDNS.Metadata { - dnsFields["powerdns.metadata."+mk] = mv - } - dnsFields["powerdns.http-version"] = dm.PowerDNS.HTTPVersion - } - - // relabeling ? - if dm.Relabeling != nil { - err := dm.ApplyRelabeling(dnsFields) - if err != nil { - return nil, err - } - } - - return dnsFields, nil -} - -func (dm *DNSMessage) ApplyRelabeling(dnsFields map[string]interface{}) error { - - for _, label := range dm.Relabeling.Rules { - regex := label.Regex - for key := range dnsFields { - if regex.MatchString(key) { - if label.Action == "rename" { - replacement := label.Replacement - if value, exists := dnsFields[replacement]; exists { - switch v := value.(type) { - case []string: - dnsFields[replacement] = append(v, ConvertToString(dnsFields[key])) - default: - dnsFields[replacement] = []string{ConvertToString(v), ConvertToString(dnsFields[key])} - } - } else { - dnsFields[replacement] = ConvertToString(dnsFields[key]) - } - } - - // delete on all case - delete(dnsFields, key) - } - } - } - - return nil -} - -func (dm *DNSMessage) Matching(matching map[string]interface{}) (error, bool) { - if len(matching) == 0 { - return nil, false - } - - dmValue := reflect.ValueOf(dm) - - if dmValue.Kind() == reflect.Ptr { - dmValue = dmValue.Elem() - } - - var isMatch = true - - for nestedKeys, value := range matching { - realValue, found := getFieldByJSONTag(dmValue, nestedKeys) - if !found { - return nil, false - } - - expectedValue := reflect.ValueOf(value) - switch expectedValue.Kind() { - // integer - case reflect.Int: - match, err := matchUserInteger(realValue, expectedValue) - if err != nil { - return err, false - } - if !match { - return nil, false - } - - // string - case reflect.String: - match, err := matchUserPattern(realValue, expectedValue) - if err != nil { - return err, false - } - if !match { - return nil, false - } - - // bool - case reflect.Bool: - match, err := matchUserBoolean(realValue, expectedValue) - if err != nil { - return err, false - } - if !match { - return nil, false - } - - // map - case reflect.Map: - match, err := matchUserMap(realValue, expectedValue) - if err != nil { - return err, false - } - if !match { - return nil, false - } - - // list/slice - case reflect.Slice: - match, err := matchUserSlice(realValue, expectedValue) - if err != nil { - return err, false - } - if !match { - return nil, false - } - - // other user types - default: - return fmt.Errorf("unsupported type value: %s", expectedValue.Kind()), false - } - - } - - return nil, isMatch -} diff --git a/dnsutils/dnsmessage_dnstap.go b/dnsutils/dnsmessage_dnstap.go new file mode 100644 index 00000000..9edb4220 --- /dev/null +++ b/dnsutils/dnsmessage_dnstap.go @@ -0,0 +1,149 @@ +package dnsutils + +import ( + "errors" + "net" + "strconv" + + "github.com/dmachard/go-dnstap-protobuf" + "github.com/dmachard/go-netutils" + "google.golang.org/protobuf/proto" +) + +func (dm *DNSMessage) ToDNSTap(extended bool) ([]byte, error) { + if len(dm.DNSTap.Payload) > 0 { + return dm.DNSTap.Payload, nil + } + + dt := &dnstap.Dnstap{} + t := dnstap.Dnstap_MESSAGE + dt.Identity = []byte(dm.DNSTap.Identity) + dt.Version = []byte("-") + dt.Type = &t + + mt := dnstap.Message_Type(dnstap.Message_Type_value[dm.DNSTap.Operation]) + + var sf dnstap.SocketFamily + if ipNet, valid := netutils.IPToInet[dm.NetworkInfo.Family]; valid { + sf = dnstap.SocketFamily(dnstap.SocketFamily_value[ipNet]) + } + sp := dnstap.SocketProtocol(dnstap.SocketProtocol_value[dm.NetworkInfo.Protocol]) + tsec := uint64(dm.DNSTap.TimeSec) + tnsec := uint32(dm.DNSTap.TimeNsec) + + var rport uint32 + var qport uint32 + if dm.NetworkInfo.ResponsePort != "-" { + if port, err := strconv.Atoi(dm.NetworkInfo.ResponsePort); err != nil { + return nil, err + } else if port < 0 || port > 65535 { + return nil, errors.New("invalid response port value") + } else { + rport = uint32(port) + } + } + + if dm.NetworkInfo.QueryPort != "-" { + if port, err := strconv.Atoi(dm.NetworkInfo.QueryPort); err != nil { + return nil, err + } else if port < 0 || port > 65535 { + return nil, errors.New("invalid query port value") + } else { + qport = uint32(port) + } + } + + msg := &dnstap.Message{Type: &mt} + + msg.SocketFamily = &sf + msg.SocketProtocol = &sp + + reqIP := net.ParseIP(dm.NetworkInfo.QueryIP) + if dm.NetworkInfo.Family == netutils.ProtoIPv4 { + msg.QueryAddress = reqIP.To4() + } else { + msg.QueryAddress = reqIP.To16() + } + msg.QueryPort = &qport + + rspIP := net.ParseIP(dm.NetworkInfo.ResponseIP) + if dm.NetworkInfo.Family == netutils.ProtoIPv4 { + msg.ResponseAddress = rspIP.To4() + } else { + msg.ResponseAddress = rspIP.To16() + } + msg.ResponsePort = &rport + + if dm.DNS.Type == DNSQuery { + msg.QueryMessage = dm.DNS.Payload + msg.QueryTimeSec = &tsec + msg.QueryTimeNsec = &tnsec + } else { + msg.ResponseTimeSec = &tsec + msg.ResponseTimeNsec = &tnsec + msg.ResponseMessage = dm.DNS.Payload + } + + dt.Message = msg + + // add dnstap extra + if len(dm.DNSTap.Extra) > 0 { + dt.Extra = []byte(dm.DNSTap.Extra) + } + + // contruct new dnstap field with all tranformations + // the original extra field is kept if exist + if extended { + ednstap := &ExtendedDnstap{} + + // add original dnstap value if exist + if len(dm.DNSTap.Extra) > 0 { + ednstap.OriginalDnstapExtra = []byte(dm.DNSTap.Extra) + } + + // add additionnals tags ? + if dm.ATags != nil { + ednstap.Atags = &ExtendedATags{ + Tags: dm.ATags.Tags, + } + } + + // add public suffix + if dm.PublicSuffix != nil { + ednstap.Normalize = &ExtendedNormalize{ + Tld: dm.PublicSuffix.QnamePublicSuffix, + EtldPlusOne: dm.PublicSuffix.QnameEffectiveTLDPlusOne, + } + } + + // add filtering + if dm.Filtering != nil { + ednstap.Filtering = &ExtendedFiltering{ + SampleRate: uint32(dm.Filtering.SampleRate), + } + } + + // add geo + if dm.Geo != nil { + ednstap.Geo = &ExtendedGeo{ + City: dm.Geo.City, + Continent: dm.Geo.Continent, + Isocode: dm.Geo.CountryIsoCode, + AsNumber: dm.Geo.AutonomousSystemNumber, + AsOrg: dm.Geo.AutonomousSystemOrg, + } + } + + extendedData, err := proto.Marshal(ednstap) + if err != nil { + return nil, err + } + dt.Extra = extendedData + } + + data, err := proto.Marshal(dt) + if err != nil { + return nil, err + } + return data, nil +} diff --git a/dnsutils/dnsmessage_json.go b/dnsutils/dnsmessage_json.go new file mode 100644 index 00000000..467b41ae --- /dev/null +++ b/dnsutils/dnsmessage_json.go @@ -0,0 +1,229 @@ +package dnsutils + +import ( + "bytes" + "encoding/json" + "strconv" +) + +func (dm *DNSMessage) ToJSON() string { + buffer := new(bytes.Buffer) + json.NewEncoder(buffer).Encode(dm) + return buffer.String() +} + +func (dm *DNSMessage) ToFlatJSON() (string, error) { + buffer := new(bytes.Buffer) + flat, err := dm.Flatten() + if err != nil { + return "", err + } + json.NewEncoder(buffer).Encode(flat) + return buffer.String(), nil +} + +func (dm *DNSMessage) Flatten() (map[string]interface{}, error) { + dnsFields := map[string]interface{}{ + "dns.flags.aa": dm.DNS.Flags.AA, + "dns.flags.ad": dm.DNS.Flags.AD, + "dns.flags.qr": dm.DNS.Flags.QR, + "dns.flags.ra": dm.DNS.Flags.RA, + "dns.flags.tc": dm.DNS.Flags.TC, + "dns.flags.rd": dm.DNS.Flags.RD, + "dns.flags.cd": dm.DNS.Flags.CD, + "dns.length": dm.DNS.Length, + "dns.malformed-packet": dm.DNS.MalformedPacket, + "dns.id": dm.DNS.ID, + "dns.opcode": dm.DNS.Opcode, + "dns.qname": dm.DNS.Qname, + "dns.qtype": dm.DNS.Qtype, + "dns.qclass": dm.DNS.Qclass, + "dns.rcode": dm.DNS.Rcode, + "dns.qdcount": dm.DNS.QdCount, + "dns.ancount": dm.DNS.AnCount, + "dns.arcount": dm.DNS.ArCount, + "dns.nscount": dm.DNS.NsCount, + "dnstap.identity": dm.DNSTap.Identity, + "dnstap.latency": dm.DNSTap.Latency, + "dnstap.operation": dm.DNSTap.Operation, + "dnstap.timestamp-rfc3339ns": dm.DNSTap.TimestampRFC3339, + "dnstap.version": dm.DNSTap.Version, + "dnstap.extra": dm.DNSTap.Extra, + "dnstap.policy-rule": dm.DNSTap.PolicyRule, + "dnstap.policy-type": dm.DNSTap.PolicyType, + "dnstap.policy-action": dm.DNSTap.PolicyAction, + "dnstap.policy-match": dm.DNSTap.PolicyMatch, + "dnstap.policy-value": dm.DNSTap.PolicyValue, + "dnstap.peer-name": dm.DNSTap.PeerName, + "dnstap.query-zone": dm.DNSTap.QueryZone, + "edns.dnssec-ok": dm.EDNS.Do, + "edns.rcode": dm.EDNS.ExtendedRcode, + "edns.udp-size": dm.EDNS.UDPSize, + "edns.version": dm.EDNS.Version, + "network.family": dm.NetworkInfo.Family, + "network.ip-defragmented": dm.NetworkInfo.IPDefragmented, + "network.protocol": dm.NetworkInfo.Protocol, + "network.query-ip": dm.NetworkInfo.QueryIP, + "network.query-port": dm.NetworkInfo.QueryPort, + "network.response-ip": dm.NetworkInfo.ResponseIP, + "network.response-port": dm.NetworkInfo.ResponsePort, + "network.tcp-reassembled": dm.NetworkInfo.TCPReassembled, + } + + // Add empty slices + if len(dm.DNS.DNSRRs.Answers) == 0 { + dnsFields["dns.resource-records.an"] = "-" + } + if len(dm.DNS.DNSRRs.Records) == 0 { + dnsFields["dns.resource-records.ar"] = "-" + } + if len(dm.DNS.DNSRRs.Nameservers) == 0 { + dnsFields["dns.resource-records.ns"] = "-" + } + if len(dm.EDNS.Options) == 0 { + dnsFields["edns.options"] = "-" + } + + // Add DNSAnswer fields: "dns.resource-records.an.0.name": "google.nl" + // nolint: goconst + for i, an := range dm.DNS.DNSRRs.Answers { + prefixAn := "dns.resource-records.an." + strconv.Itoa(i) + dnsFields[prefixAn+".name"] = an.Name + dnsFields[prefixAn+".rdata"] = an.Rdata + dnsFields[prefixAn+".rdatatype"] = an.Rdatatype + dnsFields[prefixAn+".ttl"] = an.TTL + dnsFields[prefixAn+".class"] = an.Class + } + for i, ns := range dm.DNS.DNSRRs.Nameservers { + prefixNs := "dns.resource-records.ns." + strconv.Itoa(i) + dnsFields[prefixNs+".name"] = ns.Name + dnsFields[prefixNs+".rdata"] = ns.Rdata + dnsFields[prefixNs+".rdatatype"] = ns.Rdatatype + dnsFields[prefixNs+".ttl"] = ns.TTL + dnsFields[prefixNs+".class"] = ns.Class + } + for i, ar := range dm.DNS.DNSRRs.Records { + prefixAr := "dns.resource-records.ar." + strconv.Itoa(i) + dnsFields[prefixAr+".name"] = ar.Name + dnsFields[prefixAr+".rdata"] = ar.Rdata + dnsFields[prefixAr+".rdatatype"] = ar.Rdatatype + dnsFields[prefixAr+".ttl"] = ar.TTL + dnsFields[prefixAr+".class"] = ar.Class + } + + // Add EDNSoptions fields: "edns.options.0.code": 10, + for i, opt := range dm.EDNS.Options { + prefixOpt := "edns.options." + strconv.Itoa(i) + dnsFields[prefixOpt+".code"] = opt.Code + dnsFields[prefixOpt+".data"] = opt.Data + dnsFields[prefixOpt+".name"] = opt.Name + } + + // Add TransformDNSGeo fields + if dm.Geo != nil { + dnsFields["geoip.city"] = dm.Geo.City + dnsFields["geoip.continent"] = dm.Geo.Continent + dnsFields["geoip.country-isocode"] = dm.Geo.CountryIsoCode + dnsFields["geoip.as-number"] = dm.Geo.AutonomousSystemNumber + dnsFields["geoip.as-owner"] = dm.Geo.AutonomousSystemOrg + } + + // Add TransformSuspicious fields + if dm.Suspicious != nil { + dnsFields["suspicious.score"] = dm.Suspicious.Score + dnsFields["suspicious.malformed-pkt"] = dm.Suspicious.MalformedPacket + dnsFields["suspicious.large-pkt"] = dm.Suspicious.LargePacket + dnsFields["suspicious.long-domain"] = dm.Suspicious.LongDomain + dnsFields["suspicious.slow-domain"] = dm.Suspicious.SlowDomain + dnsFields["suspicious.unallowed-chars"] = dm.Suspicious.UnallowedChars + dnsFields["suspicious.uncommon-qtypes"] = dm.Suspicious.UncommonQtypes + dnsFields["suspicious.excessive-number-labels"] = dm.Suspicious.ExcessiveNumberLabels + dnsFields["suspicious.domain"] = dm.Suspicious.Domain + } + + // Add TransformPublicSuffix fields + if dm.PublicSuffix != nil { + dnsFields["publicsuffix.tld"] = dm.PublicSuffix.QnamePublicSuffix + dnsFields["publicsuffix.etld+1"] = dm.PublicSuffix.QnameEffectiveTLDPlusOne + dnsFields["publicsuffix.managed-icann"] = dm.PublicSuffix.ManagedByICANN + } + + // Add TransformExtracted fields + if dm.Extracted != nil { + dnsFields["extracted.dns_payload"] = dm.Extracted.Base64Payload + } + + // Add TransformReducer fields + if dm.Reducer != nil { + dnsFields["reducer.occurrences"] = dm.Reducer.Occurrences + dnsFields["reducer.cumulative-length"] = dm.Reducer.CumulativeLength + } + + // Add TransformFiltering fields + if dm.Filtering != nil { + dnsFields["filtering.sample-rate"] = dm.Filtering.SampleRate + } + + // Add TransformML fields + if dm.MachineLearning != nil { + dnsFields["ml.entropy"] = dm.MachineLearning.Entropy + dnsFields["ml.length"] = dm.MachineLearning.Length + dnsFields["ml.labels"] = dm.MachineLearning.Labels + dnsFields["ml.digits"] = dm.MachineLearning.Digits + dnsFields["ml.lowers"] = dm.MachineLearning.Lowers + dnsFields["ml.uppers"] = dm.MachineLearning.Uppers + dnsFields["ml.specials"] = dm.MachineLearning.Specials + dnsFields["ml.others"] = dm.MachineLearning.Others + dnsFields["ml.ratio-digits"] = dm.MachineLearning.RatioDigits + dnsFields["ml.ratio-letters"] = dm.MachineLearning.RatioLetters + dnsFields["ml.ratio-specials"] = dm.MachineLearning.RatioSpecials + dnsFields["ml.ratio-others"] = dm.MachineLearning.RatioOthers + dnsFields["ml.consecutive-chars"] = dm.MachineLearning.ConsecutiveChars + dnsFields["ml.consecutive-vowels"] = dm.MachineLearning.ConsecutiveVowels + dnsFields["ml.consecutive-digits"] = dm.MachineLearning.ConsecutiveDigits + dnsFields["ml.consecutive-consonants"] = dm.MachineLearning.ConsecutiveConsonants + dnsFields["ml.size"] = dm.MachineLearning.Size + dnsFields["ml.occurrences"] = dm.MachineLearning.Occurrences + dnsFields["ml.uncommon-qtypes"] = dm.MachineLearning.UncommonQtypes + } + + // Add TransformATags fields + if dm.ATags != nil { + if len(dm.ATags.Tags) == 0 { + dnsFields["atags.tags"] = "-" + } + for i, tag := range dm.ATags.Tags { + dnsFields["atags.tags."+strconv.Itoa(i)] = tag + } + } + + // Add PowerDNS collectors fields + if dm.PowerDNS != nil { + if len(dm.PowerDNS.Tags) == 0 { + dnsFields["powerdns.tags"] = "-" + } + for i, tag := range dm.PowerDNS.Tags { + dnsFields["powerdns.tags."+strconv.Itoa(i)] = tag + } + dnsFields["powerdns.original-request-subnet"] = dm.PowerDNS.OriginalRequestSubnet + dnsFields["powerdns.applied-policy"] = dm.PowerDNS.AppliedPolicy + dnsFields["powerdns.applied-policy-hit"] = dm.PowerDNS.AppliedPolicyHit + dnsFields["powerdns.applied-policy-kind"] = dm.PowerDNS.AppliedPolicyKind + dnsFields["powerdns.applied-policy-trigger"] = dm.PowerDNS.AppliedPolicyTrigger + dnsFields["powerdns.applied-policy-type"] = dm.PowerDNS.AppliedPolicyType + for mk, mv := range dm.PowerDNS.Metadata { + dnsFields["powerdns.metadata."+mk] = mv + } + dnsFields["powerdns.http-version"] = dm.PowerDNS.HTTPVersion + } + + // relabeling ? + if dm.Relabeling != nil { + err := dm.ApplyRelabeling(dnsFields) + if err != nil { + return nil, err + } + } + + return dnsFields, nil +} diff --git a/dnsutils/dnsmessage_matchers.go b/dnsutils/dnsmessage_matching.go similarity index 88% rename from dnsutils/dnsmessage_matchers.go rename to dnsutils/dnsmessage_matching.go index a94e7803..c4d3ec2d 100644 --- a/dnsutils/dnsmessage_matchers.go +++ b/dnsutils/dnsmessage_matching.go @@ -8,6 +8,87 @@ import ( "strings" ) +func (dm *DNSMessage) Matching(matching map[string]interface{}) (error, bool) { + if len(matching) == 0 { + return nil, false + } + + dmValue := reflect.ValueOf(dm) + + if dmValue.Kind() == reflect.Ptr { + dmValue = dmValue.Elem() + } + + var isMatch = true + + for nestedKeys, value := range matching { + realValue, found := getFieldByJSONTag(dmValue, nestedKeys) + if !found { + return nil, false + } + + expectedValue := reflect.ValueOf(value) + switch expectedValue.Kind() { + // integer + case reflect.Int: + match, err := matchUserInteger(realValue, expectedValue) + if err != nil { + return err, false + } + if !match { + return nil, false + } + + // string + case reflect.String: + match, err := matchUserPattern(realValue, expectedValue) + if err != nil { + return err, false + } + if !match { + return nil, false + } + + // bool + case reflect.Bool: + match, err := matchUserBoolean(realValue, expectedValue) + if err != nil { + return err, false + } + if !match { + return nil, false + } + + // map + case reflect.Map: + match, err := matchUserMap(realValue, expectedValue) + if err != nil { + return err, false + } + if !match { + return nil, false + } + + // list/slice + case reflect.Slice: + match, err := matchUserSlice(realValue, expectedValue) + if err != nil { + return err, false + } + if !match { + return nil, false + } + + // other user types + default: + return fmt.Errorf("unsupported type value: %s", expectedValue.Kind()), false + } + + } + + return nil, isMatch +} + // matchUserMap matches a map based on user-provided conditions. // dns.qname: // match-source: "file://./tests/testsdata/filtering_keep_domains_regex.txt" diff --git a/dnsutils/dnsmessage_pcap.go b/dnsutils/dnsmessage_pcap.go new file mode 100644 index 00000000..d0630153 --- /dev/null +++ b/dnsutils/dnsmessage_pcap.go @@ -0,0 +1,121 @@ +package dnsutils + +import ( + "encoding/binary" + "errors" + "math" + "net" + + "github.com/dmachard/go-netutils" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" +) + +func (dm *DNSMessage) ToPacketLayer() ([]gopacket.SerializableLayer, error) { + if len(dm.DNS.Payload) == 0 { + return nil, errors.New("payload is empty") + } + + eth := &layers.Ethernet{ + SrcMAC: net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + DstMAC: net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}} + ip4 := &layers.IPv4{Version: 4, TTL: 64} + ip6 := &layers.IPv6{Version: 6} + udp := &layers.UDP{} + tcp := &layers.TCP{} + + // prepare ip + srcIP, srcPort, dstIP, dstPort := GetIPPort(dm) + if srcPort < 0 || srcPort > math.MaxUint16 { + return nil, errors.New("invalid source port value") + } + if dstPort < 0 || dstPort > math.MaxUint16 { + return nil, errors.New("invalid destination port value") + } + + // packet layer array + pkt := []gopacket.SerializableLayer{} + + // set source and destination IP + switch dm.NetworkInfo.Family { + case netutils.ProtoIPv4: + eth.EthernetType = layers.EthernetTypeIPv4 + ip4.SrcIP = net.ParseIP(srcIP) + ip4.DstIP = net.ParseIP(dstIP) + case netutils.ProtoIPv6: + eth.EthernetType = layers.EthernetTypeIPv6 + ip6.SrcIP = net.ParseIP(srcIP) + ip6.DstIP = net.ParseIP(dstIP) + default: + return nil, errors.New("family (" + dm.NetworkInfo.Family + ") not yet implemented") + } + + // set transport + switch dm.NetworkInfo.Protocol { + + // DNS over UDP + case netutils.ProtoUDP: + udp.SrcPort = layers.UDPPort(srcPort) + udp.DstPort = layers.UDPPort(dstPort) + + // update iplayer + switch dm.NetworkInfo.Family { + case netutils.ProtoIPv4: + ip4.Protocol = layers.IPProtocolUDP + udp.SetNetworkLayerForChecksum(ip4) + pkt = append(pkt, gopacket.Payload(dm.DNS.Payload), udp, ip4) + case netutils.ProtoIPv6: + ip6.NextHeader = layers.IPProtocolUDP + udp.SetNetworkLayerForChecksum(ip6) + pkt = append(pkt, gopacket.Payload(dm.DNS.Payload), udp, ip6) + } + + // DNS over TCP + case netutils.ProtoTCP: + tcp.SrcPort = layers.TCPPort(srcPort) + tcp.DstPort = layers.TCPPort(dstPort) + tcp.PSH = true + tcp.Window = 65535 + + // dns length + dnsLengthField := make([]byte, 2) + binary.BigEndian.PutUint16(dnsLengthField[0:], uint16(dm.DNS.Length)) + + // update iplayer + switch dm.NetworkInfo.Family { + case netutils.ProtoIPv4: + ip4.Protocol = layers.IPProtocolTCP + tcp.SetNetworkLayerForChecksum(ip4) + pkt = append(pkt, gopacket.Payload(append(dnsLengthField, dm.DNS.Payload...)), tcp, ip4) + case netutils.ProtoIPv6: + ip6.NextHeader = layers.IPProtocolTCP + tcp.SetNetworkLayerForChecksum(ip6) + pkt = append(pkt, gopacket.Payload(append(dnsLengthField, dm.DNS.Payload...)), tcp, ip6) + } + + // DNS over HTTPS and DNS over TLS + // These protocols are translated to DNS over UDP + case ProtoDoH, ProtoDoT: + udp.SrcPort = layers.UDPPort(srcPort) + udp.DstPort = layers.UDPPort(dstPort) + + // update iplayer + switch dm.NetworkInfo.Family { + case netutils.ProtoIPv4: + ip4.Protocol = layers.IPProtocolUDP + udp.SetNetworkLayerForChecksum(ip4) + pkt = append(pkt, gopacket.Payload(dm.DNS.Payload), udp, ip4) + case netutils.ProtoIPv6: + ip6.NextHeader = layers.IPProtocolUDP + udp.SetNetworkLayerForChecksum(ip6) + pkt = append(pkt, gopacket.Payload(dm.DNS.Payload), udp, ip6) + } + + default: + return nil, errors.New("protocol " + dm.NetworkInfo.Protocol + " not yet implemented") + } + + pkt = append(pkt, eth) + + return pkt, nil +} diff --git a/dnsutils/dnsmessage_relabelling.go b/dnsutils/dnsmessage_relabelling.go new file mode 100644 index 00000000..2e9ab235 --- /dev/null +++ b/dnsutils/dnsmessage_relabelling.go @@ -0,0 +1,30 @@ +package dnsutils + +func (dm *DNSMessage) ApplyRelabeling(dnsFields map[string]interface{}) error { + + for _, label := range dm.Relabeling.Rules { + regex := label.Regex + for key := range dnsFields { + if regex.MatchString(key) { + if label.Action == "rename" { + replacement := label.Replacement + if value, exists := dnsFields[replacement]; exists { + switch v := value.(type) { + case []string: + dnsFields[replacement] = append(v, ConvertToString(dnsFields[key])) + default: + dnsFields[replacement] = []string{ConvertToString(v), ConvertToString(dnsFields[key])} + } + } else { + dnsFields[replacement] = ConvertToString(dnsFields[key]) + } + } + + // delete on all case + delete(dnsFields, key) + } + } + } + + return nil +} diff --git a/dnsutils/dnsmessage_template.go b/dnsutils/dnsmessage_template.go new file mode 100644 index 00000000..b82327dd --- /dev/null +++ b/dnsutils/dnsmessage_template.go @@ -0,0 +1,19 @@ +package dnsutils + +import "github.com/flosch/pongo2" + +func (dm *DNSMessage) ToTextTemplate(template string) (string, error) { + context := pongo2.Context{"dm": dm} + + // Parse and execute the template + tmpl, err := pongo2.FromString(template) + if err != nil { + return "", err + } + + result, err := tmpl.Execute(context) + if err != nil { + return "", err + } + return result, nil +} diff --git a/dnsutils/dnsmessage_text.go b/dnsutils/dnsmessage_text.go new file mode 100644 index 00000000..69194eb8 --- /dev/null +++ b/dnsutils/dnsmessage_text.go @@ -0,0 +1,564 @@ +package dnsutils + +import ( + "encoding/base64" + "errors" + "fmt" + "log" + "strconv" + "strings" + "time" +) + +func (dm *DNSMessage) handleGeoIPDirectives(directive string, s *strings.Builder) error { + if dm.Geo == nil { + s.WriteString("-") + } else { + switch { + case directive == "geoip-continent": + s.WriteString(dm.Geo.Continent) + case directive == "geoip-country": + s.WriteString(dm.Geo.CountryIsoCode) + case directive == "geoip-city": + s.WriteString(dm.Geo.City) + case directive == "geoip-as-number": + s.WriteString(dm.Geo.AutonomousSystemNumber) + case directive == "geoip-as-owner": + s.WriteString(dm.Geo.AutonomousSystemOrg) + default: + return errors.New(ErrorUnexpectedDirective + directive) + } + } + return nil +} + +func (dm *DNSMessage) handlePdnsDirectives(directive string, s *strings.Builder) error { + if dm.PowerDNS == nil { + s.WriteString("-") + } else { + var directives []string + if i := strings.IndexByte(directive, ':'); i == -1 { + directives = append(directives, directive) + } else { + directives = []string{directive[:i], directive[i+1:]} + } + + switch directive := directives[0]; { + case directive == "powerdns-tags": + if dm.PowerDNS.Tags == nil { + s.WriteString("-") + } else { + if len(dm.PowerDNS.Tags) > 0 { + if len(directives) == 2 { + tagIndex, err := strconv.Atoi(directives[1]) + if err != nil { + log.Fatalf("unsupport tag index provided (integer expected): %s", directives[1]) + } + if tagIndex >= len(dm.PowerDNS.Tags) { + s.WriteString("-") + } else { + s.WriteString(dm.PowerDNS.Tags[tagIndex]) + } + } else { + for i, tag := range dm.PowerDNS.Tags { + s.WriteString(tag) + // add separator + if i+1 < len(dm.PowerDNS.Tags) { + s.WriteString(",") + } + } + } + } else { + s.WriteString("-") + } + } + case directive == "powerdns-applied-policy": + if len(dm.PowerDNS.AppliedPolicy) > 0 { + s.WriteString(dm.PowerDNS.AppliedPolicy) + } else { + s.WriteString("-") + } + case directive == "powerdns-applied-policy-hit": + if len(dm.PowerDNS.AppliedPolicyHit) > 0 { + s.WriteString(dm.PowerDNS.AppliedPolicyHit) + } else { + s.WriteString("-") + } + case directive == "powerdns-applied-policy-kind": + if len(dm.PowerDNS.AppliedPolicyKind) > 0 { + s.WriteString(dm.PowerDNS.AppliedPolicyKind) + } else { + s.WriteString("-") + } + case directive == "powerdns-applied-policy-trigger": + if len(dm.PowerDNS.AppliedPolicyTrigger) > 0 { + s.WriteString(dm.PowerDNS.AppliedPolicyTrigger) + } else { + s.WriteString("-") + } + case directive == "powerdns-applied-policy-type": + if len(dm.PowerDNS.AppliedPolicyType) > 0 { + s.WriteString(dm.PowerDNS.AppliedPolicyType) + } else { + s.WriteString("-") + } + case directive == "powerdns-original-request-subnet": + if len(dm.PowerDNS.OriginalRequestSubnet) > 0 { + s.WriteString(dm.PowerDNS.OriginalRequestSubnet) + } else { + s.WriteString("-") + } + case directive == "powerdns-metadata": + if dm.PowerDNS.Metadata == nil { + s.WriteString("-") + } else { + if len(dm.PowerDNS.Metadata) > 0 && len(directives) == 2 { + if metaValue, ok := dm.PowerDNS.Metadata[directives[1]]; ok { + if len(metaValue) > 0 { + s.WriteString(strings.ReplaceAll(metaValue, " ", "_")) + } else { + s.WriteString("-") + } + } else { + s.WriteString("-") + } + } else { + s.WriteString("-") + } + } + case directive == "powerdns-http-version": + if len(dm.PowerDNS.HTTPVersion) > 0 { + s.WriteString(dm.PowerDNS.HTTPVersion) + } else { + s.WriteString("-") + } + default: + return errors.New(ErrorUnexpectedDirective + directive) + } + } + return nil +} + +func (dm *DNSMessage) handleATagsDirectives(directive string, s *strings.Builder) error { + if dm.ATags == nil { + s.WriteString("-") + } else { + var directives []string + if i := strings.IndexByte(directive, ':'); i == -1 { + directives = append(directives, directive) + } else { + directives = []string{directive[:i], directive[i+1:]} + } + + switch directive := directives[0]; { + case directive == "atags": + if len(dm.ATags.Tags) > 0 { + if len(directives) == 2 { + tagIndex, err := strconv.Atoi(directives[1]) + if err != nil { + log.Fatalf("unsupport tag index provided (integer expected): %s", directives[1]) + } + if tagIndex >= len(dm.ATags.Tags) { + s.WriteString("-") + } else { + s.WriteString(dm.ATags.Tags[tagIndex]) + } + } else { + for i, tag := range dm.ATags.Tags { + s.WriteString(tag) + // add separator + if i+1 < len(dm.ATags.Tags) { + s.WriteString(",") + } + } + } + } else { + s.WriteString("-") + } + default: + return errors.New(ErrorUnexpectedDirective + directive) + } + } + return nil +} + +func (dm *DNSMessage) handleSuspiciousDirectives(directive string, s *strings.Builder) error { + if dm.Suspicious == nil { + s.WriteString("-") + } else { + switch { + case directive == "suspicious-score": + s.WriteString(strconv.Itoa(int(dm.Suspicious.Score))) + default: + return errors.New(ErrorUnexpectedDirective + directive) + } + } + return nil +} + +func (dm *DNSMessage) handlePublicSuffixDirectives(directive string, s *strings.Builder) error { + if dm.PublicSuffix == nil { + s.WriteString("-") + } else { + switch { + case directive == "publixsuffix-tld": + s.WriteString(dm.PublicSuffix.QnamePublicSuffix) + case directive == "publixsuffix-etld+1": + s.WriteString(dm.PublicSuffix.QnameEffectiveTLDPlusOne) + case directive == "publixsuffix-managed-icann": + if dm.PublicSuffix.ManagedByICANN { + s.WriteString("managed") + } else { + s.WriteString("private") + } + default: + return errors.New(ErrorUnexpectedDirective + directive) + } + } + return nil +} + +func (dm *DNSMessage) handleExtractedDirectives(directive string, s *strings.Builder) error { + if dm.Extracted == nil { + s.WriteString("-") + return nil + } + switch { + case directive == "extracted-dns-payload": + if len(dm.DNS.Payload) > 0 { + dst := make([]byte, base64.StdEncoding.EncodedLen(len(dm.DNS.Payload))) + base64.StdEncoding.Encode(dst, dm.DNS.Payload) + s.Write(dst) + } else { + s.WriteString("-") + } + default: + return errors.New(ErrorUnexpectedDirective + directive) + } + return nil +} + +func (dm *DNSMessage) handleFilteringDirectives(directive string, s *strings.Builder) error { + if dm.Filtering == nil { + s.WriteString("-") + } else { + switch { + case directive == "filtering-sample-rate": + s.WriteString(strconv.Itoa(dm.Filtering.SampleRate)) + default: + return errors.New(ErrorUnexpectedDirective + directive) + } + } + return nil +} + +func (dm *DNSMessage) handleReducerDirectives(directive string, s *strings.Builder) error { + if dm.Reducer == nil { + s.WriteString("-") + } else { + switch { + case directive == "reducer-occurrences": + s.WriteString(strconv.Itoa(dm.Reducer.Occurrences)) + case directive == "reducer-cumulative-length": + s.WriteString(strconv.Itoa(dm.Reducer.CumulativeLength)) + default: + return errors.New(ErrorUnexpectedDirective + directive) + } + } + return nil +} + +func (dm *DNSMessage) handleMachineLearningDirectives(directive string, s *strings.Builder) error { + if dm.MachineLearning == nil { + s.WriteString("-") + } else { + switch { + case directive == "ml-entropy": + s.WriteString(strconv.FormatFloat(dm.MachineLearning.Entropy, 'f', -1, 64)) + case directive == "ml-length": + s.WriteString(strconv.Itoa(dm.MachineLearning.Length)) + case directive == "ml-digits": + s.WriteString(strconv.Itoa(dm.MachineLearning.Digits)) + case directive == "ml-lowers": + s.WriteString(strconv.Itoa(dm.MachineLearning.Lowers)) + case directive == "ml-uppers": + s.WriteString(strconv.Itoa(dm.MachineLearning.Uppers)) + case directive == "ml-specials": + s.WriteString(strconv.Itoa(dm.MachineLearning.Specials)) + case directive == "ml-others": + s.WriteString(strconv.Itoa(dm.MachineLearning.Others)) + case directive == "ml-labels": + s.WriteString(strconv.Itoa(dm.MachineLearning.Labels)) + case directive == "ml-ratio-digits": + s.WriteString(strconv.FormatFloat(dm.MachineLearning.RatioDigits, 'f', 3, 64)) + case directive == "ml-ratio-letters": + s.WriteString(strconv.FormatFloat(dm.MachineLearning.RatioLetters, 'f', 3, 64)) + case directive == "ml-ratio-specials": + s.WriteString(strconv.FormatFloat(dm.MachineLearning.RatioSpecials, 'f', 3, 64)) + case directive == "ml-ratio-others": + s.WriteString(strconv.FormatFloat(dm.MachineLearning.RatioOthers, 'f', 3, 64)) + case directive == "ml-consecutive-chars": + s.WriteString(strconv.Itoa(dm.MachineLearning.ConsecutiveChars)) + case directive == "ml-consecutive-vowels": + s.WriteString(strconv.Itoa(dm.MachineLearning.ConsecutiveVowels)) + case directive == "ml-consecutive-digits": + s.WriteString(strconv.Itoa(dm.MachineLearning.ConsecutiveDigits)) + case directive == "ml-consecutive-consonants": + s.WriteString(strconv.Itoa(dm.MachineLearning.ConsecutiveConsonants)) + case directive == "ml-size": + s.WriteString(strconv.Itoa(dm.MachineLearning.Size)) + case directive == "ml-occurrences": + s.WriteString(strconv.Itoa(dm.MachineLearning.Occurrences)) + case directive == "ml-uncommon-qtypes": + s.WriteString(strconv.Itoa(dm.MachineLearning.UncommonQtypes)) + default: + return errors.New(ErrorUnexpectedDirective + directive) + } + } + return nil +} + +func (dm *DNSMessage) Bytes(format []string, fieldDelimiter string, fieldBoundary string) []byte { + line, err := dm.ToTextLine(format, fieldDelimiter, fieldBoundary) + if err != nil { + log.Fatalf("unsupport directive for text format: %s", err) + } + return line +} + +func (dm *DNSMessage) String(format []string, fieldDelimiter string, fieldBoundary string) string { + return string(dm.Bytes(format, fieldDelimiter, fieldBoundary)) +} + +func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBoundary string) ([]byte, error) { + var s strings.Builder + + an := dm.DNS.DNSRRs.Answers + qname := dm.DNS.Qname + flags := dm.DNS.Flags + + for i, directive := range format { + switch { + case directive == "timestamp-rfc3339ns", directive == "timestamp": + s.WriteString(dm.DNSTap.TimestampRFC3339) + case directive == "timestamp-unixms": + s.WriteString(fmt.Sprintf("%d", dm.DNSTap.Timestamp/1000000)) + case directive == "timestamp-unixus": + s.WriteString(fmt.Sprintf("%d", dm.DNSTap.Timestamp/1000)) + case directive == "timestamp-unixns": + s.WriteString(fmt.Sprintf("%d", dm.DNSTap.Timestamp)) + case directive == "localtime": + ts := time.Unix(int64(dm.DNSTap.TimeSec), int64(dm.DNSTap.TimeNsec)) + s.WriteString(ts.Format("2006-01-02 15:04:05.999999999")) + case directive == "qname": + if len(qname) == 0 { + s.WriteString(".") + } else { + QuoteStringAndWrite(&s, qname, fieldDelimiter, fieldBoundary) + } + case directive == "identity": + if len(dm.DNSTap.Identity) == 0 { + s.WriteString("-") + } else { + QuoteStringAndWrite(&s, dm.DNSTap.Identity, fieldDelimiter, fieldBoundary) + } + case directive == "peer-name": + if len(dm.DNSTap.PeerName) == 0 { + s.WriteString("-") + } else { + QuoteStringAndWrite(&s, dm.DNSTap.PeerName, fieldDelimiter, fieldBoundary) + } + case directive == "version": + if len(dm.DNSTap.Version) == 0 { + s.WriteString("-") + } else { + QuoteStringAndWrite(&s, dm.DNSTap.Version, fieldDelimiter, fieldBoundary) + } + case directive == "extra": + s.WriteString(dm.DNSTap.Extra) + case directive == "policy-rule": + s.WriteString(dm.DNSTap.PolicyRule) + case directive == "policy-type": + s.WriteString(dm.DNSTap.PolicyType) + case directive == "policy-action": + s.WriteString(dm.DNSTap.PolicyAction) + case directive == "policy-match": + s.WriteString(dm.DNSTap.PolicyMatch) + case directive == "policy-value": + s.WriteString(dm.DNSTap.PolicyValue) + case directive == "query-zone": + s.WriteString(dm.DNSTap.QueryZone) + case directive == "operation": + s.WriteString(dm.DNSTap.Operation) + case directive == "rcode": + s.WriteString(dm.DNS.Rcode) + case directive == "id": + s.WriteString(strconv.Itoa(dm.DNS.ID)) + case directive == "queryip": + s.WriteString(dm.NetworkInfo.QueryIP) + case directive == "queryport": + s.WriteString(dm.NetworkInfo.QueryPort) + case directive == "responseip": + s.WriteString(dm.NetworkInfo.ResponseIP) + case directive == "responseport": + s.WriteString(dm.NetworkInfo.ResponsePort) + case directive == "family": + s.WriteString(dm.NetworkInfo.Family) + case directive == "protocol": + s.WriteString(dm.NetworkInfo.Protocol) + case directive == "length-unit": + s.WriteString(strconv.Itoa(dm.DNS.Length) + "b") + case directive == "length": + s.WriteString(strconv.Itoa(dm.DNS.Length)) + case directive == "qtype": + s.WriteString(dm.DNS.Qtype) + case directive == "qclass": + s.WriteString(dm.DNS.Qclass) + case directive == "latency": + s.WriteString(fmt.Sprintf("%.9f", dm.DNSTap.Latency)) + case directive == "malformed": + if dm.DNS.MalformedPacket { + s.WriteString("PKTERR") + } else { + s.WriteByte('-') + } + case directive == "qr": + s.WriteString(dm.DNS.Type) + case directive == "opcode": + s.WriteString(strconv.Itoa(dm.DNS.Opcode)) + case directive == "tr": + if dm.NetworkInfo.TCPReassembled { + s.WriteString("TR") + } else { + s.WriteByte('-') + } + case directive == "df": + if dm.NetworkInfo.IPDefragmented { + s.WriteString("DF") + } else { + s.WriteByte('-') + } + case directive == "tc": + if flags.TC { + s.WriteString("TC") + } else { + s.WriteByte('-') + } + case directive == "aa": + if flags.AA { + s.WriteString("AA") + } else { + s.WriteByte('-') + } + case directive == "ra": + if flags.RA { + s.WriteString("RA") + } else { + s.WriteByte('-') + } + case directive == "ad": + if flags.AD { + s.WriteString("AD") + } else { + s.WriteByte('-') + } + case directive == "ttl": + if len(an) > 0 { + s.WriteString(strconv.Itoa(an[0].TTL)) + } else { + s.WriteByte('-') + } + case directive == "answer": + if len(an) > 0 { + s.WriteString(an[0].Rdata) + } else { + s.WriteByte('-') + } + + case directive == "questionscount" || directive == "qdcount": + s.WriteString(strconv.Itoa(dm.DNS.QdCount)) + case directive == "answercount" || directive == "ancount": + s.WriteString(strconv.Itoa(dm.DNS.AnCount)) + case directive == "nscount": + s.WriteString(strconv.Itoa(dm.DNS.NsCount)) + case directive == "arcount": + s.WriteString(strconv.Itoa(dm.DNS.ArCount)) + + case directive == "edns-csubnet": + if len(dm.EDNS.Options) > 0 { + for _, opt := range dm.EDNS.Options { + if opt.Name == "CSUBNET" { + s.WriteString(opt.Data) + break + } + } + } else { + s.WriteByte('-') + } + + // more directives from collectors + case PdnsDirectives.MatchString(directive): + err := dm.handlePdnsDirectives(directive, &s) + if err != nil { + return nil, err + } + + // more directives from transformers + case ReducerDirectives.MatchString(directive): + err := dm.handleReducerDirectives(directive, &s) + if err != nil { + return nil, err + } + case GeoIPDirectives.MatchString(directive): + err := dm.handleGeoIPDirectives(directive, &s) + if err != nil { + return nil, err + } + case SuspiciousDirectives.MatchString(directive): + err := dm.handleSuspiciousDirectives(directive, &s) + if err != nil { + return nil, err + } + case PublicSuffixDirectives.MatchString(directive): + err := dm.handlePublicSuffixDirectives(directive, &s) + if err != nil { + return nil, err + } + case ExtractedDirectives.MatchString(directive): + err := dm.handleExtractedDirectives(directive, &s) + if err != nil { + return nil, err + } + case MachineLearningDirectives.MatchString(directive): + err := dm.handleMachineLearningDirectives(directive, &s) + if err != nil { + return nil, err + } + case FilteringDirectives.MatchString(directive): + err := dm.handleFilteringDirectives(directive, &s) + if err != nil { + return nil, err + } + case ATagsDirectives.MatchString(directive): + err := dm.handleATagsDirectives(directive, &s) + if err != nil { + return nil, err + } + case RawTextDirective.MatchString(directive): + directive = strings.ReplaceAll(directive, "{", "") + directive = strings.ReplaceAll(directive, "}", "") + s.WriteString(directive) + + // handle invalid directive + default: + return nil, errors.New(ErrorUnexpectedDirective + directive) + } + + if i < len(format)-1 { + if len(fieldDelimiter) > 0 { + s.WriteString(fieldDelimiter) + } + } + } + return []byte(s.String()), nil +} diff --git a/docs/advanced_config.md b/docs/advanced_config.md index 45fcb0fb..9ea8d049 100644 --- a/docs/advanced_config.md +++ b/docs/advanced_config.md @@ -6,6 +6,7 @@ - [Server identity](#server-identity) - [Pid file](#pid-file) - [Telemetry](#telemetry) + - [Default text format](#default-text-format) - [Configuration reloading](#configuration-reloading) ## Configuration checks @@ -114,6 +115,17 @@ global: ``` +### Default text format + +These settings can be used to set the text format for all loggers. + +```yaml +text-format: "timestamp-rfc3339ns identity operation rcode queryip queryport family protocol length-unit qname qtype latency" +text-format-delimiter: " " +text-format-boundary: "\"" +text-jinja: "" +``` + ## Configuration reloading DNS-collector automatically reloads its configuration upon receiving a SIGHUP signal, allowing you to update settings without restarting the service. diff --git a/docs/dnsconversions.md b/docs/dnsconversions.md index 85c2a983..a776ca13 100644 --- a/docs/dnsconversions.md +++ b/docs/dnsconversions.md @@ -7,7 +7,7 @@ All DNStap messages or network-captured data are first transformed into the [DNS - [Text Format Inline](#text-format-inline) - [JSON Advanced Structure](#) - [Flat JSON (Key/Value)](#flat-json-format) -- [Jinja templating](#jinja-inline) +- [Jinja templating](#jinja-templating) ## Text format inline @@ -91,7 +91,7 @@ global: ``` -## Jinja inline +## Jinja templating For a more flexible format, you can use the `text-jinja` setting. The DNS to Jinja encoding is very powerful but slow, so use it only under specific conditions. Jinja templates are available with: @@ -101,17 +101,15 @@ The DNS to Jinja encoding is very powerful but slow, so use it only under specif **Default directives** All directives are in https://pkg.go.dev/github.com/dmachard/go-dnscollector/dnsutils#DNSMessage +The template can be defined in the [global settings](advanced_config.md#default-text-format) or on loggers. -To use the jinja, add the folowwing code in your text format +To use the jinja, add for example the folowwing code in your text format ```jinja -{{ dm.DNS.Opcode }} +global: + text-jinja: "OPCODE = {{ dm.DNS.Opcode }} QUERYIP = {{ dm.NetworkInfo.QueryIP }}" ``` - -**Full configuration examples** - -* [`Dig style output`](../_examples/use-case-27.yml) - +> A full example to get [`Dig style output`](./_examples/use-case-27.yml) ## JSON encoding diff --git a/go.mod b/go.mod index 881b7dff..d401bb58 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/dmachard/go-powerdns-protobuf v1.3.0 github.com/dmachard/go-topmap v1.0.2 github.com/farsightsec/golang-framestream v0.3.0 - github.com/flosch/pongo2/v6 v6.0.0 + github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3 github.com/fsnotify/fsnotify v1.7.0 github.com/gogo/protobuf v1.3.2 github.com/golang/snappy v0.0.4 diff --git a/go.sum b/go.sum index 52e0366d..3ad1bf7a 100644 --- a/go.sum +++ b/go.sum @@ -143,8 +143,8 @@ github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flosch/pongo2/v6 v6.0.0 h1:lsGru8IAzHgIAw6H2m4PCyleO58I40ow6apih0WprMU= -github.com/flosch/pongo2/v6 v6.0.0/go.mod h1:CuDpFm47R0uGGE7z13/tTlt1Y6zdxvr2RLT5LJhsHEU= +github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3 h1:fmFk0Wt3bBxxwZnu48jqMdaOR/IZ4vdtJFuaFV8MpIE= +github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3/go.mod h1:bJWSKrZyQvfTnb2OudyUjurSG4/edverV7n82+K3JiM= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= @@ -812,6 +812,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=