From 50128c68cf5950d1149c3bea2a211f9a499fa395 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Wed, 17 Jan 2024 17:25:00 +0100 Subject: [PATCH 1/9] support extended dnstap --- .gitignore | 2 + config.yml | 28 ++- dnsutils/extended_dnstap.pb.go | 315 ++++++++++++++++++++++++++++ dnsutils/extended_dnstap.proto | 21 ++ dnsutils/message.go | 26 ++- dnsutils/message_test.go | 37 +++- docs/collectors/collector_dnstap.md | 5 + docs/loggers/logger_dnstap.md | 2 + loggers/dnstapclient.go | 2 +- loggers/logfile.go | 2 +- pkgconfig/collectors.go | 2 + pkgconfig/loggers.go | 4 + processors/dnstap.go | 30 ++- 13 files changed, 464 insertions(+), 12 deletions(-) create mode 100644 dnsutils/extended_dnstap.pb.go create mode 100644 dnsutils/extended_dnstap.proto diff --git a/.gitignore b/.gitignore index d4e7687d..6cbf1513 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ __pycache__/ go-dnscollector +bin/ +include/ \ No newline at end of file diff --git a/config.yml b/config.yml index fbe356e9..0785cf31 100644 --- a/config.yml +++ b/config.yml @@ -91,6 +91,14 @@ multiplexer: # routing-policy: # default: [ filter ] +# - name: second-input +# dnstap: +# listen-ip: 0.0.0.0 +# listen-port: 6002 +# extended-extra-field: true +# routing-policy: +# default: [ console ] + # - name: filter # dnsmessage: # matching: @@ -101,18 +109,26 @@ multiplexer: # tags: [ "google"] # routing-policy: # dropped: [ outputfile ] -# default: [ console ] +# default: [ central ] + +# - name: central +# dnstapclient: +# transport: tcp +# remote-address: 127.0.0.1 +# remote-port: 6002 +# flush-interval: 5 +# extended-extra-field: true # - name: console # stdout: -# mode: text +# mode: flat-json # - name: outputfile # logfile: # file-path: "/tmp/dnstap.log" # max-size: 1000 # max-files: 10 -# mode: flat-json +# mode: text ################################################ # list of supported collectors @@ -142,9 +158,11 @@ multiplexer: # chan-buffer-size: 65535 # # Disable the minimalist DNS parser # disable-dnsparser: true +# # Decode the extended extra field sent by DNScollector +# extended-support: false # # dnstap proxifier with no protobuf decoding. -# dnstap-proxifier: +# dnstap-relay: # # listen on ip # listen-ip: 0.0.0.0 # # listening on port @@ -407,6 +425,8 @@ multiplexer: # buffer-size: 100 # # Channel buffer size for incoming packets, number of packet before to drop it. # chan-buffer-size: 65535 +# # Extend the DNStap message by incorporating additional transformations, such as filtering and ATags, into the extra field. +# extended-support: false # # resend captured dns traffic to a tcp remote destination or to unix socket # tcpclient: diff --git a/dnsutils/extended_dnstap.pb.go b/dnsutils/extended_dnstap.pb.go new file mode 100644 index 00000000..5bf2d812 --- /dev/null +++ b/dnsutils/extended_dnstap.pb.go @@ -0,0 +1,315 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.32.0 +// protoc v4.25.2 +// source: extended_dnstap.proto + +package dnsutils + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ExtendedATags struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Tags []string `protobuf:"bytes,1,rep,name=tags,proto3" json:"tags,omitempty"` +} + +func (x *ExtendedATags) Reset() { + *x = ExtendedATags{} + if protoimpl.UnsafeEnabled { + mi := &file_extended_dnstap_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ExtendedATags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExtendedATags) ProtoMessage() {} + +func (x *ExtendedATags) ProtoReflect() protoreflect.Message { + mi := &file_extended_dnstap_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExtendedATags.ProtoReflect.Descriptor instead. +func (*ExtendedATags) Descriptor() ([]byte, []int) { + return file_extended_dnstap_proto_rawDescGZIP(), []int{0} +} + +func (x *ExtendedATags) GetTags() []string { + if x != nil { + return x.Tags + } + return nil +} + +type ExtendedPublicSuffix struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Tld string `protobuf:"bytes,1,opt,name=tld,proto3" json:"tld,omitempty"` + EtldPlusOne string `protobuf:"bytes,2,opt,name=etld_plus_one,json=etldPlusOne,proto3" json:"etld_plus_one,omitempty"` +} + +func (x *ExtendedPublicSuffix) Reset() { + *x = ExtendedPublicSuffix{} + if protoimpl.UnsafeEnabled { + mi := &file_extended_dnstap_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ExtendedPublicSuffix) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExtendedPublicSuffix) ProtoMessage() {} + +func (x *ExtendedPublicSuffix) ProtoReflect() protoreflect.Message { + mi := &file_extended_dnstap_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExtendedPublicSuffix.ProtoReflect.Descriptor instead. +func (*ExtendedPublicSuffix) Descriptor() ([]byte, []int) { + return file_extended_dnstap_proto_rawDescGZIP(), []int{1} +} + +func (x *ExtendedPublicSuffix) GetTld() string { + if x != nil { + return x.Tld + } + return "" +} + +func (x *ExtendedPublicSuffix) GetEtldPlusOne() string { + if x != nil { + return x.EtldPlusOne + } + return "" +} + +type ExtendedDnstap struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + OriginalDnstapExtra []byte `protobuf:"bytes,2,opt,name=original_dnstap_extra,json=originalDnstapExtra,proto3" json:"original_dnstap_extra,omitempty"` + Atags *ExtendedATags `protobuf:"bytes,3,opt,name=atags,proto3" json:"atags,omitempty"` + PublicSuffix *ExtendedPublicSuffix `protobuf:"bytes,4,opt,name=public_suffix,json=publicSuffix,proto3" json:"public_suffix,omitempty"` +} + +func (x *ExtendedDnstap) Reset() { + *x = ExtendedDnstap{} + if protoimpl.UnsafeEnabled { + mi := &file_extended_dnstap_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ExtendedDnstap) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExtendedDnstap) ProtoMessage() {} + +func (x *ExtendedDnstap) ProtoReflect() protoreflect.Message { + mi := &file_extended_dnstap_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExtendedDnstap.ProtoReflect.Descriptor instead. +func (*ExtendedDnstap) Descriptor() ([]byte, []int) { + return file_extended_dnstap_proto_rawDescGZIP(), []int{2} +} + +func (x *ExtendedDnstap) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *ExtendedDnstap) GetOriginalDnstapExtra() []byte { + if x != nil { + return x.OriginalDnstapExtra + } + return nil +} + +func (x *ExtendedDnstap) GetAtags() *ExtendedATags { + if x != nil { + return x.Atags + } + return nil +} + +func (x *ExtendedDnstap) GetPublicSuffix() *ExtendedPublicSuffix { + if x != nil { + return x.PublicSuffix + } + return nil +} + +var File_extended_dnstap_proto protoreflect.FileDescriptor + +var file_extended_dnstap_proto_rawDesc = []byte{ + 0x0a, 0x15, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x5f, 0x64, 0x6e, 0x73, 0x74, 0x61, + 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x23, 0x0a, 0x0d, 0x45, 0x78, 0x74, 0x65, 0x6e, + 0x64, 0x65, 0x64, 0x41, 0x54, 0x61, 0x67, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x4c, 0x0a, 0x14, + 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x53, 0x75, + 0x66, 0x66, 0x69, 0x78, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x74, 0x6c, 0x64, 0x12, 0x22, 0x0a, 0x0d, 0x65, 0x74, 0x6c, 0x64, 0x5f, 0x70, + 0x6c, 0x75, 0x73, 0x5f, 0x6f, 0x6e, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x65, + 0x74, 0x6c, 0x64, 0x50, 0x6c, 0x75, 0x73, 0x4f, 0x6e, 0x65, 0x22, 0xc0, 0x01, 0x0a, 0x0e, 0x45, + 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x44, 0x6e, 0x73, 0x74, 0x61, 0x70, 0x12, 0x18, 0x0a, + 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x15, 0x6f, 0x72, 0x69, 0x67, 0x69, + 0x6e, 0x61, 0x6c, 0x5f, 0x64, 0x6e, 0x73, 0x74, 0x61, 0x70, 0x5f, 0x65, 0x78, 0x74, 0x72, 0x61, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x13, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, + 0x44, 0x6e, 0x73, 0x74, 0x61, 0x70, 0x45, 0x78, 0x74, 0x72, 0x61, 0x12, 0x24, 0x0a, 0x05, 0x61, + 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x45, 0x78, 0x74, + 0x65, 0x6e, 0x64, 0x65, 0x64, 0x41, 0x54, 0x61, 0x67, 0x73, 0x52, 0x05, 0x61, 0x74, 0x61, 0x67, + 0x73, 0x12, 0x3a, 0x0a, 0x0d, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x73, 0x75, 0x66, 0x66, + 0x69, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x6e, + 0x64, 0x65, 0x64, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x52, + 0x0c, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x42, 0x2e, 0x5a, + 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x6d, 0x61, 0x63, + 0x68, 0x61, 0x72, 0x64, 0x2f, 0x67, 0x6f, 0x2d, 0x64, 0x6e, 0x73, 0x63, 0x6f, 0x6c, 0x6c, 0x65, + 0x63, 0x74, 0x6f, 0x72, 0x3b, 0x64, 0x6e, 0x73, 0x75, 0x74, 0x69, 0x6c, 0x73, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_extended_dnstap_proto_rawDescOnce sync.Once + file_extended_dnstap_proto_rawDescData = file_extended_dnstap_proto_rawDesc +) + +func file_extended_dnstap_proto_rawDescGZIP() []byte { + file_extended_dnstap_proto_rawDescOnce.Do(func() { + file_extended_dnstap_proto_rawDescData = protoimpl.X.CompressGZIP(file_extended_dnstap_proto_rawDescData) + }) + return file_extended_dnstap_proto_rawDescData +} + +var file_extended_dnstap_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_extended_dnstap_proto_goTypes = []interface{}{ + (*ExtendedATags)(nil), // 0: ExtendedATags + (*ExtendedPublicSuffix)(nil), // 1: ExtendedPublicSuffix + (*ExtendedDnstap)(nil), // 2: ExtendedDnstap +} +var file_extended_dnstap_proto_depIdxs = []int32{ + 0, // 0: ExtendedDnstap.atags:type_name -> ExtendedATags + 1, // 1: ExtendedDnstap.public_suffix:type_name -> ExtendedPublicSuffix + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_extended_dnstap_proto_init() } +func file_extended_dnstap_proto_init() { + if File_extended_dnstap_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_extended_dnstap_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ExtendedATags); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_extended_dnstap_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ExtendedPublicSuffix); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_extended_dnstap_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ExtendedDnstap); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_extended_dnstap_proto_rawDesc, + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_extended_dnstap_proto_goTypes, + DependencyIndexes: file_extended_dnstap_proto_depIdxs, + MessageInfos: file_extended_dnstap_proto_msgTypes, + }.Build() + File_extended_dnstap_proto = out.File + file_extended_dnstap_proto_rawDesc = nil + file_extended_dnstap_proto_goTypes = nil + file_extended_dnstap_proto_depIdxs = nil +} diff --git a/dnsutils/extended_dnstap.proto b/dnsutils/extended_dnstap.proto new file mode 100644 index 00000000..ccf8868d --- /dev/null +++ b/dnsutils/extended_dnstap.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +// ../bin/protoc --proto_path=. --go_out=. --go_opt=paths=source_relative --plugin protoc-gen-go=${GOBIN}/protoc-gen-go extended_dnstap.proto + +option go_package = "github.com/dmachard/go-dnscollector;dnsutils"; + +message ExtendedATags { + repeated string tags = 1; +} + +message ExtendedPublicSuffix { + string tld = 1; + string etld_plus_one = 2; +} + +message ExtendedDnstap { + string version = 1; + bytes original_dnstap_extra = 2; + ExtendedATags atags = 3; + ExtendedPublicSuffix public_suffix = 4; +} \ No newline at end of file diff --git a/dnsutils/message.go b/dnsutils/message.go index 41d483c0..b5ed549e 100644 --- a/dnsutils/message.go +++ b/dnsutils/message.go @@ -664,7 +664,7 @@ func (dm *DNSMessage) ToFlatJSON() (string, error) { return buffer.String(), nil } -func (dm *DNSMessage) ToDNSTap() ([]byte, error) { +func (dm *DNSMessage) ToDNSTap(extended bool) ([]byte, error) { if len(dm.DNSTap.Payload) > 0 { return dm.DNSTap.Payload, nil } @@ -741,6 +741,30 @@ func (dm *DNSMessage) ToDNSTap() ([]byte, error) { dt.Extra = []byte(dm.DNSTap.Extra) } + // contruct new dnstap field with all tranformations + // the original extra field is kept if exist + if extended { + ednstap := &ExtendedDnstap{} + + // add original dnstap value if exist + if len(dm.DNSTap.Extra) > 0 { + ednstap.OriginalDnstapExtra = []byte(dm.DNSTap.Extra) + } + + // add additionnals tags ? + if dm.ATags != nil { + ednstap.Atags = &ExtendedATags{ + Tags: dm.ATags.Tags, + } + } + + extendedData, err := proto.Marshal(ednstap) + if err != nil { + return nil, err + } + dt.Extra = extendedData + } + data, err := proto.Marshal(dt) if err != nil { return nil, err diff --git a/dnsutils/message_test.go b/dnsutils/message_test.go index 6fc60e9f..6fcbe23e 100644 --- a/dnsutils/message_test.go +++ b/dnsutils/message_test.go @@ -11,12 +11,45 @@ import ( "google.golang.org/protobuf/proto" ) -func TestDnsMessage_Encode_ToDNSTap(t *testing.T) { +func TestDnsMessage_ToExtendedDNSTap(t *testing.T) { + dm := GetFakeDNSMessageWithPayload() + dm.DNSTap.Extra = "tag0:value0" + + dm.ATags = &TransformATags{ + Tags: []string{"tag1:value1"}, + } + + // encode to extended dnstap + tapMsg, err := dm.ToDNSTap(true) + if err != nil { + t.Fatalf("could not encode to extended dnstap: %v\n", err) + } + + // decode dnstap message + dt := &dnstap.Dnstap{} + err = proto.Unmarshal(tapMsg, dt) + if err != nil { + t.Fatalf("error to decode dnstap: %v", err) + } + + // decode extended part + edt := &ExtendedDnstap{} + err = proto.Unmarshal(dt.GetExtra(), edt) + if err != nil { + t.Fatalf("error to decode extended dnstap: %v", err) + } + + if string(edt.GetOriginalDnstapExtra()) != dm.DNSTap.Extra { + t.Errorf("extra field should be equal to the original value got=%v", string(dt.GetExtra())) + } +} + +func TestDnsMessage_ToDNSTap(t *testing.T) { dm := GetFakeDNSMessageWithPayload() dm.DNSTap.Extra = "extra:value" // encode to dnstap - tapMsg, err := dm.ToDNSTap() + tapMsg, err := dm.ToDNSTap(false) if err != nil { t.Fatalf("could not encode to dnstap: %v\n", err) } diff --git a/docs/collectors/collector_dnstap.md b/docs/collectors/collector_dnstap.md index 7543425f..4430a709 100644 --- a/docs/collectors/collector_dnstap.md +++ b/docs/collectors/collector_dnstap.md @@ -18,6 +18,9 @@ Options: - `reset-conn`: (bool) Reset TCP connection on exit - `chan-buffer-size`: (integer) channel buffer size used on incoming packet, number of packet before to drop it. - `disable-dnsparser"`: (bool) disable the minimalist DNS parser +- `extended-extra-field`: (boolen) decode the extended extra field sent by DNScollector + > If the `extended-extra-field` setting is enabled, DNScollector will expect receiving the specific [protobuf structure](./dnsutils/extended_dnstap.proto) in the extra field, which must be sent by another DNS collector. + > This field will contain additional metadata generated by various transformations such as filtering, ATags, and others. Default values: @@ -34,8 +37,10 @@ dnstap: reset-conn: true chan-buffer-size: 65535 disable-dnsparser: false + extended-extra-field: false ``` + ## DNS tap Proxifier Collector that receives DNSTAP traffic and relays it without decoding or transformations. diff --git a/docs/loggers/logger_dnstap.md b/docs/loggers/logger_dnstap.md index e9a60ff8..4c746ce1 100644 --- a/docs/loggers/logger_dnstap.md +++ b/docs/loggers/logger_dnstap.md @@ -21,6 +21,7 @@ Options: * `overwrite-identity`: (boolean) overwrite original identity * `buffer-size`: (integer) how many DNS messages will be buffered before being sent * `chan-buffer-size`: (integer) channel buffer size used on incoming dns message, number of messages before to drop it. +* `extended-extra-field`: (boolen) Extend the DNStap message by incorporating additional transformations, such as filtering and ATags, into the extra field. Default values: @@ -41,4 +42,5 @@ dnstapclient: overwrite-identity: false buffer-size: 100 chan-buffer-size: 65535 + extended-extra-field: false ``` diff --git a/loggers/dnstapclient.go b/loggers/dnstapclient.go index ee569372..f9a36c7c 100644 --- a/loggers/dnstapclient.go +++ b/loggers/dnstapclient.go @@ -218,7 +218,7 @@ func (ds *DnstapSender) FlushBuffer(buf *[]dnsutils.DNSMessage) { } // encode dns message to dnstap protobuf binary - data, err = dm.ToDNSTap() + data, err = dm.ToDNSTap(ds.config.Loggers.DNSTap.ExtendedSupport) if err != nil { ds.LogError("failed to encode to DNStap protobuf: %s", err) continue diff --git a/loggers/logfile.go b/loggers/logfile.go index f573b715..cd28f2a3 100644 --- a/loggers/logfile.go +++ b/loggers/logfile.go @@ -592,7 +592,7 @@ PROCESS_LOOP: // with dnstap mode case pkgconfig.ModeDNSTap: - data, err = dm.ToDNSTap() + data, err = dm.ToDNSTap(lf.config.Loggers.LogFile.ExtendedSupport) if err != nil { lf.LogError("failed to encode to DNStap protobuf: %s", err) continue diff --git a/pkgconfig/collectors.go b/pkgconfig/collectors.go index ba32508c..243b2425 100644 --- a/pkgconfig/collectors.go +++ b/pkgconfig/collectors.go @@ -31,6 +31,7 @@ type ConfigCollectors struct { ResetConn bool `yaml:"reset-conn"` ChannelBufferSize int `yaml:"chan-buffer-size"` DisableDNSParser bool `yaml:"disable-dnsparser"` + ExtendedSupport bool `yaml:"extended-support"` } `yaml:"dnstap"` DnstapProxifier struct { Enable bool `yaml:"enable"` @@ -105,6 +106,7 @@ func (c *ConfigCollectors) SetDefault() { c.Dnstap.ResetConn = true c.Dnstap.ChannelBufferSize = 65535 c.Dnstap.DisableDNSParser = false + c.Dnstap.ExtendedSupport = false c.DnstapProxifier.Enable = false c.DnstapProxifier.ListenIP = AnyIP diff --git a/pkgconfig/loggers.go b/pkgconfig/loggers.go index cc9a33c0..6a4438ba 100644 --- a/pkgconfig/loggers.go +++ b/pkgconfig/loggers.go @@ -77,6 +77,7 @@ type ConfigLoggers struct { PostRotateDelete bool `yaml:"postrotate-delete-success"` TextFormat string `yaml:"text-format"` ChannelBufferSize int `yaml:"chan-buffer-size"` + ExtendedSupport bool `yaml:"extended-support"` } `yaml:"logfile"` DNSTap struct { Enable bool `yaml:"enable"` @@ -97,6 +98,7 @@ type ConfigLoggers struct { OverwriteIdentity bool `yaml:"overwrite-identity"` BufferSize int `yaml:"buffer-size"` ChannelBufferSize int `yaml:"chan-buffer-size"` + ExtendedSupport bool `yaml:"extended-support"` } `yaml:"dnstapclient"` TCPClient struct { Enable bool `yaml:"enable"` @@ -317,6 +319,7 @@ func (c *ConfigLoggers) SetDefault() { c.DNSTap.OverwriteIdentity = false c.DNSTap.BufferSize = 100 c.DNSTap.ChannelBufferSize = 65535 + c.DNSTap.ExtendedSupport = false c.LogFile.Enable = false c.LogFile.FilePath = "" @@ -331,6 +334,7 @@ func (c *ConfigLoggers) SetDefault() { c.LogFile.PostRotateDelete = false c.LogFile.TextFormat = "" c.LogFile.ChannelBufferSize = 65535 + c.LogFile.ExtendedSupport = false c.Prometheus.Enable = false c.Prometheus.ListenIP = LocalhostIP diff --git a/processors/dnstap.go b/processors/dnstap.go index 43d739d4..d614d9da 100644 --- a/processors/dnstap.go +++ b/processors/dnstap.go @@ -124,6 +124,7 @@ func (d *DNSTapProcessor) Stop() { func (d *DNSTapProcessor) Run(defaultWorkers []pkgutils.Worker, droppedworkers []pkgutils.Worker) { dt := &dnstap.Dnstap{} + edt := &dnsutils.ExtendedDnstap{} //TODO DnstapExtended // prepare next channels defaultRoutes, defaultNames := pkgutils.GetRoutes(defaultWorkers) @@ -174,9 +175,32 @@ RUN_LOOP: } dm.DNSTap.Operation = dt.GetMessage().GetType().String() - extra := string(dt.GetExtra()) - if len(extra) > 0 { - dm.DNSTap.Extra = extra + // extended extra field ? + if d.config.Collectors.Dnstap.ExtendedSupport { + err := proto.Unmarshal(dt.GetExtra(), edt) + if err != nil { + continue + } + + // get original extra value + originalExtra := string(edt.GetOriginalDnstapExtra()) + if len(originalExtra) > 0 { + dm.DNSTap.Extra = originalExtra + } + + // get atags + atags := edt.GetAtags() + if len(atags.GetTags()) > 0 { + dm.ATags = &dnsutils.TransformATags{ + Tags: atags.GetTags(), + } + } + + } else { + extra := string(dt.GetExtra()) + if len(extra) > 0 { + dm.DNSTap.Extra = extra + } } if ipVersion, valid := netlib.IPVersion[dt.GetMessage().GetSocketFamily().String()]; valid { From e166c49bd89ac5b426687d78721f1cfc60c20743 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Wed, 17 Jan 2024 18:35:29 +0100 Subject: [PATCH 2/9] fix linter error --- .github/workflows/golint.yml | 2 +- pkglinker/multiplexer.go | 8 ++++++-- processors/dnstap.go | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/golint.yml b/.github/workflows/golint.yml index ebcc0bfc..ea34f102 100644 --- a/.github/workflows/golint.yml +++ b/.github/workflows/golint.yml @@ -37,6 +37,6 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: - version: v1.54 + version: "v1.55.2" args: --timeout 3m --verbose \ No newline at end of file diff --git a/pkglinker/multiplexer.go b/pkglinker/multiplexer.go index a1202c11..cc2df354 100644 --- a/pkglinker/multiplexer.go +++ b/pkglinker/multiplexer.go @@ -12,6 +12,10 @@ import ( "gopkg.in/yaml.v2" ) +const ( + Transformers = "-transformers" +) + func IsMuxEnabled(config *pkgconfig.Config) bool { if len(config.Multiplexer.Collectors) > 0 && len(config.Multiplexer.Loggers) > 0 && len(config.Multiplexer.Routes) > 0 { return true @@ -54,7 +58,7 @@ func GetItemConfig(section string, config *pkgconfig.Config, item pkgconfig.Mult // load config cfg := make(map[string]interface{}) cfg[section] = item.Params - cfg[section+"-transformers"] = make(map[string]interface{}) + cfg[section+Transformers] = make(map[string]interface{}) for _, p := range item.Params { p.(map[string]interface{})["enable"] = true } @@ -66,7 +70,7 @@ func GetItemConfig(section string, config *pkgconfig.Config, item pkgconfig.Mult // add transformer for k, v := range item.Transforms { v.(map[string]interface{})["enable"] = true - cfg[section+"-transformers"].(map[string]interface{})[k] = v + cfg[section+Transformers].(map[string]interface{})[k] = v } // copy global config diff --git a/processors/dnstap.go b/processors/dnstap.go index d614d9da..7cf342bc 100644 --- a/processors/dnstap.go +++ b/processors/dnstap.go @@ -124,7 +124,7 @@ func (d *DNSTapProcessor) Stop() { func (d *DNSTapProcessor) Run(defaultWorkers []pkgutils.Worker, droppedworkers []pkgutils.Worker) { dt := &dnstap.Dnstap{} - edt := &dnsutils.ExtendedDnstap{} //TODO DnstapExtended + edt := &dnsutils.ExtendedDnstap{} // prepare next channels defaultRoutes, defaultNames := pkgutils.GetRoutes(defaultWorkers) From e90063d3cb879ff69a723c9fcf618851d987c326 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Wed, 17 Jan 2024 18:41:22 +0100 Subject: [PATCH 3/9] update docs --- config.yml | 4 ++-- docs/collectors/collector_dnstap.md | 7 +++---- docs/loggers/logger_dnstap.md | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/config.yml b/config.yml index 0785cf31..ca356deb 100644 --- a/config.yml +++ b/config.yml @@ -95,7 +95,7 @@ multiplexer: # dnstap: # listen-ip: 0.0.0.0 # listen-port: 6002 -# extended-extra-field: true +# extended-support: true # routing-policy: # default: [ console ] @@ -117,7 +117,7 @@ multiplexer: # remote-address: 127.0.0.1 # remote-port: 6002 # flush-interval: 5 -# extended-extra-field: true +# extended-support: true # - name: console # stdout: diff --git a/docs/collectors/collector_dnstap.md b/docs/collectors/collector_dnstap.md index 4430a709..6d97017c 100644 --- a/docs/collectors/collector_dnstap.md +++ b/docs/collectors/collector_dnstap.md @@ -18,8 +18,8 @@ Options: - `reset-conn`: (bool) Reset TCP connection on exit - `chan-buffer-size`: (integer) channel buffer size used on incoming packet, number of packet before to drop it. - `disable-dnsparser"`: (bool) disable the minimalist DNS parser -- `extended-extra-field`: (boolen) decode the extended extra field sent by DNScollector - > If the `extended-extra-field` setting is enabled, DNScollector will expect receiving the specific [protobuf structure](./dnsutils/extended_dnstap.proto) in the extra field, which must be sent by another DNS collector. +- `extended-support`: (boolen) decode the extended extra field sent by DNScollector + > If the `extended-support` setting is enabled, DNScollector will expect receiving the specific [protobuf structure](./dnsutils/extended_dnstap.proto) in the extra field, which must be sent by another DNS collector. > This field will contain additional metadata generated by various transformations such as filtering, ATags, and others. Default values: @@ -37,10 +37,9 @@ dnstap: reset-conn: true chan-buffer-size: 65535 disable-dnsparser: false - extended-extra-field: false + extended-support: false ``` - ## DNS tap Proxifier Collector that receives DNSTAP traffic and relays it without decoding or transformations. diff --git a/docs/loggers/logger_dnstap.md b/docs/loggers/logger_dnstap.md index 4c746ce1..8bff3712 100644 --- a/docs/loggers/logger_dnstap.md +++ b/docs/loggers/logger_dnstap.md @@ -21,7 +21,7 @@ Options: * `overwrite-identity`: (boolean) overwrite original identity * `buffer-size`: (integer) how many DNS messages will be buffered before being sent * `chan-buffer-size`: (integer) channel buffer size used on incoming dns message, number of messages before to drop it. -* `extended-extra-field`: (boolen) Extend the DNStap message by incorporating additional transformations, such as filtering and ATags, into the extra field. +* `extended-support`: (boolen) Extend the DNStap message by incorporating additional transformations, such as filtering and ATags, into the extra field. Default values: @@ -42,5 +42,5 @@ dnstapclient: overwrite-identity: false buffer-size: 100 chan-buffer-size: 65535 - extended-extra-field: false + extended-support: false ``` From f7363984027dcb0daee8c8ffc5bdb72355dc10e8 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Wed, 17 Jan 2024 18:47:20 +0100 Subject: [PATCH 4/9] update docs --- docs/collectors/collector_dnstap.md | 2 +- docs/loggers/logger_dnstap.md | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/collectors/collector_dnstap.md b/docs/collectors/collector_dnstap.md index 6d97017c..3c558950 100644 --- a/docs/collectors/collector_dnstap.md +++ b/docs/collectors/collector_dnstap.md @@ -19,7 +19,7 @@ Options: - `chan-buffer-size`: (integer) channel buffer size used on incoming packet, number of packet before to drop it. - `disable-dnsparser"`: (bool) disable the minimalist DNS parser - `extended-support`: (boolen) decode the extended extra field sent by DNScollector - > If the `extended-support` setting is enabled, DNScollector will expect receiving the specific [protobuf structure](./dnsutils/extended_dnstap.proto) in the extra field, which must be sent by another DNS collector. + > If the `extended-support` setting is enabled, DNScollector will expect receiving the specific [protobuf structure](./../../dnsutils/extended_dnstap.proto) in the extra field, which must be sent by another DNS collector. > This field will contain additional metadata generated by various transformations such as filtering, ATags, and others. Default values: diff --git a/docs/loggers/logger_dnstap.md b/docs/loggers/logger_dnstap.md index 8bff3712..4f262531 100644 --- a/docs/loggers/logger_dnstap.md +++ b/docs/loggers/logger_dnstap.md @@ -7,11 +7,13 @@ Options: * `transport`: (string) network transport to use: `unix`|`tcp`|`tcp+tls` * `remote-address`: (string) remote address * `remote-port`: (integer) remote tcp port -* `sock-path` **DEPRECATED, replaced by remote-address**: (string) unix socket path +* `sock-path`: (string) unix socket path + > DEPRECATED, replaced by `remote-address` setting * `connect-timeout`: (integer) connect timeout in second * `retry-interval`: (integer) interval in second between retry reconnect * `flush-interval`: (integer) interval in second before to flush the buffer -* `tls-support` **DEPRECATED, replaced with tcp+tls flag on transport**: (boolean) enable tls +* `tls-support`: (boolean) enable tls + > DEPRECATED, replaced with `tcp+tls flag` on `transport` settings * `tls-insecure`: (boolean) insecure skip verify * `tls-min-version`: (string) min tls version, default to 1.2 * `ca-file`: (string) provide CA file to verify the server certificate From 619a8268e5fd676665993e7a1564cc0945e7c67d Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Wed, 17 Jan 2024 20:57:51 +0100 Subject: [PATCH 5/9] support geo, filtering, publicsuffix --- dnsutils/extended_dnstap.pb.go | 292 +++++++++++++++++++++++++++------ dnsutils/extended_dnstap.proto | 18 +- dnsutils/message.go | 26 +++ dnsutils/message_test.go | 96 ++++++++++- docs/dnsjson.md | 3 +- docs/loggers/logger_dnstap.md | 4 +- processors/dnstap_test.go | 2 - 7 files changed, 372 insertions(+), 69 deletions(-) diff --git a/dnsutils/extended_dnstap.pb.go b/dnsutils/extended_dnstap.pb.go index 5bf2d812..5171d8ec 100644 --- a/dnsutils/extended_dnstap.pb.go +++ b/dnsutils/extended_dnstap.pb.go @@ -67,7 +67,7 @@ func (x *ExtendedATags) GetTags() []string { return nil } -type ExtendedPublicSuffix struct { +type ExtendedNormalize struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields @@ -76,8 +76,8 @@ type ExtendedPublicSuffix struct { EtldPlusOne string `protobuf:"bytes,2,opt,name=etld_plus_one,json=etldPlusOne,proto3" json:"etld_plus_one,omitempty"` } -func (x *ExtendedPublicSuffix) Reset() { - *x = ExtendedPublicSuffix{} +func (x *ExtendedNormalize) Reset() { + *x = ExtendedNormalize{} if protoimpl.UnsafeEnabled { mi := &file_extended_dnstap_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -85,13 +85,13 @@ func (x *ExtendedPublicSuffix) Reset() { } } -func (x *ExtendedPublicSuffix) String() string { +func (x *ExtendedNormalize) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ExtendedPublicSuffix) ProtoMessage() {} +func (*ExtendedNormalize) ProtoMessage() {} -func (x *ExtendedPublicSuffix) ProtoReflect() protoreflect.Message { +func (x *ExtendedNormalize) ProtoReflect() protoreflect.Message { mi := &file_extended_dnstap_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -103,40 +103,168 @@ func (x *ExtendedPublicSuffix) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ExtendedPublicSuffix.ProtoReflect.Descriptor instead. -func (*ExtendedPublicSuffix) Descriptor() ([]byte, []int) { +// Deprecated: Use ExtendedNormalize.ProtoReflect.Descriptor instead. +func (*ExtendedNormalize) Descriptor() ([]byte, []int) { return file_extended_dnstap_proto_rawDescGZIP(), []int{1} } -func (x *ExtendedPublicSuffix) GetTld() string { +func (x *ExtendedNormalize) GetTld() string { if x != nil { return x.Tld } return "" } -func (x *ExtendedPublicSuffix) GetEtldPlusOne() string { +func (x *ExtendedNormalize) GetEtldPlusOne() string { if x != nil { return x.EtldPlusOne } return "" } +type ExtendedFiltering struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + SampleRate uint32 `protobuf:"varint,1,opt,name=sample_rate,json=sampleRate,proto3" json:"sample_rate,omitempty"` +} + +func (x *ExtendedFiltering) Reset() { + *x = ExtendedFiltering{} + if protoimpl.UnsafeEnabled { + mi := &file_extended_dnstap_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ExtendedFiltering) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExtendedFiltering) ProtoMessage() {} + +func (x *ExtendedFiltering) ProtoReflect() protoreflect.Message { + mi := &file_extended_dnstap_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExtendedFiltering.ProtoReflect.Descriptor instead. +func (*ExtendedFiltering) Descriptor() ([]byte, []int) { + return file_extended_dnstap_proto_rawDescGZIP(), []int{2} +} + +func (x *ExtendedFiltering) GetSampleRate() uint32 { + if x != nil { + return x.SampleRate + } + return 0 +} + +type ExtendedGeo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + City string `protobuf:"bytes,1,opt,name=city,proto3" json:"city,omitempty"` + Continent string `protobuf:"bytes,2,opt,name=continent,proto3" json:"continent,omitempty"` + Isocode string `protobuf:"bytes,3,opt,name=isocode,proto3" json:"isocode,omitempty"` + AsNumber string `protobuf:"bytes,4,opt,name=as_number,json=asNumber,proto3" json:"as_number,omitempty"` + AsOrg string `protobuf:"bytes,5,opt,name=as_org,json=asOrg,proto3" json:"as_org,omitempty"` +} + +func (x *ExtendedGeo) Reset() { + *x = ExtendedGeo{} + if protoimpl.UnsafeEnabled { + mi := &file_extended_dnstap_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ExtendedGeo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExtendedGeo) ProtoMessage() {} + +func (x *ExtendedGeo) ProtoReflect() protoreflect.Message { + mi := &file_extended_dnstap_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExtendedGeo.ProtoReflect.Descriptor instead. +func (*ExtendedGeo) Descriptor() ([]byte, []int) { + return file_extended_dnstap_proto_rawDescGZIP(), []int{3} +} + +func (x *ExtendedGeo) GetCity() string { + if x != nil { + return x.City + } + return "" +} + +func (x *ExtendedGeo) GetContinent() string { + if x != nil { + return x.Continent + } + return "" +} + +func (x *ExtendedGeo) GetIsocode() string { + if x != nil { + return x.Isocode + } + return "" +} + +func (x *ExtendedGeo) GetAsNumber() string { + if x != nil { + return x.AsNumber + } + return "" +} + +func (x *ExtendedGeo) GetAsOrg() string { + if x != nil { + return x.AsOrg + } + return "" +} + type ExtendedDnstap struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` - OriginalDnstapExtra []byte `protobuf:"bytes,2,opt,name=original_dnstap_extra,json=originalDnstapExtra,proto3" json:"original_dnstap_extra,omitempty"` - Atags *ExtendedATags `protobuf:"bytes,3,opt,name=atags,proto3" json:"atags,omitempty"` - PublicSuffix *ExtendedPublicSuffix `protobuf:"bytes,4,opt,name=public_suffix,json=publicSuffix,proto3" json:"public_suffix,omitempty"` + Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + OriginalDnstapExtra []byte `protobuf:"bytes,2,opt,name=original_dnstap_extra,json=originalDnstapExtra,proto3" json:"original_dnstap_extra,omitempty"` + Atags *ExtendedATags `protobuf:"bytes,3,opt,name=atags,proto3" json:"atags,omitempty"` + Normalize *ExtendedNormalize `protobuf:"bytes,4,opt,name=normalize,proto3" json:"normalize,omitempty"` + Filtering *ExtendedFiltering `protobuf:"bytes,5,opt,name=filtering,proto3" json:"filtering,omitempty"` + Geo *ExtendedGeo `protobuf:"bytes,6,opt,name=geo,proto3" json:"geo,omitempty"` } func (x *ExtendedDnstap) Reset() { *x = ExtendedDnstap{} if protoimpl.UnsafeEnabled { - mi := &file_extended_dnstap_proto_msgTypes[2] + mi := &file_extended_dnstap_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -149,7 +277,7 @@ func (x *ExtendedDnstap) String() string { func (*ExtendedDnstap) ProtoMessage() {} func (x *ExtendedDnstap) ProtoReflect() protoreflect.Message { - mi := &file_extended_dnstap_proto_msgTypes[2] + mi := &file_extended_dnstap_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -162,7 +290,7 @@ func (x *ExtendedDnstap) ProtoReflect() protoreflect.Message { // Deprecated: Use ExtendedDnstap.ProtoReflect.Descriptor instead. func (*ExtendedDnstap) Descriptor() ([]byte, []int) { - return file_extended_dnstap_proto_rawDescGZIP(), []int{2} + return file_extended_dnstap_proto_rawDescGZIP(), []int{4} } func (x *ExtendedDnstap) GetVersion() string { @@ -186,9 +314,23 @@ func (x *ExtendedDnstap) GetAtags() *ExtendedATags { return nil } -func (x *ExtendedDnstap) GetPublicSuffix() *ExtendedPublicSuffix { +func (x *ExtendedDnstap) GetNormalize() *ExtendedNormalize { if x != nil { - return x.PublicSuffix + return x.Normalize + } + return nil +} + +func (x *ExtendedDnstap) GetFiltering() *ExtendedFiltering { + if x != nil { + return x.Filtering + } + return nil +} + +func (x *ExtendedDnstap) GetGeo() *ExtendedGeo { + if x != nil { + return x.Geo } return nil } @@ -199,28 +341,44 @@ var file_extended_dnstap_proto_rawDesc = []byte{ 0x0a, 0x15, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x5f, 0x64, 0x6e, 0x73, 0x74, 0x61, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x23, 0x0a, 0x0d, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x41, 0x54, 0x61, 0x67, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x4c, 0x0a, 0x14, - 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x53, 0x75, - 0x66, 0x66, 0x69, 0x78, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x74, 0x6c, 0x64, 0x12, 0x22, 0x0a, 0x0d, 0x65, 0x74, 0x6c, 0x64, 0x5f, 0x70, - 0x6c, 0x75, 0x73, 0x5f, 0x6f, 0x6e, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x65, - 0x74, 0x6c, 0x64, 0x50, 0x6c, 0x75, 0x73, 0x4f, 0x6e, 0x65, 0x22, 0xc0, 0x01, 0x0a, 0x0e, 0x45, - 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x44, 0x6e, 0x73, 0x74, 0x61, 0x70, 0x12, 0x18, 0x0a, - 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x15, 0x6f, 0x72, 0x69, 0x67, 0x69, - 0x6e, 0x61, 0x6c, 0x5f, 0x64, 0x6e, 0x73, 0x74, 0x61, 0x70, 0x5f, 0x65, 0x78, 0x74, 0x72, 0x61, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x13, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, - 0x44, 0x6e, 0x73, 0x74, 0x61, 0x70, 0x45, 0x78, 0x74, 0x72, 0x61, 0x12, 0x24, 0x0a, 0x05, 0x61, - 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x45, 0x78, 0x74, - 0x65, 0x6e, 0x64, 0x65, 0x64, 0x41, 0x54, 0x61, 0x67, 0x73, 0x52, 0x05, 0x61, 0x74, 0x61, 0x67, - 0x73, 0x12, 0x3a, 0x0a, 0x0d, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x73, 0x75, 0x66, 0x66, - 0x69, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x6e, - 0x64, 0x65, 0x64, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x52, - 0x0c, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x53, 0x75, 0x66, 0x66, 0x69, 0x78, 0x42, 0x2e, 0x5a, - 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x6d, 0x61, 0x63, - 0x68, 0x61, 0x72, 0x64, 0x2f, 0x67, 0x6f, 0x2d, 0x64, 0x6e, 0x73, 0x63, 0x6f, 0x6c, 0x6c, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x3b, 0x64, 0x6e, 0x73, 0x75, 0x74, 0x69, 0x6c, 0x73, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x49, 0x0a, 0x11, + 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, + 0x65, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x74, 0x6c, 0x64, 0x12, 0x22, 0x0a, 0x0d, 0x65, 0x74, 0x6c, 0x64, 0x5f, 0x70, 0x6c, 0x75, 0x73, + 0x5f, 0x6f, 0x6e, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x65, 0x74, 0x6c, 0x64, + 0x50, 0x6c, 0x75, 0x73, 0x4f, 0x6e, 0x65, 0x22, 0x34, 0x0a, 0x11, 0x45, 0x78, 0x74, 0x65, 0x6e, + 0x64, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x1f, 0x0a, 0x0b, + 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x72, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x0a, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x61, 0x74, 0x65, 0x22, 0x8d, 0x01, + 0x0a, 0x0b, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x47, 0x65, 0x6f, 0x12, 0x12, 0x0a, + 0x04, 0x63, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x69, 0x74, + 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x65, 0x6e, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x65, 0x6e, 0x74, 0x12, + 0x18, 0x0a, 0x07, 0x69, 0x73, 0x6f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x69, 0x73, 0x6f, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x73, 0x5f, + 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x61, 0x73, + 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x15, 0x0a, 0x06, 0x61, 0x73, 0x5f, 0x6f, 0x72, 0x67, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x61, 0x73, 0x4f, 0x72, 0x67, 0x22, 0x88, 0x02, + 0x0a, 0x0e, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x44, 0x6e, 0x73, 0x74, 0x61, 0x70, + 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x15, 0x6f, 0x72, + 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x5f, 0x64, 0x6e, 0x73, 0x74, 0x61, 0x70, 0x5f, 0x65, 0x78, + 0x74, 0x72, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x13, 0x6f, 0x72, 0x69, 0x67, 0x69, + 0x6e, 0x61, 0x6c, 0x44, 0x6e, 0x73, 0x74, 0x61, 0x70, 0x45, 0x78, 0x74, 0x72, 0x61, 0x12, 0x24, + 0x0a, 0x05, 0x61, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, + 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x41, 0x54, 0x61, 0x67, 0x73, 0x52, 0x05, 0x61, + 0x74, 0x61, 0x67, 0x73, 0x12, 0x30, 0x0a, 0x09, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, + 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, + 0x65, 0x64, 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x52, 0x09, 0x6e, 0x6f, 0x72, + 0x6d, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x12, 0x30, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, + 0x69, 0x6e, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x45, 0x78, 0x74, 0x65, + 0x6e, 0x64, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x52, 0x09, 0x66, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x1e, 0x0a, 0x03, 0x67, 0x65, 0x6f, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, + 0x47, 0x65, 0x6f, 0x52, 0x03, 0x67, 0x65, 0x6f, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x6d, 0x61, 0x63, 0x68, 0x61, 0x72, 0x64, 0x2f, + 0x67, 0x6f, 0x2d, 0x64, 0x6e, 0x73, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x3b, + 0x64, 0x6e, 0x73, 0x75, 0x74, 0x69, 0x6c, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -235,20 +393,24 @@ func file_extended_dnstap_proto_rawDescGZIP() []byte { return file_extended_dnstap_proto_rawDescData } -var file_extended_dnstap_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_extended_dnstap_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_extended_dnstap_proto_goTypes = []interface{}{ - (*ExtendedATags)(nil), // 0: ExtendedATags - (*ExtendedPublicSuffix)(nil), // 1: ExtendedPublicSuffix - (*ExtendedDnstap)(nil), // 2: ExtendedDnstap + (*ExtendedATags)(nil), // 0: ExtendedATags + (*ExtendedNormalize)(nil), // 1: ExtendedNormalize + (*ExtendedFiltering)(nil), // 2: ExtendedFiltering + (*ExtendedGeo)(nil), // 3: ExtendedGeo + (*ExtendedDnstap)(nil), // 4: ExtendedDnstap } var file_extended_dnstap_proto_depIdxs = []int32{ 0, // 0: ExtendedDnstap.atags:type_name -> ExtendedATags - 1, // 1: ExtendedDnstap.public_suffix:type_name -> ExtendedPublicSuffix - 2, // [2:2] is the sub-list for method output_type - 2, // [2:2] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 1, // 1: ExtendedDnstap.normalize:type_name -> ExtendedNormalize + 2, // 2: ExtendedDnstap.filtering:type_name -> ExtendedFiltering + 3, // 3: ExtendedDnstap.geo:type_name -> ExtendedGeo + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_extended_dnstap_proto_init() } @@ -270,7 +432,7 @@ func file_extended_dnstap_proto_init() { } } file_extended_dnstap_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ExtendedPublicSuffix); i { + switch v := v.(*ExtendedNormalize); i { case 0: return &v.state case 1: @@ -282,6 +444,30 @@ func file_extended_dnstap_proto_init() { } } file_extended_dnstap_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ExtendedFiltering); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_extended_dnstap_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ExtendedGeo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_extended_dnstap_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ExtendedDnstap); i { case 0: return &v.state @@ -300,7 +486,7 @@ func file_extended_dnstap_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_extended_dnstap_proto_rawDesc, NumEnums: 0, - NumMessages: 3, + NumMessages: 5, NumExtensions: 0, NumServices: 0, }, diff --git a/dnsutils/extended_dnstap.proto b/dnsutils/extended_dnstap.proto index ccf8868d..431396b7 100644 --- a/dnsutils/extended_dnstap.proto +++ b/dnsutils/extended_dnstap.proto @@ -8,14 +8,28 @@ message ExtendedATags { repeated string tags = 1; } -message ExtendedPublicSuffix { +message ExtendedNormalize { string tld = 1; string etld_plus_one = 2; } +message ExtendedFiltering { + uint32 sample_rate = 1; +} + +message ExtendedGeo { + string city = 1; + string continent = 2; + string isocode = 3; + string as_number = 4; + string as_org = 5; +} + message ExtendedDnstap { string version = 1; bytes original_dnstap_extra = 2; ExtendedATags atags = 3; - ExtendedPublicSuffix public_suffix = 4; + ExtendedNormalize normalize = 4; + ExtendedFiltering filtering = 5; + ExtendedGeo geo = 6; } \ No newline at end of file diff --git a/dnsutils/message.go b/dnsutils/message.go index b5ed549e..c4c4f336 100644 --- a/dnsutils/message.go +++ b/dnsutils/message.go @@ -758,6 +758,32 @@ func (dm *DNSMessage) ToDNSTap(extended bool) ([]byte, error) { } } + // add public suffix + if dm.PublicSuffix != nil { + ednstap.Normalize = &ExtendedNormalize{ + Tld: dm.PublicSuffix.QnamePublicSuffix, + EtldPlusOne: dm.PublicSuffix.QnameEffectiveTLDPlusOne, + } + } + + // add filtering + if dm.Filtering != nil { + ednstap.Filtering = &ExtendedFiltering{ + SampleRate: uint32(dm.Filtering.SampleRate), + } + } + + // add geo + if dm.Geo != nil { + ednstap.Geo = &ExtendedGeo{ + City: dm.Geo.City, + Continent: dm.Geo.Continent, + Isocode: dm.Geo.CountryIsoCode, + AsNumber: dm.Geo.AutonomousSystemNumber, + AsOrg: dm.Geo.AutonomousSystemOrg, + } + } + extendedData, err := proto.Marshal(ednstap) if err != nil { return nil, err diff --git a/dnsutils/message_test.go b/dnsutils/message_test.go index 6fcbe23e..5ae97f65 100644 --- a/dnsutils/message_test.go +++ b/dnsutils/message_test.go @@ -11,14 +11,8 @@ import ( "google.golang.org/protobuf/proto" ) -func TestDnsMessage_ToExtendedDNSTap(t *testing.T) { - dm := GetFakeDNSMessageWithPayload() - dm.DNSTap.Extra = "tag0:value0" - - dm.ATags = &TransformATags{ - Tags: []string{"tag1:value1"}, - } - +// Tests for DNSTap format +func encodeToDNSTap(dm DNSMessage, t *testing.T) *ExtendedDnstap { // encode to extended dnstap tapMsg, err := dm.ToDNSTap(true) if err != nil { @@ -38,9 +32,91 @@ func TestDnsMessage_ToExtendedDNSTap(t *testing.T) { if err != nil { t.Fatalf("error to decode extended dnstap: %v", err) } + return edt +} + +func TestDnsMessage_ToExtendedDNSTap_GetOriginalDnstapExtra(t *testing.T) { + dm := GetFakeDNSMessageWithPayload() + dm.DNSTap.Extra = "tag0:value0" + + // encode to DNSTap and decode extended + edt := encodeToDNSTap(dm, t) + // check if string(edt.GetOriginalDnstapExtra()) != dm.DNSTap.Extra { - t.Errorf("extra field should be equal to the original value got=%v", string(dt.GetExtra())) + t.Errorf("extra field should be equal to the original value") + } +} + +func TestDnsMessage_ToExtendedDNSTap_TransformAtags(t *testing.T) { + dm := GetFakeDNSMessageWithPayload() + dm.ATags = &TransformATags{ + Tags: []string{"tag1:value1"}, + } + + // encode to DNSTap and decode extended + edt := encodeToDNSTap(dm, t) + + // check + if edt.GetAtags().Tags[0] != "tag1:value1" { + t.Errorf("invalid value on atags") + } +} + +func TestDnsMessage_ToExtendedDNSTap_TransformNormalize(t *testing.T) { + dm := GetFakeDNSMessageWithPayload() + dm.PublicSuffix = &TransformPublicSuffix{ + QnamePublicSuffix: "com", + QnameEffectiveTLDPlusOne: "dnscollector.com", + } + + // encode to DNSTap and decode extended + edt := encodeToDNSTap(dm, t) + + // checks + if edt.GetNormalize().GetTld() != "com" { + t.Errorf("invalid value on tld") + } + + if edt.GetNormalize().GetEtldPlusOne() != "dnscollector.com" { + t.Errorf("invalid value on etld+1") + } +} + +func TestDnsMessage_ToExtendedDNSTap_TransformFiltering(t *testing.T) { + dm := GetFakeDNSMessageWithPayload() + dm.Filtering = &TransformFiltering{ + SampleRate: 20, + } + + // encode to DNSTap and decode extended + edt := encodeToDNSTap(dm, t) + + // checks + if edt.GetFiltering().GetSampleRate() != 20 { + t.Errorf("invalid value sample rate") + } +} + +func TestDnsMessage_ToExtendedDNSTap_TransformGeo(t *testing.T) { + dm := GetFakeDNSMessageWithPayload() + dm.Geo = &TransformDNSGeo{ + City: "France", + Continent: "Europe", + CountryIsoCode: "44444", + AutonomousSystemNumber: "3333", + AutonomousSystemOrg: "Test", + } + + // encode to DNSTap and decode extended + edt := encodeToDNSTap(dm, t) + + // checks + if edt.GetGeo().GetCity() != "France" { + t.Errorf("invalid value for city") + } + if edt.GetGeo().GetContinent() != "Europe" { + t.Errorf("invalid value for continent") } } @@ -70,6 +146,7 @@ func TestDnsMessage_ToDNSTap(t *testing.T) { } } +// Tests for JSON format func TestDnsMessage_Json_Reference(t *testing.T) { dm := DNSMessage{} dm.Init() @@ -287,6 +364,7 @@ func TestDnsMessage_Json_Transforms_Reference(t *testing.T) { } } +// Tests for TEXT format func TestDnsMessage_TextFormat_ToString(t *testing.T) { config := pkgconfig.GetFakeConfig() diff --git a/docs/dnsjson.md b/docs/dnsjson.md index 37d66f2c..27b3507f 100644 --- a/docs/dnsjson.md +++ b/docs/dnsjson.md @@ -136,4 +136,5 @@ This JSON message can be also extended by transformer(s): - [GeoIP](transformers/transformer_geoip.md) - [Suspicious traffic detector](transformers/transform_suspiciousdetector.md) - [Public suffix](transformers/transform_normalize.md) -- [Traffic Reducer](transformers/transform_trafficreducer.md) +- [Traffic reducer](transformers/transform_trafficreducer.md) +- [Traffic filtering](transformers/transformer_trafficfiltering.md) \ No newline at end of file diff --git a/docs/loggers/logger_dnstap.md b/docs/loggers/logger_dnstap.md index 4f262531..69d9f1dd 100644 --- a/docs/loggers/logger_dnstap.md +++ b/docs/loggers/logger_dnstap.md @@ -13,9 +13,9 @@ Options: * `retry-interval`: (integer) interval in second between retry reconnect * `flush-interval`: (integer) interval in second before to flush the buffer * `tls-support`: (boolean) enable tls - > DEPRECATED, replaced with `tcp+tls flag` on `transport` settings + > DEPRECATED, replaced with `tcp+tls` flag on `transport` settings * `tls-insecure`: (boolean) insecure skip verify -* `tls-min-version`: (string) min tls version, default to 1.2 +* `tls-min-version`: (string) minimum tls version to use * `ca-file`: (string) provide CA file to verify the server certificate * `cert-file`: (string) provide client certificate file for mTLS * `key-file`: (string) provide client private key file for mTLS diff --git a/processors/dnstap_test.go b/processors/dnstap_test.go index 1f981079..9627954a 100644 --- a/processors/dnstap_test.go +++ b/processors/dnstap_test.go @@ -20,7 +20,6 @@ func Test_DnstapProcessor(t *testing.T) { // init the dnstap consumer consumer := NewDNSTapProcessor(0, pkgconfig.GetFakeConfig(), logger, "test", 512) - // chanTo := make(chan dnsutils.DNSMessage, 512) // prepare dns query dnsmsg := new(dns.Msg) @@ -198,7 +197,6 @@ func Test_DnstapProcessor_DisableDNSParser(t *testing.T) { fl := loggers.NewFakeLogger() go consumer.Run([]pkgutils.Worker{fl}, []pkgutils.Worker{fl}) - // go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"}) // add packet to consumer consumer.GetChannel() <- data From c82b8df1a85a3b15456a2c261d820970fcd956fd Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Wed, 17 Jan 2024 21:20:43 +0100 Subject: [PATCH 6/9] add test --- processors/dnstap_test.go | 69 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/processors/dnstap_test.go b/processors/dnstap_test.go index 9627954a..336cf53f 100644 --- a/processors/dnstap_test.go +++ b/processors/dnstap_test.go @@ -4,6 +4,7 @@ import ( "bytes" "testing" + "github.com/dmachard/go-dnscollector/dnsutils" "github.com/dmachard/go-dnscollector/loggers" "github.com/dmachard/go-dnscollector/pkgconfig" "github.com/dmachard/go-dnscollector/pkgutils" @@ -206,3 +207,71 @@ func Test_DnstapProcessor_DisableDNSParser(t *testing.T) { t.Errorf("DNS ID should be equal to zero: %d", dm.DNS.ID) } } + +func Test_DnstapProcessor_Extended(t *testing.T) { + logger := logger.New(true) + var o bytes.Buffer + logger.SetOutput(&o) + + // init the dnstap consumer + cfg := pkgconfig.GetFakeConfig() + cfg.Collectors.Dnstap.ExtendedSupport = true + + consumer := NewDNSTapProcessor(0, cfg, logger, "test", 512) + // chanTo := make(chan dnsutils.DNSMessage, 512) + + // prepare dns query + dnsmsg := new(dns.Msg) + dnsmsg.SetQuestion("www.google.fr.", dns.TypeA) + dnsquestion, _ := dnsmsg.Pack() + + // prepare dnstap + dt := &dnstap.Dnstap{} + dt.Type = dnstap.Dnstap_Type.Enum(1) + + dt.Message = &dnstap.Message{} + dt.Message.Type = dnstap.Message_Type.Enum(5) + dt.Message.QueryMessage = dnsquestion + + edt := &dnsutils.ExtendedDnstap{} + edt.Atags = &dnsutils.ExtendedATags{ + Tags: []string{"atags:value"}, + } + edt.OriginalDnstapExtra = []byte("originalextrafield") + edt.Normalize = &dnsutils.ExtendedNormalize{ + Tld: "org", + EtldPlusOne: "dnscollector.org", + } + edt.Filtering = &dnsutils.ExtendedFiltering{ + SampleRate: 30, + } + edtData, _ := proto.Marshal(edt) + dt.Extra = edtData + + data, _ := proto.Marshal(dt) + + // run the consumer with a fake logger + fl := loggers.NewFakeLogger() + go consumer.Run([]pkgutils.Worker{fl}, []pkgutils.Worker{fl}) + + // add packet to consumer + consumer.GetChannel() <- data + + // read dns message from dnstap consumer + dm := <-fl.GetInputChannel() + if dm.DNSTap.Extra != "originalextrafield" { + t.Errorf("invalid extra field: %s", dm.DNSTap.Extra) + } + // if dm.ATags.Tags[0] != "atags:value" { + // t.Errorf("invalid atags: %s", dm.ATags.Tags[0]) + // } + // if dm.PublicSuffix.QnameEffectiveTLDPlusOne != "dnscollector.org" { + // t.Errorf("invalid etld+1: %s", dm.PublicSuffix.QnameEffectiveTLDPlusOne) + // } + // if dm.PublicSuffix.QnamePublicSuffix != "org" { + // t.Errorf("invalid tld: %s", dm.PublicSuffix.QnamePublicSuffix) + // } + // if dm.Filtering.SampleRate != 30 { + // t.Errorf("invalid sample rate: %d", dm.Filtering.SampleRate) + // } +} From e310ecdcb145785d59755221b2e7d3b41dce1c1c Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Wed, 17 Jan 2024 21:21:07 +0100 Subject: [PATCH 7/9] add test for atags --- processors/dnstap_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/processors/dnstap_test.go b/processors/dnstap_test.go index 336cf53f..df200473 100644 --- a/processors/dnstap_test.go +++ b/processors/dnstap_test.go @@ -262,9 +262,9 @@ func Test_DnstapProcessor_Extended(t *testing.T) { if dm.DNSTap.Extra != "originalextrafield" { t.Errorf("invalid extra field: %s", dm.DNSTap.Extra) } - // if dm.ATags.Tags[0] != "atags:value" { - // t.Errorf("invalid atags: %s", dm.ATags.Tags[0]) - // } + if dm.ATags.Tags[0] != "atags:value" { + t.Errorf("invalid atags: %s", dm.ATags.Tags[0]) + } // if dm.PublicSuffix.QnameEffectiveTLDPlusOne != "dnscollector.org" { // t.Errorf("invalid etld+1: %s", dm.PublicSuffix.QnameEffectiveTLDPlusOne) // } From 4b8cbfa5cfc8546a745dc055db8988603d07f31d Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Thu, 18 Jan 2024 20:25:12 +0100 Subject: [PATCH 8/9] more tests --- processors/dnstap.go | 20 +++++++++++++++++++- processors/dnstap_test.go | 18 +++++++++--------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/processors/dnstap.go b/processors/dnstap.go index 7cf342bc..506e0918 100644 --- a/processors/dnstap.go +++ b/processors/dnstap.go @@ -190,12 +190,30 @@ RUN_LOOP: // get atags atags := edt.GetAtags() - if len(atags.GetTags()) > 0 { + if atags != nil { dm.ATags = &dnsutils.TransformATags{ Tags: atags.GetTags(), } } + // get public suffix + norm := edt.GetNormalize() + if norm != nil { + dm.PublicSuffix = &dnsutils.TransformPublicSuffix{} + if len(norm.GetTld()) > 0 { + dm.PublicSuffix.QnamePublicSuffix = norm.GetTld() + } + if len(norm.GetEtldPlusOne()) > 0 { + dm.PublicSuffix.QnameEffectiveTLDPlusOne = norm.GetEtldPlusOne() + } + } + + // filtering + sampleRate := edt.GetFiltering() + if sampleRate != nil { + dm.Filtering = &dnsutils.TransformFiltering{} + dm.Filtering.SampleRate = int(sampleRate.SampleRate) + } } else { extra := string(dt.GetExtra()) if len(extra) > 0 { diff --git a/processors/dnstap_test.go b/processors/dnstap_test.go index df200473..4a4ab2b5 100644 --- a/processors/dnstap_test.go +++ b/processors/dnstap_test.go @@ -265,13 +265,13 @@ func Test_DnstapProcessor_Extended(t *testing.T) { if dm.ATags.Tags[0] != "atags:value" { t.Errorf("invalid atags: %s", dm.ATags.Tags[0]) } - // if dm.PublicSuffix.QnameEffectiveTLDPlusOne != "dnscollector.org" { - // t.Errorf("invalid etld+1: %s", dm.PublicSuffix.QnameEffectiveTLDPlusOne) - // } - // if dm.PublicSuffix.QnamePublicSuffix != "org" { - // t.Errorf("invalid tld: %s", dm.PublicSuffix.QnamePublicSuffix) - // } - // if dm.Filtering.SampleRate != 30 { - // t.Errorf("invalid sample rate: %d", dm.Filtering.SampleRate) - // } + if dm.PublicSuffix.QnameEffectiveTLDPlusOne != "dnscollector.org" { + t.Errorf("invalid etld+1: %s", dm.PublicSuffix.QnameEffectiveTLDPlusOne) + } + if dm.PublicSuffix.QnamePublicSuffix != "org" { + t.Errorf("invalid tld: %s", dm.PublicSuffix.QnamePublicSuffix) + } + if dm.Filtering.SampleRate != 30 { + t.Errorf("invalid sample rate: %d", dm.Filtering.SampleRate) + } } From cc20d06b7533a419e11c19c6cfc62dc629a8890d Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Thu, 18 Jan 2024 20:36:04 +0100 Subject: [PATCH 9/9] Update docs --- README.md | 4 ++-- docs/collectors/collector_dnstap.md | 5 ++++- processors/dnstap_test.go | 3 +-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index fd5fd054..8fa56d0d 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@

Go Report Go version -Go tests -Go lines +Go tests +Go lines

diff --git a/docs/collectors/collector_dnstap.md b/docs/collectors/collector_dnstap.md index 3c558950..7ef167ab 100644 --- a/docs/collectors/collector_dnstap.md +++ b/docs/collectors/collector_dnstap.md @@ -15,11 +15,14 @@ Options: - `cert-file`: (string) certificate server file - `key-file`: (string) private key server file - `sock-rcvbuf`: (integer) sets the socket receive buffer in bytes SO_RCVBUF, set to zero to use the default system value + > This advanced parameter allows fine-tuning of network performance by adjusting the amount of data the socket can receive before signaling to the sender to slow down. - `reset-conn`: (bool) Reset TCP connection on exit + > Send a TCP reset to the remote sender to ensure a proper termination of the connection. - `chan-buffer-size`: (integer) channel buffer size used on incoming packet, number of packet before to drop it. - `disable-dnsparser"`: (bool) disable the minimalist DNS parser + > Some JSON keys should not be available, such as `dns.id`, `dns.flags`, ... - `extended-support`: (boolen) decode the extended extra field sent by DNScollector - > If the `extended-support` setting is enabled, DNScollector will expect receiving the specific [protobuf structure](./../../dnsutils/extended_dnstap.proto) in the extra field, which must be sent by another DNS collector. + > If this setting is enabled, DNScollector will expect receiving the specific [protobuf structure](./../../dnsutils/extended_dnstap.proto) in the extra field, which must be sent by another DNS collector. > This field will contain additional metadata generated by various transformations such as filtering, ATags, and others. Default values: diff --git a/processors/dnstap_test.go b/processors/dnstap_test.go index 4a4ab2b5..24e9a993 100644 --- a/processors/dnstap_test.go +++ b/processors/dnstap_test.go @@ -177,7 +177,6 @@ func Test_DnstapProcessor_DisableDNSParser(t *testing.T) { cfg.Collectors.Dnstap.DisableDNSParser = true consumer := NewDNSTapProcessor(0, cfg, logger, "test", 512) - // chanTo := make(chan dnsutils.DNSMessage, 512) // prepare dns query dnsmsg := new(dns.Msg) @@ -208,6 +207,7 @@ func Test_DnstapProcessor_DisableDNSParser(t *testing.T) { } } +// test to decode the extended part func Test_DnstapProcessor_Extended(t *testing.T) { logger := logger.New(true) var o bytes.Buffer @@ -218,7 +218,6 @@ func Test_DnstapProcessor_Extended(t *testing.T) { cfg.Collectors.Dnstap.ExtendedSupport = true consumer := NewDNSTapProcessor(0, cfg, logger, "test", 512) - // chanTo := make(chan dnsutils.DNSMessage, 512) // prepare dns query dnsmsg := new(dns.Msg)