-
+
-
+
@@ -70,8 +70,8 @@ - **[Transformers](./docs/transformers.md)** - - Custom [Relabeling](docs/transformers/transform_relabeling.md) for JSON structure - - Add additionnal [Tags](docs/transformers/transform_atags.md) + - [Rewrite](docs/transformers/transform_rewrite.md) DNS messages or custom [Relabeling](docs/transformers/transform_relabeling.md) for JSON output + - Add additionnal [Tags](docs/transformers/transform_atags.md) in DNS messages - Traffic [Filtering](docs/transformers/transform_trafficfiltering.md) and [Reducer](docs/transformers/transform_trafficreducer.md) - Latency [Computing](docs/transformers/transform_latency.md) - Apply [User Privacy](docs/transformers/transform_userprivacy.md) diff --git a/config.yml b/config.yml index f625124f..fd762676 100644 --- a/config.yml +++ b/config.yml @@ -49,26 +49,3 @@ pipelines: - name: console stdout: mode: text - -################################################ -# DEPRECATED - multiplexer configuration -# more details: https://github.com/dmachard/go-dnscollector/blob/main/docs/running_mode.md#multiplexer -# workers: https://github.com/dmachard/go-dnscollector/blob/main/docs/workers.md -# transformers: https://github.com/dmachard/go-dnscollector/blob/main/docs/transformers.md -################################################ -# multiplexer: -# collectors: -# - name: tap -# dnstap: -# listen-ip: 0.0.0.0 -# listen-port: 6000 -# transforms: -# normalize: -# qname-lowercase: true -# loggers: -# - name: console -# stdout: -# mode: text -# routes: -# - from: [ tap ] -# to: [ console ] diff --git a/dnsutils/message.go b/dnsutils/message.go index 0ce65015..d7b24da3 100644 --- a/dnsutils/message.go +++ b/dnsutils/message.go @@ -673,26 +673,26 @@ func (dm *DNSMessage) ToTextLine(format []string, fieldDelimiter string, fieldBo if len(qname) == 0 { s.WriteString(".") } else { - if len(fieldDelimiter) > 0 { - if strings.Contains(qname, fieldDelimiter) { - qnameEscaped := qname - if strings.Contains(qname, fieldBoundary) { - qnameEscaped = strings.ReplaceAll(qnameEscaped, fieldBoundary, "\\"+fieldBoundary) - } - s.WriteString(fmt.Sprintf(fieldBoundary+"%s"+fieldBoundary, qnameEscaped)) - } else { - s.WriteString(qname) - } - } else { - s.WriteString(qname) - } + quoteStringAndWrite(&s, qname, fieldDelimiter, fieldBoundary) } case directive == "identity": - s.WriteString(dm.DNSTap.Identity) + if len(dm.DNSTap.Identity) == 0 { + s.WriteString("-") + } else { + quoteStringAndWrite(&s, dm.DNSTap.Identity, fieldDelimiter, fieldBoundary) + } case directive == "peer-name": - s.WriteString(dm.DNSTap.PeerName) + if len(dm.DNSTap.PeerName) == 0 { + s.WriteString("-") + } else { + quoteStringAndWrite(&s, dm.DNSTap.PeerName, fieldDelimiter, fieldBoundary) + } case directive == "version": - s.WriteString(dm.DNSTap.Version) + if len(dm.DNSTap.Version) == 0 { + s.WriteString("-") + } else { + quoteStringAndWrite(&s, dm.DNSTap.Version, fieldDelimiter, fieldBoundary) + } case directive == "extra": s.WriteString(dm.DNSTap.Extra) case directive == "policy-rule": @@ -1171,6 +1171,7 @@ func (dm *DNSMessage) Flatten() (map[string]interface{}, error) { "dns.qtype": dm.DNS.Qtype, "dns.qclass": dm.DNS.Qclass, "dns.rcode": dm.DNS.Rcode, + "dns.questions-count": dm.DNS.QuestionsCount, "dnstap.identity": dm.DNSTap.Identity, "dnstap.latency": dm.DNSTap.LatencySec, "dnstap.operation": dm.DNSTap.Operation, @@ -1858,7 +1859,9 @@ func GetFakeDNSMessage() DNSMessage { dm := DNSMessage{} dm.Init() dm.DNSTap.Identity = "collector" + dm.DNSTap.Version = "dnscollector 1.0.0" dm.DNSTap.Operation = "CLIENT_QUERY" + dm.DNSTap.PeerName = "localhost (127.0.0.1)" dm.DNS.Type = DNSQuery dm.DNS.Qname = pkgconfig.ProgQname dm.NetworkInfo.QueryIP = "1.2.3.4" @@ -1913,3 +1916,19 @@ func convertToString(value interface{}) string { return fmt.Sprintf("%v", v) } } + +func quoteStringAndWrite(s *strings.Builder, fieldString, fieldDelimiter, fieldBoundary string) { + if len(fieldDelimiter) > 0 { + if strings.Contains(fieldString, fieldDelimiter) { + fieldEscaped := fieldString + if strings.Contains(fieldString, fieldBoundary) { + fieldEscaped = strings.ReplaceAll(fieldEscaped, fieldBoundary, "\\"+fieldBoundary) + } + s.WriteString(fmt.Sprintf(fieldBoundary+"%s"+fieldBoundary, fieldEscaped)) + } else { + s.WriteString(fieldString) + } + } else { + s.WriteString(fieldString) + } +} diff --git a/dnsutils/message_test.go b/dnsutils/message_test.go index e3683d87..eb5bda19 100644 --- a/dnsutils/message_test.go +++ b/dnsutils/message_test.go @@ -469,6 +469,7 @@ func TestDnsMessage_JsonFlatten_Reference(t *testing.T) { "dns.qtype": "-", "dns.rcode": "-", "dns.qclass": "-", + "dns.questions-count": 0, "dns.resource-records.an.0.name": "google.nl", "dns.resource-records.an.0.rdata": "142.251.39.99", "dns.resource-records.an.0.rdatatype": "A", @@ -807,6 +808,7 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) { boundary string format string qname string + identity string expected string }{ { @@ -815,7 +817,8 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) { boundary: config.Global.TextFormatBoundary, format: config.Global.TextFormat, qname: "dnscollector.fr", - expected: "- - - - - - - - 0b dnscollector.fr - -", + identity: "collector", + expected: "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b dnscollector.fr A -", }, { name: "custom_delimiter", @@ -823,7 +826,8 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) { boundary: config.Global.TextFormatBoundary, format: config.Global.TextFormat, qname: "dnscollector.fr", - expected: "-;-;-;-;-;-;-;-;0b;dnscollector.fr;-;-", + identity: "collector", + expected: "-;collector;CLIENT_QUERY;NOERROR;1.2.3.4;1234;-;-;0b;dnscollector.fr;A;-", }, { name: "empty_delimiter", @@ -831,7 +835,8 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) { boundary: config.Global.TextFormatBoundary, format: config.Global.TextFormat, qname: "dnscollector.fr", - expected: "--------0bdnscollector.fr--", + identity: "collector", + expected: "-collectorCLIENT_QUERYNOERROR1.2.3.41234--0bdnscollector.frA-", }, { name: "qname_quote", @@ -839,7 +844,8 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) { boundary: config.Global.TextFormatBoundary, format: config.Global.TextFormat, qname: "dns collector.fr", - expected: "- - - - - - - - 0b \"dns collector.fr\" - -", + identity: "collector", + expected: "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b \"dns collector.fr\" A -", }, { name: "default_boundary", @@ -847,7 +853,8 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) { boundary: config.Global.TextFormatBoundary, format: config.Global.TextFormat, qname: "dns\"coll tor\".fr", - expected: "- - - - - - - - 0b \"dns\\\"coll tor\\\".fr\" - -", + identity: "collector", + expected: "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b \"dns\\\"coll tor\\\".fr\" A -", }, { name: "custom_boundary", @@ -855,7 +862,8 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) { boundary: "!", format: config.Global.TextFormat, qname: "dnscoll tor.fr", - expected: "- - - - - - - - 0b !dnscoll tor.fr! - -", + identity: "collector", + expected: "- collector CLIENT_QUERY NOERROR 1.2.3.4 1234 - - 0b !dnscoll tor.fr! A -", }, { name: "custom_text", @@ -863,16 +871,44 @@ func TestDnsMessage_TextFormat_ToString(t *testing.T) { boundary: config.Global.TextFormatBoundary, format: "qname {IN} qtype", qname: "dnscollector.fr", - expected: "dnscollector.fr IN -", + identity: "", + expected: "dnscollector.fr IN A", + }, + { + name: "quote_dnstap_version", + delimiter: config.Global.TextFormatDelimiter, + boundary: config.Global.TextFormatBoundary, + format: "identity version qname", + qname: "dnscollector.fr", + identity: "collector", + expected: "collector \"dnscollector 1.0.0\" dnscollector.fr", + }, + { + name: "quote_dnstap_identity", + delimiter: config.Global.TextFormatDelimiter, + boundary: config.Global.TextFormatBoundary, + format: "identity qname", + qname: "dnscollector.fr", + identity: "dns collector", + expected: "\"dns collector\" dnscollector.fr", + }, + { + name: "quote_dnstap_peername", + delimiter: config.Global.TextFormatDelimiter, + boundary: config.Global.TextFormatBoundary, + format: "peer-name qname", + qname: "dnscollector.fr", + identity: "", + expected: "\"localhost (127.0.0.1)\" dnscollector.fr", }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - dm := DNSMessage{} - dm.Init() + dm := GetFakeDNSMessage() dm.DNS.Qname = tc.qname + dm.DNSTap.Identity = tc.identity line := dm.String(strings.Fields(tc.format), tc.delimiter, tc.boundary) if line != tc.expected { diff --git a/docs/development.md b/docs/development.md index 27080bb7..f881f5f1 100644 --- a/docs/development.md +++ b/docs/development.md @@ -2,7 +2,7 @@ # DNS-collector - Development To compile DNS-collector, we assume you have a working Go setup. -First, make sure your golang version is `1.20` or higher +First, make sure your golang version is `1.21` or higher How to userguides: diff --git a/docs/dnsjson.md b/docs/dnsjson.md index e68c36a7..c4bfc342 100644 --- a/docs/dnsjson.md +++ b/docs/dnsjson.md @@ -29,6 +29,7 @@ Example: "qtype": "A", "id": 23455, "qclass": "IN", + "questions-count": 0, "flags": { "qr": true, "tc": false, @@ -116,6 +117,7 @@ Here's a flat JSON output formatted using `jq`: "dns.qtype": "A", "dns.rcode": "NOERROR", "dns.qclass": "IN", + "dns.questions-count": 0, "dns.resource-records.an.0.name": "google.nl", "dns.resource-records.an.0.rdata": "142.251.39.99", "dns.resource-records.an.0.rdatatype": "A", diff --git a/docs/transformers.md b/docs/transformers.md index 713754d2..c9babba9 100644 --- a/docs/transformers.md +++ b/docs/transformers.md @@ -27,3 +27,4 @@ Transformers processing is currently in this order : | [Traffic Prediction](transformers/transform_trafficprediction.md) | Features to train machine learning models | | [Additionnal Tags](transformers/transform_atags.md) | Add additionnal tags | | [JSON relabeling](transformers/transform_relabeling.md) | JSON relabeling to rename or remove keys | +| [DNS message rewrite](transformers/transform_rewrite.md) | Rewrite value for DNS messages structure | diff --git a/docs/transformers/transform_relabeling.md b/docs/transformers/transform_relabeling.md index 4c1930cb..15269c34 100644 --- a/docs/transformers/transform_relabeling.md +++ b/docs/transformers/transform_relabeling.md @@ -1,7 +1,7 @@ # Transformer: Relabeling Use this transformer to remove or rename some JSON keys. -This transformation is only applied to the [`flat-json`](../dnsjson.md) output format. +This transformation is only applied to the [`flat-json`](../dnsjson.md#flat-json-format-recommended) output format. Options: @@ -14,28 +14,27 @@ Options: Configuration example ```yaml - loggers: - - name: console - stdout: - mode: flat-json - transforms: - relabeling: - rename: - - regex: "dnstap\\.timestamp-rfc3339ns" - replacement: "timestamp" - - regex: "dns\\.qname" - replacement: "query" - - regex: "network\\.query-ip" - replacement: "client" - - regex: "network\\.response-ip" - replacement: "server" - - regex: "dnstap\\.identity" - replacement: "client_id" - - regex: "^dns\\.resource-records\\.an\\..*\\.rdata$" - replacement: "answers_rdata" - remove: - - regex: "dns" - - regex: "network" +- name: console + stdout: + mode: flat-json + transforms: + relabeling: + rename: + - regex: "dnstap\\.timestamp-rfc3339ns" + replacement: "timestamp" + - regex: "dns\\.qname" + replacement: "query" + - regex: "network\\.query-ip" + replacement: "client" + - regex: "network\\.response-ip" + replacement: "server" + - regex: "dnstap\\.identity" + replacement: "client_id" + - regex: "^dns\\.resource-records\\.an\\..*\\.rdata$" + replacement: "answers_rdata" + remove: + - regex: "dns" + - regex: "network" ``` This config produces the following flat-json ouput: diff --git a/docs/transformers/transform_rewrite.md b/docs/transformers/transform_rewrite.md new file mode 100644 index 00000000..c985f665 --- /dev/null +++ b/docs/transformers/transform_rewrite.md @@ -0,0 +1,25 @@ +# Transformer: Rewrite + +Use this transformer to rewrite the content of DNS messages according to the [structure](../dnsjson.md#dns-collector---json-encoding). +For more details, see the feature request [here](https://github.com/dmachard/go-dnscollector/issues/527). + +> Only fields with int and string types are supported. + +Options: + +* `identifiers` (map) + > Expect a key/value where the key is the namf of the field to rewrite (Please refer to the [`flat-json`](../dnsjson.md#flat-json-format-recommended) output to see all identifiers keys ) and the value is the new one. + +Config example to remove the DNStap version and update the identity name. + +```yaml +- name: tap +dnstap: + listen-ip: 0.0.0.0 + listen-port: 6000 +transforms: + rewrite: + identifiers: + dnstap.version: "" + dnstap.identity: "foo" +``` \ No newline at end of file diff --git a/pkgconfig/transformers.go b/pkgconfig/transformers.go index 37be6ff9..8d87f0c0 100644 --- a/pkgconfig/transformers.go +++ b/pkgconfig/transformers.go @@ -90,6 +90,10 @@ type ConfigTransformers struct { Rename []RelabelingConfig `yaml:"rename,flow"` Remove []RelabelingConfig `yaml:"remove,flow"` } `yaml:"relabeling"` + Rewrite struct { + Enable bool `yaml:"enable" default:"false"` + Identifiers map[string]interface{} `yaml:"identifiers,flow"` + } `yaml:"rewrite"` } func (c *ConfigTransformers) SetDefault() { diff --git a/transformers/rewrite.go b/transformers/rewrite.go new file mode 100644 index 00000000..29c7df64 --- /dev/null +++ b/transformers/rewrite.go @@ -0,0 +1,98 @@ +package transformers + +import ( + "errors" + "reflect" + "strings" + + "github.com/dmachard/go-dnscollector/dnsutils" + "github.com/dmachard/go-dnscollector/pkgconfig" + "github.com/dmachard/go-logger" +) + +type RewriteTransform struct { + GenericTransformer +} + +func NewRewriteTransform(config *pkgconfig.ConfigTransformers, logger *logger.Logger, name string, instance int, nextWorkers []chan dnsutils.DNSMessage) *RewriteTransform { + t := &RewriteTransform{GenericTransformer: NewTransformer(config, logger, "rewrite", name, instance, nextWorkers)} + return t +} + +func (t *RewriteTransform) GetTransforms() ([]Subtransform, error) { + subtransforms := []Subtransform{} + if len(t.config.Rewrite.Identifiers) > 0 { + subtransforms = append(subtransforms, Subtransform{name: "rewrite", processFunc: t.UpdateValues}) + } + return subtransforms, nil +} + +func (t *RewriteTransform) UpdateValues(dm *dnsutils.DNSMessage) (int, error) { + dmValue := reflect.ValueOf(dm) + if dmValue.Kind() == reflect.Ptr { + dmValue = dmValue.Elem() + } + + for nestedKeys, value := range t.config.Rewrite.Identifiers { + realValue, found := getFieldByTag(dmValue, nestedKeys) + switch { + case !found: + return 0, errors.New("field not found: " + nestedKeys) + case !realValue.CanSet(): + return 0, errors.New("field cannot be set: " + nestedKeys) + default: + newValue := reflect.ValueOf(value) + + switch realValue.Kind() { + case reflect.Int, reflect.String: + if realValue.Kind() == newValue.Kind() { + realValue.Set(newValue) + } else { + return 0, errors.New("unable to set value (" + newValue.Type().String() + ") for " + nestedKeys + "(" + realValue.Type().String() + ")") + } + default: + // Ignore unsupported types + continue + } + + } + } + + return ReturnKeep, nil +} + +func getFieldByTag(value reflect.Value, nestedKeys string) (reflect.Value, bool) { + listKeys := strings.SplitN(nestedKeys, ".", 2) + + for j, jsonKey := range listKeys { + // Iterate over the fields of the structure + for i := 0; i < value.NumField(); i++ { + field := value.Type().Field(i) + + // Get JSON tag + tag := field.Tag.Get("json") + tagClean := strings.TrimSuffix(tag, ",omitempty") + + // Check if the JSON tag matches + if tagClean == jsonKey { + switch field.Type.Kind() { + // ptr + case reflect.Ptr: + if fieldValue, found := getFieldByTag(value.Field(i).Elem(), listKeys[j+1]); found { + return fieldValue, true + } + + // struct + case reflect.Struct: + if fieldValue, found := getFieldByTag(value.Field(i), listKeys[j+1]); found { + return fieldValue, true + } + default: + return value.Field(i), true + } + } + } + } + + return reflect.Value{}, false +} diff --git a/transformers/rewrite_test.go b/transformers/rewrite_test.go new file mode 100644 index 00000000..548a6074 --- /dev/null +++ b/transformers/rewrite_test.go @@ -0,0 +1,63 @@ +package transformers + +import ( + "strings" + "testing" + + "github.com/dmachard/go-dnscollector/dnsutils" + "github.com/dmachard/go-dnscollector/pkgconfig" + "github.com/dmachard/go-logger" +) + +func TestRewrite_UpdateFields(t *testing.T) { + // enable feature + config := pkgconfig.GetFakeConfigTransformers() + config.Rewrite.Enable = true + config.Rewrite.Identifiers = make(map[string]interface{}) + config.Rewrite.Identifiers["dnstap.identity"] = "testidentity" + + // init the processor + outChans := []chan dnsutils.DNSMessage{} + rewrite := NewRewriteTransform(config, logger.New(false), "test", 0, outChans) + + // get fake + dm := dnsutils.GetFakeDNSMessage() + + rewrite.GetTransforms() + returnCode, err := rewrite.UpdateValues(&dm) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if returnCode != ReturnKeep { + t.Errorf("Return code is %v, want keep(%v)", returnCode, ReturnKeep) + } + + if dm.DNSTap.Identity != "testidentity" { + t.Errorf("Want testidentity, got %v", dm.DNSTap.Identity) + } +} + +func TestRewrite_UpdateFields_InvalidType(t *testing.T) { + // enable feature + config := pkgconfig.GetFakeConfigTransformers() + config.Rewrite.Enable = true + config.Rewrite.Identifiers = make(map[string]interface{}) + config.Rewrite.Identifiers["dnstap.identity"] = 0 + + // init the processor + outChans := []chan dnsutils.DNSMessage{} + rewrite := NewRewriteTransform(config, logger.New(false), "test", 0, outChans) + + // get fake + dm := dnsutils.GetFakeDNSMessage() + + rewrite.GetTransforms() + _, err := rewrite.UpdateValues(&dm) + if err == nil { + t.Fatalf("Expected error, got nil") + } + + if !strings.Contains(err.Error(), "unable to set value") { + t.Errorf("invalid error: %s", err) + } +} diff --git a/transformers/transformers.go b/transformers/transformers.go index ad311a5a..915c4886 100644 --- a/transformers/transformers.go +++ b/transformers/transformers.go @@ -83,6 +83,7 @@ func NewTransforms(config *pkgconfig.ConfigTransformers, logger *logger.Logger, d.availableTransforms = append(d.availableTransforms, TransformEntry{NewMachineLearningTransform(config, logger, name, instance, nextWorkers)}) d.availableTransforms = append(d.availableTransforms, TransformEntry{NewLatencyTransform(config, logger, name, instance, nextWorkers)}) d.availableTransforms = append(d.availableTransforms, TransformEntry{NewDNSGeoIPTransform(config, logger, name, instance, nextWorkers)}) + d.availableTransforms = append(d.availableTransforms, TransformEntry{NewRewriteTransform(config, logger, name, instance, nextWorkers)}) d.Prepare() return d