From c8377667c906a224aec827b722149a03ffbe4649 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Mon, 16 Dec 2024 20:54:46 +0100 Subject: [PATCH] feat(transformer): add configurable unique fields to traffic reducer --- dnsutils/dnsmessage_matching.go | 10 +++--- docs/transformers/transform_trafficreducer.md | 34 ++++++++++++------- pkgconfig/transformers.go | 9 ++--- transformers/reducer.go | 17 +++++++--- 4 files changed, 44 insertions(+), 26 deletions(-) diff --git a/dnsutils/dnsmessage_matching.go b/dnsutils/dnsmessage_matching.go index c4d3ec2d..374be4c4 100644 --- a/dnsutils/dnsmessage_matching.go +++ b/dnsutils/dnsmessage_matching.go @@ -22,7 +22,7 @@ func (dm *DNSMessage) Matching(matching map[string]interface{}) (error, bool) { var isMatch = true for nestedKeys, value := range matching { - realValue, found := getFieldByJSONTag(dmValue, nestedKeys) + realValue, found := GetFieldByJSONTag(dmValue, nestedKeys) if !found { return nil, false } @@ -429,7 +429,7 @@ func matchUserPattern(realValue, expectedValue reflect.Value) (bool, error) { } // getFieldByJSONTag retrieves a field value from a struct based on JSON tags. -func getFieldByJSONTag(value reflect.Value, nestedKeys string) (reflect.Value, bool) { +func GetFieldByJSONTag(value reflect.Value, nestedKeys string) (reflect.Value, bool) { listKeys := strings.SplitN(nestedKeys, ".", 2) jsonKey := listKeys[0] var remainingKeys string @@ -464,18 +464,18 @@ func getFieldByJSONTag(value reflect.Value, nestedKeys string) (reflect.Value, b // Recurse into structs or handle slices switch fieldValue.Kind() { case reflect.Struct: - return getFieldByJSONTag(fieldValue, remainingKeys) + return GetFieldByJSONTag(fieldValue, remainingKeys) case reflect.Slice: if sliceElem, leftKey, found := getSliceElement(fieldValue, remainingKeys); found { // Handle the slice element based on its kind switch sliceElem.Kind() { case reflect.Struct: - return getFieldByJSONTag(sliceElem, leftKey) + return GetFieldByJSONTag(sliceElem, leftKey) case reflect.Slice, reflect.Array: var result []interface{} for i := 0; i < sliceElem.Len(); i++ { if subElem := sliceElem.Index(i); subElem.Kind() == reflect.Struct { - if nestedValue, found := getFieldByJSONTag(subElem, leftKey); found { + if nestedValue, found := GetFieldByJSONTag(subElem, leftKey); found { result = append(result, nestedValue.Interface()) } } else { diff --git a/docs/transformers/transform_trafficreducer.md b/docs/transformers/transform_trafficreducer.md index 196b3dc1..d0a54ccb 100644 --- a/docs/transformers/transform_trafficreducer.md +++ b/docs/transformers/transform_trafficreducer.md @@ -2,26 +2,30 @@ # Transformer: Traffic Reducer Use this transformer to detect repetitive traffic. -A query or reply is repeated when the following criterias are the same. +A query or reply is considered repeated when the specified criteria match. -The following criterias are used: +The following criteria can be configured for detecting repetitions (default one): -- server identity -- operation -- qname or qname+1 -- query ip -- qtype +- Server identity +- Operation +- Qname or Qname+1 +- Query IP +- Qtype Options: * `repetitive-traffic-detector` (boolean) - > detect repetitive traffic + > Detect repetitive traffic * `qname-plus-one` (boolean) - > use qname+1 instead of the complete one + > Use qname+1 instead of the full Qname for matching. * `watch-interval` (integer) - > watch interval in seconds + > Interval in seconds to aggregate and process the traffic. + +* `unique-fields` (array of strings) + > Define custom fields for uniqueness matching. This allows greater flexibility in detecting repetitive traffic. + > Complete list of [fields](../dnsconversions.md#json-encoding) available. Default values: @@ -31,12 +35,18 @@ transforms: repetitive-traffic-detector: true qname-plus-one: false watch-interval: 5 + unique-fields: + - dnstap.identity + - dnstap.operation + - network.query-ip + - dns.qname + - dns.qtype ``` -Specific text directive(s) available for the text format: +Specific directives available for the text output format: * `reducer-occurrences`: display the number of detected duplication -* `cumulative-length`: sum of the length of each occurrences +* `cumulative-length`: sums the lengths of all occurrences. When the feature is enabled, the following json field are populated in your DNS message: diff --git a/pkgconfig/transformers.go b/pkgconfig/transformers.go index a6c83dc3..47b148b7 100644 --- a/pkgconfig/transformers.go +++ b/pkgconfig/transformers.go @@ -38,10 +38,11 @@ type ConfigTransformers struct { QueriesTimeout int `yaml:"queries-timeout" default:"2"` } `yaml:"latency"` Reducer struct { - Enable bool `yaml:"enable" default:"false"` - RepetitiveTrafficDetector bool `yaml:"repetitive-traffic-detector" default:"false"` - QnamePlusOne bool `yaml:"qname-plus-one" default:"false"` - WatchInterval int `yaml:"watch-interval" default:"5"` + Enable bool `yaml:"enable" default:"false"` + RepetitiveTrafficDetector bool `yaml:"repetitive-traffic-detector" default:"false"` + QnamePlusOne bool `yaml:"qname-plus-one" default:"false"` + WatchInterval int `yaml:"watch-interval" default:"5"` + UniqueFields []string `yaml:"unique-fields" default:"[\"dnstap.identity\", \"dnstap.operation\", \"network.query-ip\", \"dns.qname\", \"dns.qtype\"]"` } `yaml:"reducer"` Filtering struct { Enable bool `yaml:"enable" default:"false"` diff --git a/transformers/reducer.go b/transformers/reducer.go index c5f75e59..5b7c1018 100644 --- a/transformers/reducer.go +++ b/transformers/reducer.go @@ -2,6 +2,8 @@ package transformers import ( "container/list" + "fmt" + "reflect" "strings" "sync" "time" @@ -133,9 +135,8 @@ func (t *ReducerTransform) repetitiveTrafficDetector(dm *dnsutils.DNSMessage) (i } t.strBuilder.Reset() - t.strBuilder.WriteString(dm.DNSTap.Identity) - t.strBuilder.WriteString(dm.DNSTap.Operation) - t.strBuilder.WriteString(dm.NetworkInfo.QueryIP) + + // update qname ? if t.config.Reducer.QnamePlusOne { qname := strings.ToLower(dm.DNS.Qname) qname = strings.TrimSuffix(qname, ".") @@ -143,8 +144,14 @@ func (t *ReducerTransform) repetitiveTrafficDetector(dm *dnsutils.DNSMessage) (i dm.DNS.Qname = etld } } - t.strBuilder.WriteString(dm.DNS.Qname) - t.strBuilder.WriteString(dm.DNS.Qtype) + + dmValue := reflect.ValueOf(dm).Elem() // Get the struct value of the DNSMessage + for _, field := range t.config.Reducer.UniqueFields { + if value, found := dnsutils.GetFieldByJSONTag(dmValue, field); found { + t.strBuilder.WriteString(fmt.Sprintf("%v", value.Interface())) // Append field value + } + } + dmTag := t.strBuilder.String() dmCopy := *dm