From e94f4bfefa2b26db3bc444d2d92c07fb4794432c Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Mon, 11 Mar 2024 14:04:05 +0100 Subject: [PATCH] add test support all type of value --- README.md | 1 + dnsutils/message.go | 101 ++++++++++++++-------- dnsutils/message_test.go | 61 ++++++++++++- docs/dnsjson.md | 4 +- docs/transformers/transform_relabeling.md | 31 +++++++ pkgconfig/transformers.go | 1 - transformers/relabel.go | 60 +++++++------ transformers/subprocessors.go | 5 +- 8 files changed, 199 insertions(+), 65 deletions(-) create mode 100644 docs/transformers/transform_relabeling.md diff --git a/README.md b/README.md index 057c572d..63f695bc 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ - **[Transformers](./docs/transformers.md)** + - Relabeling `JSON` output [Relabeling](docs/transformers/transform_relabeling.md) - Add additionnal [Tags](docs/transformers/transform_atags.md) - Traffic [Filtering](docs/transformers/transform_trafficfiltering.md) and [Reducer](docs/transformers/transform_trafficreducer.md) - Latency [Computing](docs/transformers/transform_latency.md) diff --git a/dnsutils/message.go b/dnsutils/message.go index c66e0661..18249f65 100644 --- a/dnsutils/message.go +++ b/dnsutils/message.go @@ -228,28 +228,31 @@ type TransformATags struct { Tags []string `json:"tags"` } -type TransformRelabeling struct { - Action string - Regex string +type RelabelingRule struct { + Regex *regexp.Regexp Replacement string + Action string +} + +type TransformRelabeling struct { + Rules []RelabelingRule } type DNSMessage struct { - NetworkInfo DNSNetInfo `json:"network"` - DNS DNS `json:"dns"` - EDNS DNSExtended `json:"edns"` - DNSTap DNSTap `json:"dnstap"` - Geo *TransformDNSGeo `json:"geoip,omitempty"` - PowerDNS *PowerDNS `json:"powerdns,omitempty"` - Suspicious *TransformSuspicious `json:"suspicious,omitempty"` - PublicSuffix *TransformPublicSuffix `json:"publicsuffix,omitempty"` - Extracted *TransformExtracted `json:"extracted,omitempty"` - Reducer *TransformReducer `json:"reducer,omitempty"` - MachineLearning *TransformML `json:"ml,omitempty"` - Filtering *TransformFiltering `json:"filtering,omitempty"` - ATags *TransformATags `json:"atags,omitempty"` - RelabelingRemove []TransformRelabeling `json:"-"` - RelabelingRename []TransformRelabeling `json:"-"` + NetworkInfo DNSNetInfo `json:"network"` + DNS DNS `json:"dns"` + EDNS DNSExtended `json:"edns"` + DNSTap DNSTap `json:"dnstap"` + Geo *TransformDNSGeo `json:"geoip,omitempty"` + PowerDNS *PowerDNS `json:"powerdns,omitempty"` + Suspicious *TransformSuspicious `json:"suspicious,omitempty"` + PublicSuffix *TransformPublicSuffix `json:"publicsuffix,omitempty"` + Extracted *TransformExtracted `json:"extracted,omitempty"` + Reducer *TransformReducer `json:"reducer,omitempty"` + MachineLearning *TransformML `json:"ml,omitempty"` + Filtering *TransformFiltering `json:"filtering,omitempty"` + ATags *TransformATags `json:"atags,omitempty"` + Relabeling *TransformRelabeling `json:"-"` } func (dm *DNSMessage) Init() { @@ -311,6 +314,7 @@ func (dm *DNSMessage) InitTransforms() { dm.Suspicious = &TransformSuspicious{} dm.PowerDNS = &PowerDNS{} dm.Geo = &TransformDNSGeo{} + dm.Relabeling = &TransformRelabeling{} } func (dm *DNSMessage) handleGeoIPDirectives(directive string, s *strings.Builder) error { @@ -1256,34 +1260,44 @@ func (dm *DNSMessage) Flatten() (map[string]interface{}, error) { dnsFields["powerdns.http-version"] = dm.PowerDNS.HTTPVersion } - // remove or update keys ? - for _, label := range dm.RelabelingRename { - regex := regexp.MustCompile(label.Regex) + // relabeling ? + if dm.Relabeling != nil { + err := dm.ApplyRelabeling(dnsFields) + if err != nil { + return nil, err + } + } + + return dnsFields, nil +} + +func (dm *DNSMessage) ApplyRelabeling(dnsFields map[string]interface{}) error { + + for _, label := range dm.Relabeling.Rules { + regex := label.Regex for key := range dnsFields { if regex.MatchString(key) { - if value, exists := dnsFields[label.Replacement]; exists { - if _, ok := value.([]string); ok { - dnsFields[label.Replacement] = append(dnsFields[label.Replacement].([]string), dnsFields[key].(string)) + if label.Action == "rename" { + replacement := label.Replacement + if value, exists := dnsFields[replacement]; exists { + switch v := value.(type) { + case []string: + dnsFields[replacement] = append(v, convertToString(dnsFields[key])) + default: + dnsFields[replacement] = []string{convertToString(v), convertToString(dnsFields[key])} + } } else { - dnsFields[label.Replacement] = []string{dnsFields[label.Replacement].(string), dnsFields[key].(string)} + dnsFields[replacement] = convertToString(dnsFields[key]) } - } else { - dnsFields[label.Replacement] = dnsFields[key] } - delete(dnsFields, key) - } - } - } - for _, label := range dm.RelabelingRemove { - regex := regexp.MustCompile(label.Regex) - for key := range dnsFields { - if regex.MatchString(key) { + + // delete on all case delete(dnsFields, key) } } } - return dnsFields, nil + return nil } func (dm *DNSMessage) Matching(matching map[string]interface{}) (error, bool) { @@ -1799,3 +1813,18 @@ func GetReferenceDNSMessage() DNSMessage { dm.InitTransforms() return dm } + +func convertToString(value interface{}) string { + switch v := value.(type) { + case int: + return strconv.Itoa(v) + case bool: + return strconv.FormatBool(v) + case float64: + return strconv.FormatFloat(v, 'f', -1, 64) + case string: + return v + default: + return fmt.Sprintf("%v", v) + } +} diff --git a/dnsutils/message_test.go b/dnsutils/message_test.go index 2f90c7e8..36003dea 100644 --- a/dnsutils/message_test.go +++ b/dnsutils/message_test.go @@ -3,6 +3,7 @@ package dnsutils import ( "encoding/json" "reflect" + "regexp" "strings" "testing" @@ -1403,7 +1404,65 @@ func BenchmarkDnsMessage_ToPacketLayer(b *testing.B) { } } -// Others tests +// Flatten and relabeling +func TestDnsMessage_ApplyRelabeling(t *testing.T) { + // Créer un DNSMessage avec des règles de relabeling pour le test + dm := &DNSMessage{ + Relabeling: &TransformRelabeling{ + Rules: []RelabelingRule{ + {Regex: regexp.MustCompile("^old_"), Replacement: "new_field", Action: "rename"}, + {Regex: regexp.MustCompile("^foo_"), Action: "remove"}, + }, + }, + } + + // test map + dnsFields := map[string]interface{}{ + "old_field": "value1", + "foo_field": "value2", + "other_field": "value3", + } + + // apply relabeling + err := dm.ApplyRelabeling(dnsFields) + if err != nil { + t.Errorf("ApplyRelabeling() return an error: %v", err) + } + + // check + expectedDNSFields := map[string]interface{}{ + "new_field": "value1", + "other_field": "value3", + } + if !reflect.DeepEqual(dnsFields, expectedDNSFields) { + t.Errorf("Want: %v, Get: %v", expectedDNSFields, dnsFields) + } +} + +func BenchmarkDnsMessage_ToFlatten_Relabelling(b *testing.B) { + dm := DNSMessage{} + dm.Init() + dm.InitTransforms() + + dm.Relabeling.Rules = append(dm.Relabeling.Rules, RelabelingRule{ + Regex: regexp.MustCompile(`dns.qname`), + Action: "remove", + }) + dm.Relabeling.Rules = append(dm.Relabeling.Rules, RelabelingRule{ + Regex: regexp.MustCompile(`dns.qtype`), + Replacement: "qtype", + Action: "rename", + }) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := dm.Flatten() + if err != nil { + b.Fatalf("could not flat: %v\n", err) + } + } +} + func BenchmarkDnsMessage_ToFlatten(b *testing.B) { dm := DNSMessage{} dm.Init() diff --git a/docs/dnsjson.md b/docs/dnsjson.md index 2c0003a2..bf33f9ad 100644 --- a/docs/dnsjson.md +++ b/docs/dnsjson.md @@ -3,7 +3,7 @@ The `DNS-collector` enables the transformation of DNS queries or replies into `JSON` format. The JSON format contains DNS messages with additionnal metadata added by transformers or collectors. -Main default JSON payload parts: +The default JSON payload parts: - `network`: Query/response IP and port, the protocol, and family used. - `dnstap`: Message type, arrival packet time, latency. @@ -95,6 +95,8 @@ At times, a single level key-value output in JSON is easier to ingest than multi Utilizing `flat-json` delivers every output field as its own key/value pair but requires more processing on the host running DNS-collector. +This format is recommended because custom relabeling can be applied on it (drop keys or rename it). + Here's a flat JSON output formatted using `jq`: ```json diff --git a/docs/transformers/transform_relabeling.md b/docs/transformers/transform_relabeling.md new file mode 100644 index 00000000..adc564df --- /dev/null +++ b/docs/transformers/transform_relabeling.md @@ -0,0 +1,31 @@ +# Transformer: Relabeling + +Use this transformer to remove or rename some JSON keys. +Only works on [`flat-json`](../dnsjson.md) output format. + +Configuration example + +```yaml + loggers: + - name: console + stdout: + mode: flat-json + transforms: + relabeling: + rename: + - regex: "dnstap\\.timestamp-rfc3339ns" + replacement: "timestamp" + - regex: "dns\\.qname" + replacement: "query" + - regex: "network\\.query-ip" + replacement: "client" + - regex: "network\\.response-ip" + replacement: "server" + - regex: "dnstap\\.identity" + replacement: "client_id" + - regex: "^dns\\.resource-records\\.an\\..*\\.rdata$" + replacement: "answers_rdata" + remove: + - regex: "dns" + - regex: "network" +``` \ No newline at end of file diff --git a/pkgconfig/transformers.go b/pkgconfig/transformers.go index a899370c..e7af80ab 100644 --- a/pkgconfig/transformers.go +++ b/pkgconfig/transformers.go @@ -1,7 +1,6 @@ package pkgconfig type RelabelingConfig struct { - Action string `yaml:"action"` Regex string `yaml:"regex"` Replacement string `yaml:"replacement"` } diff --git a/transformers/relabel.go b/transformers/relabel.go index 383fbf0d..81b45425 100644 --- a/transformers/relabel.go +++ b/transformers/relabel.go @@ -1,19 +1,22 @@ package transformers import ( + "regexp" + "github.com/dmachard/go-dnscollector/dnsutils" "github.com/dmachard/go-dnscollector/pkgconfig" "github.com/dmachard/go-logger" ) type RelabelProcessor struct { - config *pkgconfig.ConfigTransformers - logger *logger.Logger - name string - instance int - outChannels []chan dnsutils.DNSMessage - logInfo func(msg string, v ...interface{}) - logError func(msg string, v ...interface{}) + config *pkgconfig.ConfigTransformers + logger *logger.Logger + name string + instance int + outChannels []chan dnsutils.DNSMessage + logInfo func(msg string, v ...interface{}) + logError func(msg string, v ...interface{}) + RelabelingRules []dnsutils.RelabelingRule } func NewRelabelSubprocessor(config *pkgconfig.ConfigTransformers, logger *logger.Logger, name string, @@ -28,7 +31,7 @@ func NewRelabelSubprocessor(config *pkgconfig.ConfigTransformers, logger *logger logInfo: logInfo, logError: logError, } - + s.Precompile() return s } @@ -36,24 +39,31 @@ func (p *RelabelProcessor) ReloadConfig(config *pkgconfig.ConfigTransformers) { p.config = config } -func (p *RelabelProcessor) InitDNSMessage(dm *dnsutils.DNSMessage) {} - -func (p *RelabelProcessor) IsEnabled() bool { - return p.config.Relabeling.Enable +func (p *RelabelProcessor) Precompile() { + // Pre-compile regular expressions + for _, label := range p.config.Relabeling.Rename { + p.RelabelingRules = append(p.RelabelingRules, dnsutils.RelabelingRule{ + Regex: regexp.MustCompile(label.Regex), + Replacement: label.Replacement, + Action: "rename", + }) + } + for _, label := range p.config.Relabeling.Remove { + p.RelabelingRules = append(p.RelabelingRules, dnsutils.RelabelingRule{ + Regex: regexp.MustCompile(label.Regex), + Replacement: label.Replacement, + Action: "drop", + }) + } } - -func (p *RelabelProcessor) AddLabelConfig(dm *dnsutils.DNSMessage) int { - if p.config.Relabeling.Enable { - for _, label := range p.config.Relabeling.Rename { - dm.RelabelingRename = append(dm.RelabelingRename, dnsutils.TransformRelabeling{ - Regex: label.Regex, - Replacement: label.Replacement}) - } - for _, label := range p.config.Relabeling.Remove { - dm.RelabelingRemove = append(dm.RelabelingRemove, dnsutils.TransformRelabeling{ - Regex: label.Regex, - Replacement: label.Replacement}) +func (p *RelabelProcessor) InitDNSMessage(dm *dnsutils.DNSMessage) { + if dm.Relabeling == nil { + dm.Relabeling = &dnsutils.TransformRelabeling{ + Rules: p.RelabelingRules, } } - return ReturnSuccess +} + +func (p *RelabelProcessor) IsEnabled() bool { + return p.config.Relabeling.Enable } diff --git a/transformers/subprocessors.go b/transformers/subprocessors.go index 9995dc6e..f77f2f81 100644 --- a/transformers/subprocessors.go +++ b/transformers/subprocessors.go @@ -177,7 +177,6 @@ func (p *Transforms) Prepare() error { } if p.config.Relabeling.Enable { - p.activeTransforms = append(p.activeTransforms, p.RelabelTransform.AddLabelConfig) p.LogInfo(prefixlog + "relabeling subprocessor is enabled") } @@ -220,6 +219,10 @@ func (p *Transforms) InitDNSMessageFormat(dm *dnsutils.DNSMessage) { if p.config.ATags.Enable { p.ATagsTransform.InitDNSMessage(dm) } + + if p.config.Relabeling.Enable { + p.RelabelTransform.InitDNSMessage(dm) + } } func (p *Transforms) Reset() {