add test

support all type of value

README.md

@@ -71,6 +71,7 @@
 
 - **[Transformers](./docs/transformers.md)**
 
+  - Relabeling `JSON` output [Relabeling](docs/transformers/transform_relabeling.md)
   - Add additionnal [Tags](docs/transformers/transform_atags.md)
   - Traffic [Filtering](docs/transformers/transform_trafficfiltering.md) and [Reducer](docs/transformers/transform_trafficreducer.md)
   - Latency [Computing](docs/transformers/transform_latency.md)

dnsutils/message.go

@@ -228,28 +228,31 @@ type TransformATags struct {
 	Tags []string `json:"tags"`
 }
 
-type TransformRelabeling struct {
-	Action      string
-	Regex       string
+type RelabelingRule struct {
+	Regex       *regexp.Regexp
 	Replacement string
+	Action      string
+}
+
+type TransformRelabeling struct {
+	Rules []RelabelingRule
 }
 
 type DNSMessage struct {
-	NetworkInfo      DNSNetInfo             `json:"network"`
-	DNS              DNS                    `json:"dns"`
-	EDNS             DNSExtended            `json:"edns"`
-	DNSTap           DNSTap                 `json:"dnstap"`
-	Geo              *TransformDNSGeo       `json:"geoip,omitempty"`
-	PowerDNS         *PowerDNS              `json:"powerdns,omitempty"`
-	Suspicious       *TransformSuspicious   `json:"suspicious,omitempty"`
-	PublicSuffix     *TransformPublicSuffix `json:"publicsuffix,omitempty"`
-	Extracted        *TransformExtracted    `json:"extracted,omitempty"`
-	Reducer          *TransformReducer      `json:"reducer,omitempty"`
-	MachineLearning  *TransformML           `json:"ml,omitempty"`
-	Filtering        *TransformFiltering    `json:"filtering,omitempty"`
-	ATags            *TransformATags        `json:"atags,omitempty"`
-	RelabelingRemove []TransformRelabeling  `json:"-"`
-	RelabelingRename []TransformRelabeling  `json:"-"`
+	NetworkInfo     DNSNetInfo             `json:"network"`
+	DNS             DNS                    `json:"dns"`
+	EDNS            DNSExtended            `json:"edns"`
+	DNSTap          DNSTap                 `json:"dnstap"`
+	Geo             *TransformDNSGeo       `json:"geoip,omitempty"`
+	PowerDNS        *PowerDNS              `json:"powerdns,omitempty"`
+	Suspicious      *TransformSuspicious   `json:"suspicious,omitempty"`
+	PublicSuffix    *TransformPublicSuffix `json:"publicsuffix,omitempty"`
+	Extracted       *TransformExtracted    `json:"extracted,omitempty"`
+	Reducer         *TransformReducer      `json:"reducer,omitempty"`
+	MachineLearning *TransformML           `json:"ml,omitempty"`
+	Filtering       *TransformFiltering    `json:"filtering,omitempty"`
+	ATags           *TransformATags        `json:"atags,omitempty"`
+	Relabeling      *TransformRelabeling   `json:"-"`
 }
 
 func (dm *DNSMessage) Init() {
@@ -311,6 +314,7 @@ func (dm *DNSMessage) InitTransforms() {
 	dm.Suspicious = &TransformSuspicious{}
 	dm.PowerDNS = &PowerDNS{}
 	dm.Geo = &TransformDNSGeo{}
+	dm.Relabeling = &TransformRelabeling{}
 }
 
 func (dm *DNSMessage) handleGeoIPDirectives(directive string, s *strings.Builder) error {
@@ -1256,34 +1260,44 @@ func (dm *DNSMessage) Flatten() (map[string]interface{}, error) {
 		dnsFields["powerdns.http-version"] = dm.PowerDNS.HTTPVersion
 	}
 
-	// remove or update keys ?
-	for _, label := range dm.RelabelingRename {
-		regex := regexp.MustCompile(label.Regex)
+	// relabeling ?
+	if dm.Relabeling != nil {
+		err := dm.ApplyRelabeling(dnsFields)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return dnsFields, nil
+}
+
+func (dm *DNSMessage) ApplyRelabeling(dnsFields map[string]interface{}) error {
+
+	for _, label := range dm.Relabeling.Rules {
+		regex := label.Regex
 		for key := range dnsFields {
 			if regex.MatchString(key) {
-				if value, exists := dnsFields[label.Replacement]; exists {
-					if _, ok := value.([]string); ok {
-						dnsFields[label.Replacement] = append(dnsFields[label.Replacement].([]string), dnsFields[key].(string))
+				if label.Action == "rename" {
+					replacement := label.Replacement
+					if value, exists := dnsFields[replacement]; exists {
+						switch v := value.(type) {
+						case []string:
+							dnsFields[replacement] = append(v, convertToString(dnsFields[key]))
+						default:
+							dnsFields[replacement] = []string{convertToString(v), convertToString(dnsFields[key])}
+						}
 					} else {
-						dnsFields[label.Replacement] = []string{dnsFields[label.Replacement].(string), dnsFields[key].(string)}
+						dnsFields[replacement] = convertToString(dnsFields[key])
 					}
-				} else {
-					dnsFields[label.Replacement] = dnsFields[key]
 				}
-				delete(dnsFields, key)
-			}
-		}
-	}
-	for _, label := range dm.RelabelingRemove {
-		regex := regexp.MustCompile(label.Regex)
-		for key := range dnsFields {
-			if regex.MatchString(key) {
+
+				// delete on all case
 				delete(dnsFields, key)
 			}
 		}
 	}
 
-	return dnsFields, nil
+	return nil
 }
 
 func (dm *DNSMessage) Matching(matching map[string]interface{}) (error, bool) {
@@ -1799,3 +1813,18 @@ func GetReferenceDNSMessage() DNSMessage {
 	dm.InitTransforms()
 	return dm
 }
+
+func convertToString(value interface{}) string {
+	switch v := value.(type) {
+	case int:
+		return strconv.Itoa(v)
+	case bool:
+		return strconv.FormatBool(v)
+	case float64:
+		return strconv.FormatFloat(v, 'f', -1, 64)
+	case string:
+		return v
+	default:
+		return fmt.Sprintf("%v", v)
+	}
+}

dnsutils/message_test.go

@@ -3,6 +3,7 @@ package dnsutils
 import (
 	"encoding/json"
 	"reflect"
+	"regexp"
 	"strings"
 	"testing"
 
@@ -1403,7 +1404,65 @@ func BenchmarkDnsMessage_ToPacketLayer(b *testing.B) {
 	}
 }
 
-// Others tests
+// Flatten and relabeling
+func TestDnsMessage_ApplyRelabeling(t *testing.T) {
+	// Créer un DNSMessage avec des règles de relabeling pour le test
+	dm := &DNSMessage{
+		Relabeling: &TransformRelabeling{
+			Rules: []RelabelingRule{
+				{Regex: regexp.MustCompile("^old_"), Replacement: "new_field", Action: "rename"},
+				{Regex: regexp.MustCompile("^foo_"), Action: "remove"},
+			},
+		},
+	}
+
+	// test map
+	dnsFields := map[string]interface{}{
+		"old_field":   "value1",
+		"foo_field":   "value2",
+		"other_field": "value3",
+	}
+
+	// apply relabeling
+	err := dm.ApplyRelabeling(dnsFields)
+	if err != nil {
+		t.Errorf("ApplyRelabeling() return an error: %v", err)
+	}
+
+	// check
+	expectedDNSFields := map[string]interface{}{
+		"new_field":   "value1",
+		"other_field": "value3",
+	}
+	if !reflect.DeepEqual(dnsFields, expectedDNSFields) {
+		t.Errorf("Want: %v, Get: %v", expectedDNSFields, dnsFields)
+	}
+}
+
+func BenchmarkDnsMessage_ToFlatten_Relabelling(b *testing.B) {
+	dm := DNSMessage{}
+	dm.Init()
+	dm.InitTransforms()
+
+	dm.Relabeling.Rules = append(dm.Relabeling.Rules, RelabelingRule{
+		Regex:  regexp.MustCompile(`dns.qname`),
+		Action: "remove",
+	})
+	dm.Relabeling.Rules = append(dm.Relabeling.Rules, RelabelingRule{
+		Regex:       regexp.MustCompile(`dns.qtype`),
+		Replacement: "qtype",
+		Action:      "rename",
+	})
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := dm.Flatten()
+		if err != nil {
+			b.Fatalf("could not flat: %v\n", err)
+		}
+	}
+}
+
 func BenchmarkDnsMessage_ToFlatten(b *testing.B) {
 	dm := DNSMessage{}
 	dm.Init()

docs/dnsjson.md

@@ -3,7 +3,7 @@
 The `DNS-collector` enables the transformation of DNS queries or replies into `JSON` format.
 The JSON format contains DNS messages with additionnal metadata added by transformers or collectors.
 
-Main default JSON payload parts:
+The default JSON payload parts:
 
 - `network`:  Query/response IP and port, the protocol, and family used.
 - `dnstap`: Message type, arrival packet time, latency.
@@ -95,6 +95,8 @@ At times, a single level key-value output in JSON is easier to ingest than multi
 Utilizing `flat-json` delivers every output field as its own key/value pair but requires more processing
 on the host running DNS-collector.
 
+This format is recommended because custom relabeling can be applied on it (drop keys or rename it).
+
 Here's a flat JSON output formatted using `jq`:
 
 ```json

docs/transformers/transform_relabeling.md

@@ -0,0 +1,31 @@
+# Transformer: Relabeling
+
+Use this transformer to remove or rename some JSON keys.
+Only works on [`flat-json`](../dnsjson.md) output format.
+
+Configuration example
+
+```yaml
+  loggers:
+    - name: console
+      stdout:
+        mode: flat-json
+      transforms:
+        relabeling:
+          rename:
+            - regex: "dnstap\\.timestamp-rfc3339ns"
+              replacement: "timestamp"
+            - regex: "dns\\.qname"
+              replacement: "query"
+            - regex: "network\\.query-ip"
+              replacement: "client"
+            - regex: "network\\.response-ip"
+              replacement: "server"
+            - regex: "dnstap\\.identity"
+              replacement: "client_id"
+            - regex: "^dns\\.resource-records\\.an\\..*\\.rdata$"
+              replacement: "answers_rdata"
+          remove:
+            - regex: "dns"
+            - regex: "network"
+```

pkgconfig/transformers.go

@@ -1,7 +1,6 @@
 package pkgconfig
 
 type RelabelingConfig struct {
-	Action      string `yaml:"action"`
 	Regex       string `yaml:"regex"`
 	Replacement string `yaml:"replacement"`
 }

transformers/relabel.go

@@ -1,19 +1,22 @@
 package transformers
 
 import (
+	"regexp"
+
 	"github.com/dmachard/go-dnscollector/dnsutils"
 	"github.com/dmachard/go-dnscollector/pkgconfig"
 	"github.com/dmachard/go-logger"
 )
 
 type RelabelProcessor struct {
-	config      *pkgconfig.ConfigTransformers
-	logger      *logger.Logger
-	name        string
-	instance    int
-	outChannels []chan dnsutils.DNSMessage
-	logInfo     func(msg string, v ...interface{})
-	logError    func(msg string, v ...interface{})
+	config          *pkgconfig.ConfigTransformers
+	logger          *logger.Logger
+	name            string
+	instance        int
+	outChannels     []chan dnsutils.DNSMessage
+	logInfo         func(msg string, v ...interface{})
+	logError        func(msg string, v ...interface{})
+	RelabelingRules []dnsutils.RelabelingRule
 }
 
 func NewRelabelSubprocessor(config *pkgconfig.ConfigTransformers, logger *logger.Logger, name string,
@@ -28,32 +31,39 @@ func NewRelabelSubprocessor(config *pkgconfig.ConfigTransformers, logger *logger
 		logInfo:     logInfo,
 		logError:    logError,
 	}
-
+	s.Precompile()
 	return s
 }
 
 func (p *RelabelProcessor) ReloadConfig(config *pkgconfig.ConfigTransformers) {
 	p.config = config
 }
 
-func (p *RelabelProcessor) InitDNSMessage(dm *dnsutils.DNSMessage) {}
-
-func (p *RelabelProcessor) IsEnabled() bool {
-	return p.config.Relabeling.Enable
+func (p *RelabelProcessor) Precompile() {
+	// Pre-compile regular expressions
+	for _, label := range p.config.Relabeling.Rename {
+		p.RelabelingRules = append(p.RelabelingRules, dnsutils.RelabelingRule{
+			Regex:       regexp.MustCompile(label.Regex),
+			Replacement: label.Replacement,
+			Action:      "rename",
+		})
+	}
+	for _, label := range p.config.Relabeling.Remove {
+		p.RelabelingRules = append(p.RelabelingRules, dnsutils.RelabelingRule{
+			Regex:       regexp.MustCompile(label.Regex),
+			Replacement: label.Replacement,
+			Action:      "drop",
+		})
+	}
 }
-
-func (p *RelabelProcessor) AddLabelConfig(dm *dnsutils.DNSMessage) int {
-	if p.config.Relabeling.Enable {
-		for _, label := range p.config.Relabeling.Rename {
-			dm.RelabelingRename = append(dm.RelabelingRename, dnsutils.TransformRelabeling{
-				Regex:       label.Regex,
-				Replacement: label.Replacement})
-		}
-		for _, label := range p.config.Relabeling.Remove {
-			dm.RelabelingRemove = append(dm.RelabelingRemove, dnsutils.TransformRelabeling{
-				Regex:       label.Regex,
-				Replacement: label.Replacement})
+func (p *RelabelProcessor) InitDNSMessage(dm *dnsutils.DNSMessage) {
+	if dm.Relabeling == nil {
+		dm.Relabeling = &dnsutils.TransformRelabeling{
+			Rules: p.RelabelingRules,
 		}
 	}
-	return ReturnSuccess
+}
+
+func (p *RelabelProcessor) IsEnabled() bool {
+	return p.config.Relabeling.Enable
 }

transformers/subprocessors.go

@@ -177,7 +177,6 @@ func (p *Transforms) Prepare() error {
 	}
 
 	if p.config.Relabeling.Enable {
-		p.activeTransforms = append(p.activeTransforms, p.RelabelTransform.AddLabelConfig)
 		p.LogInfo(prefixlog + "relabeling subprocessor is enabled")
 	}
 
@@ -220,6 +219,10 @@ func (p *Transforms) InitDNSMessageFormat(dm *dnsutils.DNSMessage) {
 	if p.config.ATags.Enable {
 		p.ATagsTransform.InitDNSMessage(dm)
 	}
+
+	if p.config.Relabeling.Enable {
+		p.RelabelTransform.InitDNSMessage(dm)
+	}
 }
 
 func (p *Transforms) Reset() {