redesign and refactor transformers

Makefile

@@ -8,7 +8,7 @@ GO_DNSTAP_PROTOBUF := 1.0.1
 GO_FRAMESTREAM := 0.10.0
 GO_CLIENTSYSLOG := 0.4.0
 GO_TOPMAP := 1.0.0
-GO_NETUTILS := 0.2.0
+GO_NETUTILS := 0.4.0
 
 BUILD_TIME := $(shell LANG=en_US date +"%F_%T_%z")
 COMMIT := $(shell git rev-parse --short HEAD)

README.md

@@ -1,9 +1,9 @@
 <p align="center">
 <img src="https://goreportcard.com/badge/github.com/dmachard/go-dns-collector" alt="Go Report"/>
 <img src="https://img.shields.io/badge/go%20version-min%201.21-green" alt="Go version"/>
-<img src="https://img.shields.io/badge/go%20tests-427-green" alt="Go tests"/>
+<img src="https://img.shields.io/badge/go%20tests-425-green" alt="Go tests"/>
 <img src="https://img.shields.io/badge/go%20bench-20-green" alt="Go bench"/>
-<img src="https://img.shields.io/badge/go%20lines-30545-green" alt="Go lines"/>
+<img src="https://img.shields.io/badge/go%20lines-29697-green" alt="Go lines"/>
 </p>
 
 <p align="center">

docs/development.md

@@ -97,32 +97,20 @@ func (c *ConfigTransformers) SetDefault() {
 }
 ```
 
-Create the following file `transformers/mytransform.go` and `loggers/mytransform_test.go`
+Create the following file `transformers/mytransform.go` and `transformers/mytransform_test.go`
 
 ```golang
 type MyTransform struct {
- config *pkgconfig.ConfigTransformers
+	GenericTransformer
 }
 
-func NewMyTransform(config *pkgconfig.ConfigTransformers) MyTransform {
- s := MyTransform{
-  config: config,
- }
-
- return s
-}
-```
-
-Declare the transfomer in the following file `subprocessor.go`
-
-```golang
-func NewTransforms(
-    d := Transforms{
-            MyTransform:     NewMyTransform(config, logger, name, outChannels),
-    }
+func MyTransform(config *pkgconfig.ConfigTransformers, logger *logger.Logger, name string, instance int, nextWorkers []chan dnsutils.DNSMessage) *MyTransform {
+	t := &MyTransform{GenericTransformer: NewTransformer(config, logger, "mytransform", name, instance, nextWorkers)}
+	return t
 }
 ```
 
+Declare the transfomer in the following file `tranformers.go`
 Finally update the docs `doc/transformers.md` and `README.md`
 
 ### Add a worker (collector or logger)

docs/transformers/transform_atags.md

@@ -7,15 +7,15 @@ In such cases, you can include a tag for identification."
 
 Options:
 
-* `tag` (list)
+* `add-tags` (list)
   > A list of string
 
 Configuration example:
 
 ```yaml
 transforms:
   atags:
-    tags: [ "TXT:google", "MX:apple" ]
+    add-tags: [ "TXT:google", "MX:apple" ]
 ```
 
 When the feature is enabled, the following json field are populated in your DNS message:

docs/transformers/transform_userprivacy.md

@@ -17,8 +17,11 @@ Options:
 * `anonymize-v6bits` (string)
   > summarize IPv6 down to the /integer level, default is `::/64`
 
-* `hash-ip` (boolean)
-  > hashes the query and response IP with the specified algorithm.
+* `hash-query-ip` (boolean)
+  > hashes the query IP with the specified algorithm.
+
+* `hash-reply-ip` (boolean)
+  > hashes the response IP with the specified algorithm.
 
 * `hash-ip-algo` (string)
   > algorithm to use for IP hashing, currently supported `sha1` (default), `sha256`, `sha512`
@@ -32,7 +35,8 @@ transforms:
     anonymize-ip: false
     anonymize-v4bits: "/16"
     anonymize-v6bits: "::/64"
-    hash-ip: false
+    hash-query-ip: false
+    hash-reply-ip: false
     hash-ip-algo: "sha1"
     minimaze-qname: false
 ```

go.mod

@@ -1,6 +1,6 @@
 module github.com/dmachard/go-dnscollector
 
-go 1.21.0
+go 1.22
 
 toolchain go1.22.1
 
@@ -13,7 +13,7 @@ require (
 	github.com/dmachard/go-dnstap-protobuf v1.0.1
 	github.com/dmachard/go-framestream v0.10.0
 	github.com/dmachard/go-logger v0.4.0
-	github.com/dmachard/go-netutils v0.2.0
+	github.com/dmachard/go-netutils v0.4.0
 	github.com/dmachard/go-powerdns-protobuf v1.1.1
 	github.com/dmachard/go-topmap v1.0.0
 	github.com/farsightsec/golang-framestream v0.3.0

go.sum

@@ -477,10 +477,10 @@ github.com/dmachard/go-framestream v0.10.0 h1:NzDOkpJOdrgV/c0XKCsVxijILbdTxsUcUl
 github.com/dmachard/go-framestream v0.10.0/go.mod h1:CiSK1RmU/7hVsM/NhsroqpBxDH3meawKIXR8x8O+LP4=
 github.com/dmachard/go-logger v0.4.0 h1:JJJW8C5Ri6OaWIECAE6dUNqLs4ym1+WX3xD6h5MxLI4=
 github.com/dmachard/go-logger v0.4.0/go.mod h1:Gf6Au3CX5l3rZ+Tb3yX31u6h4lwVeZQSBklUI3h8gCA=
-github.com/dmachard/go-netutils v0.1.0 h1:VdlUeUvpv7t4iPbNfHIKRC0itE/OwEZRl/GIdsc0mqc=
-github.com/dmachard/go-netutils v0.1.0/go.mod h1:KgAYMuJcF+1Xwtm0SlpJ4S7jBvkFghj+7tFesaHv3BY=
-github.com/dmachard/go-netutils v0.2.0 h1:jizdNnEftIPOf+f3bUg2yWorX8wDQjBfeXYPAS6zGp8=
-github.com/dmachard/go-netutils v0.2.0/go.mod h1:KgAYMuJcF+1Xwtm0SlpJ4S7jBvkFghj+7tFesaHv3BY=
+github.com/dmachard/go-netutils v0.3.0 h1:laE11IdBnNB6aGPBoVuQQ4Y7sh61bAR+yvFaXlm05H0=
+github.com/dmachard/go-netutils v0.3.0/go.mod h1:KgAYMuJcF+1Xwtm0SlpJ4S7jBvkFghj+7tFesaHv3BY=
+github.com/dmachard/go-netutils v0.4.0 h1:u28T4N1sfni0IHC6sqgYIMW8ExtBG8SsYHW7+32gfvE=
+github.com/dmachard/go-netutils v0.4.0/go.mod h1:KgAYMuJcF+1Xwtm0SlpJ4S7jBvkFghj+7tFesaHv3BY=
 github.com/dmachard/go-powerdns-protobuf v1.1.1 h1:HhgkjPGJN9QCLVFWxiTIwP3E0He8ET9uJZaT7/+6HXw=
 github.com/dmachard/go-powerdns-protobuf v1.1.1/go.mod h1:3sewpdCN4u5KpXBxrLpidHAC18v24y+f4OZ4GKfLaME=
 github.com/dmachard/go-topmap v1.0.0 h1:FzCnB80WJMSPhpEfWt/79y97XotTQjhlrsXKR6435ow=

pkgconfig/config_test.go

@@ -245,7 +245,7 @@ multiplexer:
       listen-ip: 0.0.0.0
     transforms:
       atags:
-        tags: [ "TXT:google", "MX:apple" ]
+        add-tags: [ "TXT:google", "MX:apple" ]
 `,
 			wantErr: false,
 		},

pkgconfig/transformers.go

@@ -18,7 +18,8 @@ type ConfigTransformers struct {
 		AnonymizeIPV4Bits string `yaml:"anonymize-v4bits" default:"0.0.0.0/16"`
 		AnonymizeIPV6Bits string `yaml:"anonymize-v6bits" default:"::/64"`
 		MinimazeQname     bool   `yaml:"minimaze-qname" default:"false"`
-		HashIP            bool   `yaml:"hash-ip" default:"false"`
+		HashQueryIP       bool   `yaml:"hash-query-ip" default:"false"`
+		HashReplyIP       bool   `yaml:"hash-reply-ip" default:"false"`
 		HashIPAlgo        string `yaml:"hash-ip-algo" default:"sha1"`
 	} `yaml:"user-privacy"`
 	Normalize struct {
@@ -80,8 +81,8 @@ type ConfigTransformers struct {
 		AddFeatures bool `yaml:"add-features" default:"false"`
 	} `yaml:"machine-learning"`
 	ATags struct {
-		Enable bool     `yaml:"enable" default:"false"`
-		Tags   []string `yaml:"tags,flow" default:"[]"`
+		Enable  bool     `yaml:"enable" default:"false"`
+		AddTags []string `yaml:"add-tags,flow" default:"[]"`
 	} `yaml:"atags"`
 	Relabeling struct {
 		Enable bool               `yaml:"enable" default:"false"`

transformers/atags.go

@@ -1,56 +1,33 @@
 package transformers
 
 import (
-	"fmt"
-
 	"github.com/dmachard/go-dnscollector/dnsutils"
 	"github.com/dmachard/go-dnscollector/pkgconfig"
 	"github.com/dmachard/go-logger"
 )
 
-type ATagsProcessor struct {
-	config            *pkgconfig.ConfigTransformers
-	logger            *logger.Logger
-	outChannels       []chan dnsutils.DNSMessage
-	LogInfo, LogError func(msg string, v ...interface{})
+type ATagsTransform struct {
+	GenericTransformer
 }
 
-func NewATagsTransform(config *pkgconfig.ConfigTransformers, logger *logger.Logger, name string,
-	instance int, outChannels []chan dnsutils.DNSMessage) ATagsProcessor {
-	s := ATagsProcessor{config: config, logger: logger, outChannels: outChannels}
-
-	s.LogInfo = func(msg string, v ...interface{}) {
-		log := fmt.Sprintf("transformer - [%s] conn #%d - atags - ", name, instance)
-		logger.Info(log+msg, v...)
-	}
-
-	s.LogError = func(msg string, v ...interface{}) {
-		log := fmt.Sprintf("transformer - [%s] conn #%d - atags - ", name, instance)
-		logger.Error(log+msg, v...)
-	}
-	return s
+func NewATagsTransform(config *pkgconfig.ConfigTransformers, logger *logger.Logger, name string, instance int, nextWorkers []chan dnsutils.DNSMessage) *ATagsTransform {
+	t := &ATagsTransform{GenericTransformer: NewTransformer(config, logger, "atags", name, instance, nextWorkers)}
+	return t
 }
 
-func (p *ATagsProcessor) ReloadConfig(config *pkgconfig.ConfigTransformers) {
-	p.config = config
+func (t *ATagsTransform) GetTransforms() ([]Subtransform, error) {
+	subtransforms := []Subtransform{}
+	if len(t.config.ATags.AddTags) > 0 {
+		subtransforms = append(subtransforms, Subtransform{name: "atags:add", processFunc: t.addTags})
+	}
+	return subtransforms, nil
 }
 
-func (p *ATagsProcessor) InitDNSMessage(dm *dnsutils.DNSMessage) {
+func (t *ATagsTransform) addTags(dm *dnsutils.DNSMessage) (int, error) {
 	if dm.ATags == nil {
-		dm.ATags = &dnsutils.TransformATags{
-			Tags: []string{},
-		}
-
+		dm.ATags = &dnsutils.TransformATags{Tags: []string{}}
 	}
-}
 
-func (p *ATagsProcessor) IsEnabled() bool {
-	return p.config.ATags.Enable
-}
-
-func (p *ATagsProcessor) AddTags(dm *dnsutils.DNSMessage) int {
-	if p.config.ATags.Enable {
-		dm.ATags.Tags = append(dm.ATags.Tags, p.config.ATags.Tags...)
-	}
-	return ReturnSuccess
+	dm.ATags.Tags = append(dm.ATags.Tags, t.config.ATags.AddTags...)
+	return ReturnKeep, nil
 }

transformers/atags_test.go

@@ -12,25 +12,21 @@ func TestATags_AddTag(t *testing.T) {
 	// enable feature
 	config := pkgconfig.GetFakeConfigTransformers()
 	config.ATags.Enable = true
-	config.ATags.Tags = append(config.ATags.Tags, "tag1")
-	config.ATags.Tags = append(config.ATags.Tags, "tag2")
+	config.ATags.AddTags = append(config.ATags.AddTags, "tag1")
+	config.ATags.AddTags = append(config.ATags.AddTags, "tag2")
 
 	// init the processor
 	outChans := []chan dnsutils.DNSMessage{}
-	transform := NewATagsTransform(config, logger.New(false), "test", 0, outChans)
-
-	if !transform.IsEnabled() {
-		t.Errorf("subprocessor should be enabled")
-	}
+	atags := NewATagsTransform(config, logger.New(false), "test", 0, outChans)
 
+	// add tags
 	dm := dnsutils.GetFakeDNSMessage()
+	atags.addTags(&dm)
 
-	transform.InitDNSMessage(&dm)
+	// check results
 	if dm.ATags == nil {
 		t.Errorf("DNSMessage.Atags should be not nil")
 	}
-
-	transform.AddTags(&dm)
 	if len(dm.ATags.Tags) != 2 {
 		t.Errorf("incorrect number of tag in DNSMessage")
 	}

transformers/extract.go

@@ -1,52 +1,33 @@
 package transformers
 
 import (
-	"fmt"
-
 	"github.com/dmachard/go-dnscollector/dnsutils"
 	"github.com/dmachard/go-dnscollector/pkgconfig"
 	"github.com/dmachard/go-logger"
 )
 
-type ExtractProcessor struct {
-	config            *pkgconfig.ConfigTransformers
-	logger            *logger.Logger
-	outChannels       []chan dnsutils.DNSMessage
-	LogInfo, LogError func(msg string, v ...interface{})
+type ExtractTransform struct {
+	GenericTransformer
 }
 
-func NewExtractTransform(config *pkgconfig.ConfigTransformers, logger *logger.Logger, name string,
-	instance int, outChannels []chan dnsutils.DNSMessage) ExtractProcessor {
-	s := ExtractProcessor{config: config, logger: logger, outChannels: outChannels}
-	s.LogInfo = func(msg string, v ...interface{}) {
-		log := fmt.Sprintf("transformer - [%s] conn #%d - extract - ", name, instance)
-		logger.Info(log+msg, v...)
-	}
-
-	s.LogError = func(msg string, v ...interface{}) {
-		log := fmt.Sprintf("transformer - [%s] conn #%d - extract - ", name, instance)
-		logger.Error(log+msg, v...)
-	}
-	return s
+func NewExtractTransform(config *pkgconfig.ConfigTransformers, logger *logger.Logger, name string, instance int, nextWorkers []chan dnsutils.DNSMessage) *ExtractTransform {
+	t := &ExtractTransform{GenericTransformer: NewTransformer(config, logger, "extract", name, instance, nextWorkers)}
+	return t
 }
 
-func (p *ExtractProcessor) ReloadConfig(config *pkgconfig.ConfigTransformers) {
-	p.config = config
+func (t *ExtractTransform) GetTransforms() ([]Subtransform, error) {
+	subtransforms := []Subtransform{}
+	if t.config.Extract.AddPayload {
+		subtransforms = append(subtransforms, Subtransform{name: "extract:add-base64payload", processFunc: t.addBase64Payload})
+	}
+	return subtransforms, nil
 }
 
-func (p *ExtractProcessor) InitDNSMessage(dm *dnsutils.DNSMessage) {
+func (t *ExtractTransform) addBase64Payload(dm *dnsutils.DNSMessage) (int, error) {
 	if dm.Extracted == nil {
-		dm.Extracted = &dnsutils.TransformExtracted{
-			Base64Payload: []byte("-"),
-		}
+		dm.Extracted = &dnsutils.TransformExtracted{Base64Payload: []byte("-")}
 	}
-}
-
-func (p *ExtractProcessor) IsEnabled() bool {
-	return p.config.Extract.Enable
-}
 
-func (p *ExtractProcessor) AddBase64Payload(dm *dnsutils.DNSMessage) []byte {
-	// Encode to base64 is done automatically by the json encoder ([]byte)
-	return dm.DNS.Payload
+	dm.Extracted.Base64Payload = dm.DNS.Payload
+	return ReturnKeep, nil
 }