test atags transform

add multiple conditions in matching

collectors/dnsmessage.go

@@ -160,22 +160,24 @@ RUN_LOOP:
 			}
 
 			// matching enabled, filtering DNS messages ?
-			matched := false
+			matchedInclude := false
+			matchedGreaterThan := false
 			if len(c.config.Collectors.DNSMessage.Matching.Include) > 0 {
-				err, matched = dm.Matching(c.config.Collectors.DNSMessage.Matching.Include, "include")
+				err, matchedInclude = dm.Matching(c.config.Collectors.DNSMessage.Matching.Include, dnsutils.MatchingModeInclude)
 				if err != nil {
 					c.LogError(err.Error())
 				}
 			}
 			if len(c.config.Collectors.DNSMessage.Matching.GreaterThan) > 0 {
-				err, matched = dm.Matching(c.config.Collectors.DNSMessage.Matching.GreaterThan, "greater-than")
+				err, matchedGreaterThan = dm.Matching(c.config.Collectors.DNSMessage.Matching.GreaterThan, dnsutils.MatchingModeGreaterThan)
 				if err != nil {
 					c.LogError(err.Error())
 				}
 			}
 
 			// apply tranforms on matched packets only
 			// init dns message with additionnals parts if necessary
+			matched := matchedInclude && matchedGreaterThan
 			if matched {
 				subprocessors.InitDNSMessageFormat(&dm)
 				if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {

config.yml

@@ -95,17 +95,19 @@ pipelines:
     dnsmessage:
       matching:
         include:
-          dns.flags.ad: true
-          #dnstap.operation: "CLIENT_QUERY"
-          dnstap.operation: "CLIENT_Q.*"
+          dnstap.operation: "CLIENT_QUERY"
+          dns.qname: ".*\\.google\\.com"
         greater-than:
-          dns.length: 100
+          dns.length: 50
       policy: "drop-unmatched" #passthrough
+    transforms:
+      atags:
+        tags: [ "TAG-QUERIES" ]
     routes: [ log-queries ]
 
   - name: log-queries
     stdout:
-      mode: text
+      mode: flat-json
 
 ################################################
 # list of supported collectors

dnsutils/constant.go

@@ -15,4 +15,7 @@ const (
 	DNSTapClientQuery    = "CLIENT_QUERY"
 
 	DNSTapIdentityTest = "test_id"
+
+	MatchingModeInclude     = "include"
+	MatchingModeGreaterThan = "greater-than"
 )

dnsutils/message.go

@@ -206,6 +206,10 @@ type TransformML struct {
 	UncommonQtypes        int     `json:"uncommon-qtypes" msgpack:"uncommon-qtypes"`
 }
 
+type TransformATags struct {
+	Tags []string `json:"tags" msgpack:"tags"`
+}
+
 type DNSMessage struct {
 	NetworkInfo     DNSNetInfo             `json:"network" msgpack:"network"`
 	DNS             DNS                    `json:"dns" msgpack:"dns"`
@@ -219,6 +223,7 @@ type DNSMessage struct {
 	Reducer         *TransformReducer      `json:"reducer,omitempty" msgpack:"reducer"`
 	MachineLearning *TransformML           `json:"ml,omitempty" msgpack:"ml"`
 	Filtering       *TransformFiltering    `json:"filtering,omitempty" msgpack:"filtering"`
+	ATags           *TransformATags        `json:"atags,omitempty" msgpack:"atags"`
 }
 
 func (dm *DNSMessage) Init() {
@@ -863,7 +868,7 @@ func (dm *DNSMessage) Matching(matching map[string]interface{}, operator string)
 		reflectedValue := reflect.ValueOf(value)
 
 		switch operator {
-		case "include":
+		case MatchingModeInclude:
 			// regex support for string
 			if reflectedValue.Kind() == reflect.String {
 				pattern := regexp.MustCompile(reflectedValue.Interface().(string))
@@ -877,7 +882,7 @@ func (dm *DNSMessage) Matching(matching map[string]interface{}, operator string)
 				isMatch = false
 				break
 			}
-		case "greater-than":
+		case MatchingModeGreaterThan:
 			if reflectedValue.Kind() == reflect.Int {
 				if fieldValue.Interface().(int) < reflectedValue.Interface().(int) {
 					isMatch = false

pkgconfig/transformers.go

@@ -64,6 +64,10 @@ type ConfigTransformers struct {
 		Enable      bool `yaml:"enable"`
 		AddFeatures bool `yaml:"add-features"`
 	} `yaml:"machine-learning"`
+	ATags struct {
+		Enable bool     `yaml:"enable"`
+		Tags   []string `yaml:"tags,flow"`
+	} `yaml:"atags"`
 }
 
 func (c *ConfigTransformers) SetDefault() {
@@ -119,6 +123,9 @@ func (c *ConfigTransformers) SetDefault() {
 
 	c.MachineLearning.Enable = false
 	c.MachineLearning.AddFeatures = false
+
+	c.ATags.Enable = false
+	c.ATags.Tags = []string{}
 }
 
 func GetFakeConfigTransformers() *ConfigTransformers {

transformers/subprocessors.go

@@ -33,6 +33,7 @@ type Transforms struct {
 	ReducerTransform         *ReducerProcessor
 	ExtractProcessor         ExtractProcessor
 	MachineLearningTransform MlProcessor
+	ATagsTransform           ATagsProcessor
 
 	activeTransforms []func(dm *dnsutils.DNSMessage) int
 }
@@ -55,6 +56,7 @@ func NewTransforms(config *pkgconfig.ConfigTransformers, logger *logger.Logger,
 	d.FilteringTransform = NewFilteringProcessor(config, logger, name, instance, outChannels, d.LogInfo, d.LogError)
 	d.GeoipTransform = NewDNSGeoIPProcessor(config, logger, name, instance, outChannels, d.LogInfo, d.LogError)
 	d.MachineLearningTransform = NewMachineLearningSubprocessor(config, logger, name, instance, outChannels, d.LogInfo, d.LogError)
+	d.ATagsTransform = NewATagsSubprocessor(config, logger, name, instance, outChannels, d.LogInfo, d.LogError)
 
 	d.Prepare()
 	return d
@@ -71,6 +73,7 @@ func (p *Transforms) ReloadConfig(config *pkgconfig.ConfigTransformers) {
 	p.ReducerTransform.ReloadConfig(config)
 	p.ExtractProcessor.ReloadConfig(config)
 	p.MachineLearningTransform.ReloadConfig(config)
+	p.ATagsTransform.ReloadConfig(config)
 
 	p.Prepare()
 }
@@ -169,6 +172,11 @@ func (p *Transforms) Prepare() error {
 		p.LogInfo(prefixlog + enabled)
 	}
 
+	if p.config.ATags.Enable {
+		prefixlog := fmt.Sprintf("transformer=atags#%d - ", p.instance)
+		p.LogInfo(prefixlog + "subprocessor atags is enabled")
+	}
+
 	return nil
 }
 
@@ -204,6 +212,10 @@ func (p *Transforms) InitDNSMessageFormat(dm *dnsutils.DNSMessage) {
 	if p.config.MachineLearning.Enable {
 		p.MachineLearningTransform.InitDNSMessage(dm)
 	}
+
+	if p.config.ATags.Enable {
+		p.ATagsTransform.InitDNSMessage(dm)
+	}
 }
 
 func (p *Transforms) Reset() {
@@ -294,6 +306,11 @@ func (p *Transforms) ProcessMessage(dm *dnsutils.DNSMessage) int {
 		return ReturnDrop
 	}
 
+	// add tags
+	if p.config.ATags.Enable {
+		dm.ATags.Tags = append(dm.ATags.Tags, p.config.ATags.Tags...)
+	}
+
 	//  and finaly apply other transformation
 	var rCode int
 	for _, fn := range p.activeTransforms {

transformers/tags.go

@@ -0,0 +1,50 @@
+package transformers
+
+import (
+	"github.com/dmachard/go-dnscollector/dnsutils"
+	"github.com/dmachard/go-dnscollector/pkgconfig"
+	"github.com/dmachard/go-logger"
+)
+
+type ATagsProcessor struct {
+	config      *pkgconfig.ConfigTransformers
+	logger      *logger.Logger
+	name        string
+	instance    int
+	outChannels []chan dnsutils.DNSMessage
+	logInfo     func(msg string, v ...interface{})
+	logError    func(msg string, v ...interface{})
+}
+
+func NewATagsSubprocessor(config *pkgconfig.ConfigTransformers, logger *logger.Logger, name string,
+	instance int, outChannels []chan dnsutils.DNSMessage,
+	logInfo func(msg string, v ...interface{}), logError func(msg string, v ...interface{})) ATagsProcessor {
+	s := ATagsProcessor{
+		config:      config,
+		logger:      logger,
+		name:        name,
+		instance:    instance,
+		outChannels: outChannels,
+		logInfo:     logInfo,
+		logError:    logError,
+	}
+
+	return s
+}
+
+func (p *ATagsProcessor) ReloadConfig(config *pkgconfig.ConfigTransformers) {
+	p.config = config
+}
+
+func (p *ATagsProcessor) InitDNSMessage(dm *dnsutils.DNSMessage) {
+	if dm.ATags == nil {
+		dm.ATags = &dnsutils.TransformATags{
+			Tags: []string{},
+		}
+
+	}
+}
+
+func (p *ATagsProcessor) IsEnabled() bool {
+	return p.config.ATags.Enable
+}