feat: new domain tracker transformer

README.md

@@ -1,7 +1,7 @@
 <p align="center">
   <img src="https://goreportcard.com/badge/github.com/dmachard/go-dns-collector" alt="Go Report"/>
   <img src="https://img.shields.io/badge/go%20version-min%201.21-green" alt="Go version"/>
-  <img src="https://img.shields.io/badge/go%20tests-510-green" alt="Go tests"/>
+  <img src="https://img.shields.io/badge/go%20tests-511-green" alt="Go tests"/>
   <img src="https://img.shields.io/badge/go%20bench-21-green" alt="Go bench"/>
   <img src="https://img.shields.io/badge/go%20lines-31977-green" alt="Go lines"/>
 </p>

pkgconfig/transformers.go

@@ -95,6 +95,11 @@ type ConfigTransformers struct {
 		Enable      bool                   `yaml:"enable" default:"false"`
 		Identifiers map[string]interface{} `yaml:"identifiers,flow"`
 	} `yaml:"rewrite"`
+	NewDomainTracker struct {
+		Enable    bool `yaml:"enable" default:"false"`
+		TTL       int  `yaml:"ttl" default:"3600"`
+		CacheSize int  `yaml:"cache-size" default:"100000"`
+	} `yaml:"new-domain-tracker"`
 }
 
 func (c *ConfigTransformers) SetDefault() {

transformers/newdomain.go

@@ -0,0 +1,103 @@
+package transformers
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/dmachard/go-dnscollector/dnsutils"
+	"github.com/dmachard/go-dnscollector/pkgconfig"
+	"github.com/dmachard/go-logger"
+	lru "github.com/hashicorp/golang-lru"
+)
+
+// NewDomainTracker transformer to detect newly observed domains
+type NewDomainTracker struct {
+	ttl      time.Duration // Time window to consider a domain as "new"
+	cache    *lru.Cache    // LRU Cache to store observed domains
+	logInfo  func(msg string, v ...interface{})
+	logError func(msg string, v ...interface{})
+}
+
+// NewNewDomainTracker initializes the NewDomainTracker transformer
+func NewNewDomainTracker(ttl time.Duration, maxSize int, logInfo, logError func(msg string, v ...interface{})) (*NewDomainTracker, error) {
+	cache, err := lru.New(maxSize)
+	if err != nil {
+		return nil, err
+	}
+
+	if ttl <= 0 {
+		return nil, fmt.Errorf("invalid TTL value: %v", ttl)
+	}
+
+	return &NewDomainTracker{
+		ttl:      ttl,
+		cache:    cache,
+		logInfo:  logInfo,
+		logError: logError,
+	}, nil
+}
+
+// IsNewDomain checks if the domain is newly observed
+func (ndt *NewDomainTracker) IsNewDomain(domain string) bool {
+	now := time.Now()
+
+	// Check if the domain exists in the cache
+	if lastSeen, exists := ndt.cache.Get(domain); exists {
+		if now.Sub(lastSeen.(time.Time)) < ndt.ttl {
+			// Domain was recently seen, not new
+			return false
+		}
+	}
+
+	// Otherwise, mark the domain as new and update the cache
+	ndt.cache.Add(domain, now)
+	return true
+}
+
+// NewDomainTransform is the Transformer for DNS messages
+type NewDomainTrackerTransform struct {
+	GenericTransformer
+	domainTracker *NewDomainTracker
+}
+
+// NewNewDomainTransform creates a new instance of the transformer
+func NewNewDomainTrackerTransform(config *pkgconfig.ConfigTransformers, logger *logger.Logger, name string, instance int, nextWorkers []chan dnsutils.DNSMessage) *NewDomainTrackerTransform {
+	t := &NewDomainTrackerTransform{GenericTransformer: NewTransformer(config, logger, "new-domain-tracker", name, instance, nextWorkers)}
+
+	// Initialize the domain tracker
+	ttl := time.Duration(config.NewDomainTracker.TTL) * time.Second
+	maxSize := config.NewDomainTracker.CacheSize
+	tracker, err := NewNewDomainTracker(ttl, maxSize, t.LogInfo, t.LogError)
+	if err != nil {
+		t.LogError("failed to initialize NewDomainTracker: %v", err)
+		return nil
+	}
+
+	t.domainTracker = tracker
+	return t
+}
+
+// ReloadConfig reloads the configuration
+func (t *NewDomainTrackerTransform) ReloadConfig(config *pkgconfig.ConfigTransformers) {
+	t.GenericTransformer.ReloadConfig(config)
+	ttl := time.Duration(config.NewDomainTracker.TTL) * time.Second
+	t.domainTracker.ttl = ttl
+	t.LogInfo("new-domain-transformer configuration reloaded")
+}
+
+func (t *NewDomainTrackerTransform) GetTransforms() ([]Subtransform, error) {
+	subtransforms := []Subtransform{}
+	if t.config.NewDomainTracker.Enable {
+		subtransforms = append(subtransforms, Subtransform{name: "new-domain-tracker:detect", processFunc: t.trackNewDomain})
+	}
+	return subtransforms, nil
+}
+
+// Process processes DNS messages and detects newly observed domains
+func (t *NewDomainTrackerTransform) trackNewDomain(dm *dnsutils.DNSMessage) (int, error) {
+	// Check if the domain is newly observed
+	if t.domainTracker.IsNewDomain(dm.DNS.Qname) {
+		return ReturnKeep, nil
+	}
+	return ReturnDrop, nil
+}

transformers/newdomain_test.go

@@ -0,0 +1,41 @@
+package transformers
+
+import (
+	"testing"
+	"time"
+
+	"github.com/dmachard/go-dnscollector/dnsutils"
+	"github.com/dmachard/go-dnscollector/pkgconfig"
+	"github.com/dmachard/go-logger"
+)
+
+func TestNewDomainTracker(t *testing.T) {
+	// config
+	config := pkgconfig.GetFakeConfigTransformers()
+	config.NewDomainTracker.Enable = true
+	config.NewDomainTracker.TTL = 2
+	config.NewDomainTracker.CacheSize = 10
+
+	outChans := []chan dnsutils.DNSMessage{}
+
+	// init subproccesor
+	tracker := NewNewDomainTrackerTransform(config, logger.New(false), "test", 0, outChans)
+
+	// first send
+	dm := dnsutils.GetFakeDNSMessage()
+	if result, _ := tracker.trackNewDomain(&dm); result != ReturnKeep {
+		t.Errorf("1. this domain should be new!!")
+	}
+	if result, _ := tracker.trackNewDomain(&dm); result != ReturnDrop {
+		t.Errorf("2. this domain should NOT be new!!")
+	}
+
+	// wait ttl for expiration
+	time.Sleep(3 * time.Second)
+
+	// recheck
+	if result, _ := tracker.trackNewDomain(&dm); result != ReturnKeep {
+		t.Errorf("3. this domain should be new!!")
+	}
+
+}

transformers/transformers.go

@@ -84,6 +84,7 @@ func NewTransforms(config *pkgconfig.ConfigTransformers, logger *logger.Logger,
 	d.availableTransforms = append(d.availableTransforms, TransformEntry{NewLatencyTransform(config, logger, name, instance, nextWorkers)})
 	d.availableTransforms = append(d.availableTransforms, TransformEntry{NewDNSGeoIPTransform(config, logger, name, instance, nextWorkers)})
 	d.availableTransforms = append(d.availableTransforms, TransformEntry{NewRewriteTransform(config, logger, name, instance, nextWorkers)})
+	d.availableTransforms = append(d.availableTransforms, TransformEntry{NewNewDomainTrackerTransform(config, logger, name, instance, nextWorkers)})
 
 	d.Prepare()
 	return d