diff --git a/.github/workflows/testing-go.yml b/.github/workflows/testing-go.yml
index 7c9be4ab..c3467473 100644
--- a/.github/workflows/testing-go.yml
+++ b/.github/workflows/testing-go.yml
@@ -23,7 +23,17 @@ jobs:
matrix:
os-version: ['ubuntu-22.04', 'macos-latest' ]
go-version: [ '1.20', '1.21' ]
- package: ['.', 'pkgconfig', 'dnsutils', 'collectors', 'loggers', 'transformers', 'netlib', 'processors']
+ package:
+ - '.'
+ - 'pkgconfig'
+ - 'pkglinker'
+ - 'pkgutils'
+ - 'dnsutils'
+ - 'collectors'
+ - 'loggers'
+ - 'transformers'
+ - 'netlib'
+ - 'processors'
exclude:
- os-version: macos-latest
go-version: '1.20'
diff --git a/Makefile b/Makefile
index fc139328..27191b1b 100644
--- a/Makefile
+++ b/Makefile
@@ -50,7 +50,7 @@ dep: check-go
# Builds the project using go build.
build: check-go
- CGO_ENABLED=0 go build -v -ldflags="$(LD_FLAGS)" -o ${BINARY_NAME} dnscollector.go multiplexer.go
+ CGO_ENABLED=0 go build -v -ldflags="$(LD_FLAGS)" -o ${BINARY_NAME} dnscollector.go
# Builds and runs the project.
run: build
@@ -68,7 +68,9 @@ lint:
tests: check-go
@go test -race -cover -v
@go test ./pkgconfig/ -race -cover -v
+ @go test ./pkgutils/ -race -cover -v
@go test ./dnsutils/ -race -cover -v
+ @go test ./routing/ -race -cover -v
@go test ./netlib/ -race -cover -v
@go test -timeout 30s ./transformers/ -race -cover -v
@go test -timeout 30s ./collectors/ -race -cover -v
diff --git a/README.md b/README.md
index cdb3390b..48459d79 100644
--- a/README.md
+++ b/README.md
@@ -3,8 +3,8 @@
[](https://goreportcard.com/report/dmachard/go-dns-collector)
-
-
+
+
diff --git a/collectors/dnsmessage.go b/collectors/dnsmessage.go
new file mode 100644
index 00000000..1b58e77a
--- /dev/null
+++ b/collectors/dnsmessage.go
@@ -0,0 +1,311 @@
+package collectors
+
+import (
+ "bufio"
+ "fmt"
+ "net/http"
+ "os"
+ "reflect"
+ "regexp"
+ "strings"
+
+ "github.com/dmachard/go-dnscollector/dnsutils"
+ "github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
+ "github.com/dmachard/go-dnscollector/transformers"
+ "github.com/dmachard/go-logger"
+)
+
+func isFileSource(matchSource string) bool {
+ return strings.HasPrefix(matchSource, "file://")
+}
+
+func isURLSource(matchSource string) bool {
+ return strings.HasPrefix(matchSource, "http://") || strings.HasPrefix(matchSource, "https://")
+}
+
+type MatchSource struct {
+ regexList []*regexp.Regexp
+ stringList []string
+}
+
+type DNSMessage struct {
+ doneRun chan bool
+ doneMonitor chan bool
+ stopRun chan bool
+ stopMonitor chan bool
+ config *pkgconfig.Config
+ configChan chan *pkgconfig.Config
+ inputChan chan dnsutils.DNSMessage
+ logger *logger.Logger
+ name string
+ RoutingHandler pkgutils.RoutingHandler
+}
+
+func NewDNSMessage(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *DNSMessage {
+ logger.Info("[%s] collector=dnsmessage - enabled", name)
+ s := &DNSMessage{
+ doneRun: make(chan bool),
+ doneMonitor: make(chan bool),
+ stopRun: make(chan bool),
+ stopMonitor: make(chan bool),
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ inputChan: make(chan dnsutils.DNSMessage, config.Collectors.DNSMessage.ChannelBufferSize),
+ logger: logger,
+ name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
+ }
+ s.ReadConfig()
+ return s
+}
+
+func (c *DNSMessage) GetName() string { return c.name }
+
+func (c *DNSMessage) AddDroppedRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDroppedRoute(wrk)
+}
+
+func (c *DNSMessage) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDefaultRoute(wrk)
+}
+
+// deprecated function
+func (c *DNSMessage) SetLoggers(loggers []pkgutils.Worker) {}
+
+// deprecated function
+func (c *DNSMessage) Loggers() ([]chan dnsutils.DNSMessage, []string) {
+ return nil, nil
+}
+
+func (c *DNSMessage) ReadConfigMatching(value interface{}) {
+ reflectedValue := reflect.ValueOf(value)
+ if reflectedValue.Kind() == reflect.Map {
+ keys := reflectedValue.MapKeys()
+ matchSrc := ""
+ srcKind := dnsutils.MatchingKindString
+ for _, k := range keys {
+ v := reflectedValue.MapIndex(k)
+ if k.Interface().(string) == "match-source" {
+ matchSrc = v.Interface().(string)
+ }
+ if k.Interface().(string) == "source-kind" {
+ srcKind = v.Interface().(string)
+ }
+ }
+ if len(matchSrc) > 0 {
+ sourceData, err := c.LoadData(matchSrc, srcKind)
+ if err != nil {
+ c.logger.Fatal(err)
+ }
+ if len(sourceData.regexList) > 0 {
+ value.(map[interface{}]interface{})[srcKind] = sourceData.regexList
+ }
+ if len(sourceData.stringList) > 0 {
+ value.(map[interface{}]interface{})[srcKind] = sourceData.stringList
+ }
+ }
+ }
+}
+
+func (c *DNSMessage) GetInputChannel() chan dnsutils.DNSMessage {
+ return c.inputChan
+}
+
+func (c *DNSMessage) ReadConfig() {
+ // load external file for include
+ if len(c.config.Collectors.DNSMessage.Matching.Include) > 0 {
+ for _, value := range c.config.Collectors.DNSMessage.Matching.Include {
+ c.ReadConfigMatching(value)
+ }
+ }
+ // load external file for exclude
+ if len(c.config.Collectors.DNSMessage.Matching.Exclude) > 0 {
+ for _, value := range c.config.Collectors.DNSMessage.Matching.Exclude {
+ c.ReadConfigMatching(value)
+ }
+ }
+}
+
+func (c *DNSMessage) LoadData(matchSource string, srcKind string) (MatchSource, error) {
+ if isFileSource(matchSource) {
+ dataSource, err := c.LoadFromFile(matchSource, srcKind)
+ if err != nil {
+ c.logger.Fatal(err)
+ }
+ return dataSource, nil
+ } else if isURLSource(matchSource) {
+ dataSource, err := c.LoadFromURL(matchSource, srcKind)
+ if err != nil {
+ c.logger.Fatal(err)
+ }
+ return dataSource, nil
+ }
+ return MatchSource{}, fmt.Errorf("match source not supported %s", matchSource)
+}
+
+func (c *DNSMessage) LoadFromURL(matchSource string, srcKind string) (MatchSource, error) {
+ c.LogInfo("loading matching source from url=%s", matchSource)
+ resp, err := http.Get(matchSource)
+ if err != nil {
+ return MatchSource{}, err
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return MatchSource{}, fmt.Errorf("invalid status code: %d", resp.StatusCode)
+ }
+
+ matchSources := MatchSource{}
+ scanner := bufio.NewScanner(resp.Body)
+
+ switch srcKind {
+ case dnsutils.MatchingKindRegexp:
+ for scanner.Scan() {
+ matchSources.regexList = append(matchSources.regexList, regexp.MustCompile(scanner.Text()))
+ }
+ c.LogInfo("remote source loaded with %d entries kind=%s", len(matchSources.regexList), srcKind)
+ case dnsutils.MatchingKindString:
+ for scanner.Scan() {
+ matchSources.stringList = append(matchSources.stringList, scanner.Text())
+ }
+ c.LogInfo("remote source loaded with %d entries kind=%s", len(matchSources.stringList), srcKind)
+ }
+
+ return matchSources, nil
+}
+
+func (c *DNSMessage) LoadFromFile(filePath string, srcKind string) (MatchSource, error) {
+ localFile := strings.TrimPrefix(filePath, "file://")
+
+ c.LogInfo("loading matching source from file=%s", localFile)
+ file, err := os.Open(localFile)
+ if err != nil {
+ return MatchSource{}, fmt.Errorf("unable to open file: %w", err)
+ }
+
+ matchSources := MatchSource{}
+ scanner := bufio.NewScanner(file)
+
+ switch srcKind {
+ case dnsutils.MatchingKindRegexp:
+ for scanner.Scan() {
+ matchSources.regexList = append(matchSources.regexList, regexp.MustCompile(scanner.Text()))
+ }
+ c.LogInfo("file loaded with %d entries kind=%s", len(matchSources.regexList), srcKind)
+ case dnsutils.MatchingKindString:
+ for scanner.Scan() {
+ matchSources.stringList = append(matchSources.stringList, scanner.Text())
+ }
+ c.LogInfo("file loaded with %d entries kind=%s", len(matchSources.stringList), srcKind)
+ }
+
+ return matchSources, nil
+}
+
+func (c *DNSMessage) ReloadConfig(config *pkgconfig.Config) {
+ c.LogInfo("reload configuration...")
+ c.configChan <- config
+}
+
+func (c *DNSMessage) LogInfo(msg string, v ...interface{}) {
+ c.logger.Info("["+c.name+"] collector=dnsmessage - "+msg, v...)
+}
+
+func (c *DNSMessage) LogError(msg string, v ...interface{}) {
+ c.logger.Error("["+c.name+"] collector=dnsmessage - "+msg, v...)
+}
+
+func (c *DNSMessage) Stop() {
+ c.LogInfo("stopping routing handler...")
+ c.RoutingHandler.Stop()
+
+ // read done channel and block until run is terminated
+ c.LogInfo("stopping run...")
+ c.stopRun <- true
+ <-c.doneRun
+}
+
+func (c *DNSMessage) Run() {
+ c.LogInfo("starting collector...")
+ var err error
+
+ // prepare next channels
+ defaultRoutes, defaultNames := c.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := c.RoutingHandler.GetDroppedRoutes()
+
+ // prepare transforms
+ subprocessors := transformers.NewTransforms(&c.config.IngoingTransformers, c.logger, c.name, defaultRoutes, 0)
+
+RUN_LOOP:
+ for {
+ select {
+ case <-c.stopRun:
+ c.doneRun <- true
+ break RUN_LOOP
+
+ case cfg := <-c.configChan:
+
+ // save the new config
+ c.config = cfg
+ c.ReadConfig()
+
+ case dm, opened := <-c.inputChan:
+ if !opened {
+ c.LogInfo("channel closed, exit")
+ return
+ }
+
+ // matching enabled, filtering DNS messages ?
+ matched := true
+ matchedInclude := false
+ matchedExclude := false
+
+ if len(c.config.Collectors.DNSMessage.Matching.Include) > 0 {
+ err, matchedInclude = dm.Matching(c.config.Collectors.DNSMessage.Matching.Include)
+ if err != nil {
+ c.LogError(err.Error())
+ }
+ if matched && matchedInclude {
+ matched = true
+ } else {
+ matched = false
+ }
+ }
+
+ if len(c.config.Collectors.DNSMessage.Matching.Exclude) > 0 {
+ err, matchedExclude = dm.Matching(c.config.Collectors.DNSMessage.Matching.Exclude)
+ if err != nil {
+ c.LogError(err.Error())
+ }
+ if matched && !matchedExclude {
+ matched = true
+ } else {
+ matched = false
+ }
+ }
+
+ // apply tranforms on matched packets only
+ // init dns message with additionnals parts if necessary
+ if matched {
+ subprocessors.InitDNSMessageFormat(&dm)
+ if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ c.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
+ continue
+ }
+ }
+
+ // drop packet ?
+ if !matched {
+ c.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
+ continue
+ }
+
+ // send to next
+ c.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
+ }
+
+ }
+ c.LogInfo("run terminated")
+}
diff --git a/collectors/dnstap.go b/collectors/dnstap.go
index fa9f0994..928ec4b7 100644
--- a/collectors/dnstap.go
+++ b/collectors/dnstap.go
@@ -15,45 +15,47 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/processors"
"github.com/dmachard/go-framestream"
"github.com/dmachard/go-logger"
)
type Dnstap struct {
- doneRun chan bool
- doneMonitor chan bool
- stopRun chan bool
- stopMonitor chan bool
- listen net.Listener
- conns []net.Conn
- sockPath string
- loggers []dnsutils.Worker
- config *pkgconfig.Config
- configChan chan *pkgconfig.Config
- logger *logger.Logger
- name string
- connMode string
- connID int
- droppedCount int
- dropped chan int
- tapProcessors []processors.DNSTapProcessor
+ doneRun chan bool
+ doneMonitor chan bool
+ stopRun chan bool
+ stopMonitor chan bool
+ listen net.Listener
+ conns []net.Conn
+ sockPath string
+ defaultRoutes []pkgutils.Worker
+ droppedRoutes []pkgutils.Worker
+ config *pkgconfig.Config
+ configChan chan *pkgconfig.Config
+ logger *logger.Logger
+ name string
+ connMode string
+ connID int
+ droppedCount int
+ droppedProcessor chan int
+ tapProcessors []processors.DNSTapProcessor
sync.RWMutex
}
-func NewDnstap(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *Dnstap {
+func NewDnstap(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *Dnstap {
logger.Info("[%s] collector=dnstap - enabled", name)
s := &Dnstap{
- doneRun: make(chan bool),
- doneMonitor: make(chan bool),
- stopRun: make(chan bool),
- stopMonitor: make(chan bool),
- dropped: make(chan int),
- config: config,
- configChan: make(chan *pkgconfig.Config),
- loggers: loggers,
- logger: logger,
- name: name,
+ doneRun: make(chan bool),
+ doneMonitor: make(chan bool),
+ stopRun: make(chan bool),
+ stopMonitor: make(chan bool),
+ droppedProcessor: make(chan int),
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ defaultRoutes: loggers,
+ logger: logger,
+ name: name,
}
s.ReadConfig()
return s
@@ -61,18 +63,20 @@ func NewDnstap(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logg
func (c *Dnstap) GetName() string { return c.name }
-func (c *Dnstap) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *Dnstap) AddDroppedRoute(wrk pkgutils.Worker) {
+ c.droppedRoutes = append(c.droppedRoutes, wrk)
+}
+
+func (c *Dnstap) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.defaultRoutes = append(c.defaultRoutes, wrk)
+}
+
+func (c *Dnstap) SetLoggers(loggers []pkgutils.Worker) {
+ c.defaultRoutes = loggers
}
func (c *Dnstap) Loggers() ([]chan dnsutils.DNSMessage, []string) {
- channels := []chan dnsutils.DNSMessage{}
- names := []string{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
- names = append(names, p.GetName())
- }
- return channels, names
+ return pkgutils.GetRoutes(c.defaultRoutes)
}
func (c *Dnstap) ReadConfig() {
@@ -127,12 +131,20 @@ func (c *Dnstap) HandleConn(conn net.Conn) {
peer := conn.RemoteAddr().String()
c.LogConnInfo(connID, "new connection from %s", peer)
- // start dnstap subprocessor
- dnstapProcessor := processors.NewDNSTapProcessor(connID, c.config, c.logger, c.name, c.config.Collectors.Dnstap.ChannelBufferSize)
+ // start dnstap processor
+ dnstapProcessor := processors.NewDNSTapProcessor(
+ connID,
+ c.config,
+ c.logger,
+ c.name,
+ c.config.Collectors.Dnstap.ChannelBufferSize,
+ )
c.Lock()
c.tapProcessors = append(c.tapProcessors, dnstapProcessor)
c.Unlock()
- go dnstapProcessor.Run(c.Loggers())
+
+ // run processor
+ go dnstapProcessor.Run(c.defaultRoutes, c.droppedRoutes)
// frame stream library
r := bufio.NewReader(conn)
@@ -191,7 +203,7 @@ func (c *Dnstap) HandleConn(conn net.Conn) {
select {
case dnstapProcessor.GetChannel() <- frame.Data(): // Successful send to channel
default:
- c.dropped <- 1
+ c.droppedProcessor <- 1
}
}
@@ -216,7 +228,7 @@ func (c *Dnstap) HandleConn(conn net.Conn) {
c.LogConnInfo(connID, "connection handler terminated")
}
-func (c *Dnstap) Channel() chan dnsutils.DNSMessage {
+func (c *Dnstap) GetInputChannel() chan dnsutils.DNSMessage {
return nil
}
@@ -312,16 +324,16 @@ func (c *Dnstap) MonitorCollector() {
MONITOR_LOOP:
for {
select {
- case <-c.dropped:
+ case <-c.droppedProcessor:
c.droppedCount++
case <-c.stopMonitor:
- close(c.dropped)
+ close(c.droppedProcessor)
bufferFull.Stop()
c.doneMonitor <- true
break MONITOR_LOOP
case <-bufferFull.C:
if c.droppedCount > 0 {
- c.LogError("recv buffer is full, %d packet(s) dropped", c.droppedCount)
+ c.LogError("processor buffer is full, %d packet(s) dropped", c.droppedCount)
c.droppedCount = 0
}
bufferFull.Reset(watchInterval)
diff --git a/collectors/dnstap_proxifier.go b/collectors/dnstap_proxifier.go
index 4fdc6711..de9e8eb2 100644
--- a/collectors/dnstap_proxifier.go
+++ b/collectors/dnstap_proxifier.go
@@ -11,34 +11,38 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-framestream"
"github.com/dmachard/go-logger"
)
type DnstapProxifier struct {
- doneRun chan bool
- stopRun chan bool
- listen net.Listener
- conns []net.Conn
- sockPath string
- loggers []dnsutils.Worker
- config *pkgconfig.Config
- configChan chan *pkgconfig.Config
- logger *logger.Logger
- name string
- stopping bool
-}
-
-func NewDnstapProxifier(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *DnstapProxifier {
+ doneRun chan bool
+ stopRun chan bool
+ listen net.Listener
+ conns []net.Conn
+ sockPath string
+ defaultRoutes []pkgutils.Worker
+ droppedRoutes []pkgutils.Worker
+ config *pkgconfig.Config
+ configChan chan *pkgconfig.Config
+ logger *logger.Logger
+ name string
+ stopping bool
+ RoutingHandler pkgutils.RoutingHandler
+}
+
+func NewDnstapProxifier(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *DnstapProxifier {
logger.Info("[%s] collector=dnstaprelay - enabled", name)
s := &DnstapProxifier{
- doneRun: make(chan bool),
- stopRun: make(chan bool),
- config: config,
- configChan: make(chan *pkgconfig.Config),
- loggers: loggers,
- logger: logger,
- name: name,
+ doneRun: make(chan bool),
+ stopRun: make(chan bool),
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ defaultRoutes: loggers,
+ logger: logger,
+ name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
}
s.ReadConfig()
return s
@@ -46,14 +50,22 @@ func NewDnstapProxifier(loggers []dnsutils.Worker, config *pkgconfig.Config, log
func (c *DnstapProxifier) GetName() string { return c.name }
-func (c *DnstapProxifier) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *DnstapProxifier) AddDroppedRoute(wrk pkgutils.Worker) {
+ c.droppedRoutes = append(c.droppedRoutes, wrk)
+}
+
+func (c *DnstapProxifier) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.defaultRoutes = append(c.defaultRoutes, wrk)
+}
+
+func (c *DnstapProxifier) SetLoggers(loggers []pkgutils.Worker) {
+ c.defaultRoutes = loggers
}
func (c *DnstapProxifier) Loggers() []chan dnsutils.DNSMessage {
channels := []chan dnsutils.DNSMessage{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
+ for _, p := range c.defaultRoutes {
+ channels = append(channels, p.GetInputChannel())
}
return channels
}
@@ -130,7 +142,7 @@ func (c *DnstapProxifier) HandleConn(conn net.Conn) {
c.LogInfo("%s - connection closed\n", peer)
}
-func (c *DnstapProxifier) Channel() chan dnsutils.DNSMessage {
+func (c *DnstapProxifier) GetInputChannel() chan dnsutils.DNSMessage {
return nil
}
diff --git a/collectors/dnstap_proxifier_test.go b/collectors/dnstap_proxifier_test.go
index 9ca81d9c..48a73d38 100644
--- a/collectors/dnstap_proxifier_test.go
+++ b/collectors/dnstap_proxifier_test.go
@@ -7,10 +7,10 @@ import (
"testing"
"time"
- "github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/loggers"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/processors"
"github.com/dmachard/go-framestream"
"github.com/dmachard/go-logger"
@@ -56,7 +56,7 @@ func Test_DnstapProxifier(t *testing.T) {
config.Collectors.DnstapProxifier.SockPath = tc.address
}
- c := NewDnstapProxifier([]dnsutils.Worker{g}, config, logger.New(false), "test")
+ c := NewDnstapProxifier([]pkgutils.Worker{g}, config, logger.New(false), "test")
if err := c.Listen(); err != nil {
log.Fatal("collector dnstap relay error: ", err)
}
@@ -100,7 +100,7 @@ func Test_DnstapProxifier(t *testing.T) {
}
// waiting message in channel
- msg := <-g.Channel()
+ msg := <-g.GetInputChannel()
if len(msg.DNSTap.Payload) == 0 {
t.Errorf("DNStap payload is empty")
}
@@ -117,7 +117,7 @@ func Test_DnstapProxifier(t *testing.T) {
// config.Collectors.DnstapProxifier.ListenPort = 6100
// config.Collectors.DnstapProxifier.SockPath = "/tmp/dnscollector_relay.sock"
-// c := NewDnstapProxifier([]dnsutils.Worker{g}, config, logger.New(false), "test")
+// c := NewDnstapProxifier([]pkgutils.Worker{g}, config, logger.New(false), "test")
// if err := c.Listen(); err != nil {
// log.Fatal("collector dnstap relay tcp listening error: ", err)
// }
@@ -170,7 +170,7 @@ func Test_DnstapProxifier(t *testing.T) {
// g := loggers.NewFakeLogger()
// config := pkgconfig.GetFakeConfig()
// config.Collectors.DnstapProxifier.SockPath = "/tmp/dnscollector_relay.sock"
-// c := NewDnstapProxifier([]dnsutils.Worker{g}, config, logger.New(false), "test")
+// c := NewDnstapProxifier([]pkgutils.Worker{g}, config, logger.New(false), "test")
// if err := c.Listen(); err != nil {
// log.Fatal("collector dnstap replay unix listening error: ", err)
// }
diff --git a/collectors/dnstap_test.go b/collectors/dnstap_test.go
index 1998677b..3c790704 100644
--- a/collectors/dnstap_test.go
+++ b/collectors/dnstap_test.go
@@ -9,10 +9,10 @@ import (
"testing"
"time"
- "github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/loggers"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/processors"
"github.com/dmachard/go-framestream"
"github.com/dmachard/go-logger"
@@ -62,7 +62,7 @@ func Test_DnstapCollector(t *testing.T) {
config.Collectors.Dnstap.SockPath = tc.address
}
- c := NewDnstap([]dnsutils.Worker{g}, config, logger.New(false), "test")
+ c := NewDnstap([]pkgutils.Worker{g}, config, logger.New(false), "test")
if err := c.Listen(); err != nil {
log.Fatal("collector listening error: ", err)
}
@@ -106,7 +106,7 @@ func Test_DnstapCollector(t *testing.T) {
}
// waiting message in channel
- msg := <-g.Channel()
+ msg := <-g.GetInputChannel()
if msg.DNSTap.Operation != tc.operation {
t.Errorf("want %s, got %s", tc.operation, msg.DNSTap.Operation)
}
@@ -129,7 +129,7 @@ func Test_DnstapCollector_CloseFrameStream(t *testing.T) {
// start the collector in unix mode
g := loggers.NewFakeLogger()
- c := NewDnstap([]dnsutils.Worker{g}, config, lg, "test")
+ c := NewDnstap([]pkgutils.Worker{g}, config, lg, "test")
if err := c.Listen(); err != nil {
log.Fatal("collector listening error: ", err)
}
diff --git a/collectors/file_ingestor.go b/collectors/file_ingestor.go
index 5207a115..d2b6b697 100644
--- a/collectors/file_ingestor.go
+++ b/collectors/file_ingestor.go
@@ -14,6 +14,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/processors"
"github.com/dmachard/go-logger"
framestream "github.com/farsightsec/golang-framestream"
@@ -38,7 +39,8 @@ func IsValidMode(mode string) bool {
type FileIngestor struct {
done chan bool
exit chan bool
- loggers []dnsutils.Worker
+ droppedRoutes []pkgutils.Worker
+ defaultRoutes []pkgutils.Worker
config *pkgconfig.Config
configChan chan *pkgconfig.Config
logger *logger.Logger
@@ -52,14 +54,14 @@ type FileIngestor struct {
mu sync.Mutex
}
-func NewFileIngestor(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *FileIngestor {
+func NewFileIngestor(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *FileIngestor {
logger.Info("[%s] collector=fileingestor - enabled", name)
s := &FileIngestor{
done: make(chan bool),
exit: make(chan bool),
config: config,
configChan: make(chan *pkgconfig.Config),
- loggers: loggers,
+ defaultRoutes: loggers,
logger: logger,
name: name,
watcherTimers: make(map[string]*time.Timer),
@@ -70,18 +72,20 @@ func NewFileIngestor(loggers []dnsutils.Worker, config *pkgconfig.Config, logger
func (c *FileIngestor) GetName() string { return c.name }
-func (c *FileIngestor) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *FileIngestor) AddDroppedRoute(wrk pkgutils.Worker) {
+ c.droppedRoutes = append(c.droppedRoutes, wrk)
+}
+
+func (c *FileIngestor) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.defaultRoutes = append(c.defaultRoutes, wrk)
+}
+
+func (c *FileIngestor) SetLoggers(loggers []pkgutils.Worker) {
+ c.defaultRoutes = loggers
}
func (c *FileIngestor) Loggers() ([]chan dnsutils.DNSMessage, []string) {
- channels := []chan dnsutils.DNSMessage{}
- names := []string{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
- names = append(names, p.GetName())
- }
- return channels, names
+ return pkgutils.GetRoutes(c.defaultRoutes)
}
func (c *FileIngestor) ReadConfig() {
@@ -110,7 +114,7 @@ func (c *FileIngestor) LogError(msg string, v ...interface{}) {
c.logger.Error("["+c.name+"] collector=fileingestor - "+msg, v...)
}
-func (c *FileIngestor) Channel() chan dnsutils.DNSMessage {
+func (c *FileIngestor) GetInputChannel() chan dnsutils.DNSMessage {
return nil
}
@@ -374,11 +378,17 @@ func (c *FileIngestor) Run() {
c.LogInfo("starting collector...")
c.dnsProcessor = processors.NewDNSProcessor(c.config, c.logger, c.name, c.config.Collectors.FileIngestor.ChannelBufferSize)
- go c.dnsProcessor.Run(c.Loggers())
+ go c.dnsProcessor.Run(c.defaultRoutes, c.droppedRoutes)
// start dnstap subprocessor
- c.dnstapProcessor = processors.NewDNSTapProcessor(0, c.config, c.logger, c.name, c.config.Collectors.FileIngestor.ChannelBufferSize)
- go c.dnstapProcessor.Run(c.Loggers())
+ c.dnstapProcessor = processors.NewDNSTapProcessor(
+ 0,
+ c.config,
+ c.logger,
+ c.name,
+ c.config.Collectors.FileIngestor.ChannelBufferSize,
+ )
+ go c.dnstapProcessor.Run(c.defaultRoutes, c.droppedRoutes)
// read current folder content
entries, err := os.ReadDir(c.config.Collectors.FileIngestor.WatchDir)
diff --git a/collectors/file_ingestor_test.go b/collectors/file_ingestor_test.go
index 26a23b96..52c8e1aa 100644
--- a/collectors/file_ingestor_test.go
+++ b/collectors/file_ingestor_test.go
@@ -6,6 +6,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/loggers"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-logger"
)
@@ -17,13 +18,13 @@ func Test_FileIngestor_Pcap(t *testing.T) {
config.Collectors.FileIngestor.WatchDir = "./../testsdata/pcap/"
// init collector
- c := NewFileIngestor([]dnsutils.Worker{g}, config, logger.New(false), "test")
+ c := NewFileIngestor([]pkgutils.Worker{g}, config, logger.New(false), "test")
go c.Run()
// waiting message in channel
for {
// read dns message from channel
- msg := <-g.Channel()
+ msg := <-g.GetInputChannel()
// check qname
if msg.DNSTap.Operation == dnsutils.DNSTapClientQuery {
diff --git a/collectors/file_tail.go b/collectors/file_tail.go
index 4685273c..cd429b2d 100644
--- a/collectors/file_tail.go
+++ b/collectors/file_tail.go
@@ -11,6 +11,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
"github.com/hpcloud/tail"
@@ -18,26 +19,26 @@ import (
)
type Tail struct {
- doneRun chan bool
- stopRun chan bool
- tailf *tail.Tail
- loggers []dnsutils.Worker
- config *pkgconfig.Config
- configChan chan *pkgconfig.Config
- logger *logger.Logger
- name string
+ doneRun chan bool
+ stopRun chan bool
+ tailf *tail.Tail
+ defaultRoutes []pkgutils.Worker
+ config *pkgconfig.Config
+ configChan chan *pkgconfig.Config
+ logger *logger.Logger
+ name string
}
-func NewTail(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *Tail {
+func NewTail(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *Tail {
logger.Info("[%s] collector=tail - enabled", name)
s := &Tail{
- doneRun: make(chan bool),
- stopRun: make(chan bool),
- config: config,
- configChan: make(chan *pkgconfig.Config),
- loggers: loggers,
- logger: logger,
- name: name,
+ doneRun: make(chan bool),
+ stopRun: make(chan bool),
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ defaultRoutes: loggers,
+ logger: logger,
+ name: name,
}
s.ReadConfig()
return s
@@ -45,14 +46,22 @@ func NewTail(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger
func (c *Tail) GetName() string { return c.name }
-func (c *Tail) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *Tail) AddDroppedRoute(wrk pkgutils.Worker) {
+ // TODO
+}
+
+func (c *Tail) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.defaultRoutes = append(c.defaultRoutes, wrk)
+}
+
+func (c *Tail) SetLoggers(loggers []pkgutils.Worker) {
+ c.defaultRoutes = loggers
}
func (c *Tail) Loggers() []chan dnsutils.DNSMessage {
channels := []chan dnsutils.DNSMessage{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
+ for _, p := range c.defaultRoutes {
+ channels = append(channels, p.GetInputChannel())
}
return channels
}
@@ -72,7 +81,7 @@ func (c *Tail) LogError(msg string, v ...interface{}) {
c.logger.Error("["+c.name+"] collector=tail - "+msg, v...)
}
-func (c *Tail) Channel() chan dnsutils.DNSMessage {
+func (c *Tail) GetInputChannel() chan dnsutils.DNSMessage {
return nil
}
diff --git a/collectors/file_tail_test.go b/collectors/file_tail_test.go
index a758ad67..7f276637 100644
--- a/collectors/file_tail_test.go
+++ b/collectors/file_tail_test.go
@@ -7,9 +7,9 @@ import (
"testing"
"time"
- "github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/loggers"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-logger"
)
@@ -29,7 +29,7 @@ func TestTailRun(t *testing.T) {
// init collector
g := loggers.NewFakeLogger()
- c := NewTail([]dnsutils.Worker{g}, config, logger.New(false), "test")
+ c := NewTail([]pkgutils.Worker{g}, config, logger.New(false), "test")
if err := c.Follow(); err != nil {
t.Errorf("collector tail following error: %e", err)
}
@@ -45,7 +45,7 @@ func TestTailRun(t *testing.T) {
w.Flush()
// waiting message in channel
- msg := <-g.Channel()
+ msg := <-g.GetInputChannel()
if msg.DNS.Qname != "www.google.org" {
t.Errorf("want www.google.org, got %s", msg.DNS.Qname)
}
diff --git a/collectors/powerdns.go b/collectors/powerdns.go
index 57954d8f..50dd694a 100644
--- a/collectors/powerdns.go
+++ b/collectors/powerdns.go
@@ -14,6 +14,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/processors"
"github.com/dmachard/go-logger"
powerdns_protobuf "github.com/dmachard/go-powerdns-protobuf"
@@ -28,7 +29,8 @@ type ProtobufPowerDNS struct {
listen net.Listener
connID int
conns []net.Conn
- loggers []dnsutils.Worker
+ droppedRoutes []pkgutils.Worker
+ defaultRoutes []pkgutils.Worker
config *pkgconfig.Config
configChan chan *pkgconfig.Config
logger *logger.Logger
@@ -39,20 +41,20 @@ type ProtobufPowerDNS struct {
sync.RWMutex
}
-func NewProtobufPowerDNS(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *ProtobufPowerDNS {
+func NewProtobufPowerDNS(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *ProtobufPowerDNS {
logger.Info("[%s] pdns collector - enabled", name)
s := &ProtobufPowerDNS{
- doneRun: make(chan bool),
- doneMonitor: make(chan bool),
- stopRun: make(chan bool),
- stopMonitor: make(chan bool),
- cleanup: make(chan bool),
- dropped: make(chan int),
- config: config,
- configChan: make(chan *pkgconfig.Config),
- loggers: loggers,
- logger: logger,
- name: name,
+ doneRun: make(chan bool),
+ doneMonitor: make(chan bool),
+ stopRun: make(chan bool),
+ stopMonitor: make(chan bool),
+ cleanup: make(chan bool),
+ dropped: make(chan int),
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ defaultRoutes: loggers,
+ logger: logger,
+ name: name,
}
s.ReadConfig()
return s
@@ -60,18 +62,20 @@ func NewProtobufPowerDNS(loggers []dnsutils.Worker, config *pkgconfig.Config, lo
func (c *ProtobufPowerDNS) GetName() string { return c.name }
-func (c *ProtobufPowerDNS) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *ProtobufPowerDNS) AddDroppedRoute(wrk pkgutils.Worker) {
+ c.droppedRoutes = append(c.droppedRoutes, wrk)
+}
+
+func (c *ProtobufPowerDNS) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.defaultRoutes = append(c.defaultRoutes, wrk)
+}
+
+func (c *ProtobufPowerDNS) SetLoggers(loggers []pkgutils.Worker) {
+ c.defaultRoutes = loggers
}
func (c *ProtobufPowerDNS) Loggers() ([]chan dnsutils.DNSMessage, []string) {
- channels := []chan dnsutils.DNSMessage{}
- names := []string{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
- names = append(names, p.GetName())
- }
- return channels, names
+ return pkgutils.GetRoutes(c.defaultRoutes)
}
func (c *ProtobufPowerDNS) ReadConfig() {
@@ -122,7 +126,7 @@ func (c *ProtobufPowerDNS) HandleConn(conn net.Conn) {
c.Lock()
c.pdnsProcessors = append(c.pdnsProcessors, &pdnsProc)
c.Unlock()
- go pdnsProc.Run(c.Loggers())
+ go pdnsProc.Run(c.defaultRoutes, c.droppedRoutes)
r := bufio.NewReader(conn)
pbs := powerdns_protobuf.NewProtobufStream(r, conn, 5*time.Second)
@@ -186,7 +190,7 @@ func (c *ProtobufPowerDNS) HandleConn(conn net.Conn) {
c.LogConnInfo(connID, "connection handler terminated")
}
-func (c *ProtobufPowerDNS) Channel() chan dnsutils.DNSMessage {
+func (c *ProtobufPowerDNS) GetInputChannel() chan dnsutils.DNSMessage {
return nil
}
diff --git a/collectors/powerdns_test.go b/collectors/powerdns_test.go
index bf29452b..aff4196f 100644
--- a/collectors/powerdns_test.go
+++ b/collectors/powerdns_test.go
@@ -5,17 +5,17 @@ import (
"net"
"testing"
- "github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/loggers"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-logger"
)
func TestPowerDNS_Run(t *testing.T) {
g := loggers.NewFakeLogger()
- c := NewProtobufPowerDNS([]dnsutils.Worker{g}, pkgconfig.GetFakeConfig(), logger.New(false), "test")
+ c := NewProtobufPowerDNS([]pkgutils.Worker{g}, pkgconfig.GetFakeConfig(), logger.New(false), "test")
if err := c.Listen(); err != nil {
log.Fatal("collector powerdns listening error: ", err)
}
diff --git a/collectors/sniffer_afpacket.go b/collectors/sniffer_afpacket.go
index 422706b4..66411bbf 100644
--- a/collectors/sniffer_afpacket.go
+++ b/collectors/sniffer_afpacket.go
@@ -15,6 +15,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/processors"
"github.com/dmachard/go-logger"
"github.com/google/gopacket"
@@ -149,27 +150,28 @@ func RemoveBpfFilter(fd int) (err error) {
}
type AfpacketSniffer struct {
- done chan bool
- exit chan bool
- fd int
- identity string
- loggers []dnsutils.Worker
- config *pkgconfig.Config
- configChan chan *pkgconfig.Config
- logger *logger.Logger
- name string
+ done chan bool
+ exit chan bool
+ fd int
+ identity string
+ defaultRoutes []pkgutils.Worker
+ droppedRoutes []pkgutils.Worker
+ config *pkgconfig.Config
+ configChan chan *pkgconfig.Config
+ logger *logger.Logger
+ name string
}
-func NewAfpacketSniffer(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *AfpacketSniffer {
+func NewAfpacketSniffer(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *AfpacketSniffer {
logger.Info("[%s] collector=afpacket - enabled", name)
s := &AfpacketSniffer{
- done: make(chan bool),
- exit: make(chan bool),
- config: config,
- configChan: make(chan *pkgconfig.Config),
- loggers: loggers,
- logger: logger,
- name: name,
+ done: make(chan bool),
+ exit: make(chan bool),
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ defaultRoutes: loggers,
+ logger: logger,
+ name: name,
}
s.ReadConfig()
return s
@@ -185,18 +187,20 @@ func (c *AfpacketSniffer) LogError(msg string, v ...interface{}) {
func (c *AfpacketSniffer) GetName() string { return c.name }
-func (c *AfpacketSniffer) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *AfpacketSniffer) AddDroppedRoute(wrk pkgutils.Worker) {
+ c.droppedRoutes = append(c.droppedRoutes, wrk)
+}
+
+func (c *AfpacketSniffer) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.defaultRoutes = append(c.defaultRoutes, wrk)
+}
+
+func (c *AfpacketSniffer) SetLoggers(loggers []pkgutils.Worker) {
+ c.defaultRoutes = loggers
}
func (c *AfpacketSniffer) Loggers() ([]chan dnsutils.DNSMessage, []string) {
- channels := []chan dnsutils.DNSMessage{}
- names := []string{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
- names = append(names, p.GetName())
- }
- return channels, names
+ return pkgutils.GetRoutes(c.defaultRoutes)
}
func (c *AfpacketSniffer) ReadConfig() {
@@ -208,7 +212,7 @@ func (c *AfpacketSniffer) ReloadConfig(config *pkgconfig.Config) {
c.configChan <- config
}
-func (c *AfpacketSniffer) Channel() chan dnsutils.DNSMessage {
+func (c *AfpacketSniffer) GetInputChannel() chan dnsutils.DNSMessage {
return nil
}
@@ -278,8 +282,13 @@ func (c *AfpacketSniffer) Run() {
}
}
- dnsProcessor := processors.NewDNSProcessor(c.config, c.logger, c.name, c.config.Collectors.AfpacketLiveCapture.ChannelBufferSize)
- go dnsProcessor.Run(c.Loggers())
+ dnsProcessor := processors.NewDNSProcessor(
+ c.config,
+ c.logger,
+ c.name,
+ c.config.Collectors.AfpacketLiveCapture.ChannelBufferSize,
+ )
+ go dnsProcessor.Run(c.defaultRoutes, c.droppedRoutes)
dnsChan := make(chan netlib.DNSPacket)
udpChan := make(chan gopacket.Packet)
diff --git a/collectors/sniffer_afpacket_darwin.go b/collectors/sniffer_afpacket_darwin.go
index d0e06c9e..04d06084 100644
--- a/collectors/sniffer_afpacket_darwin.go
+++ b/collectors/sniffer_afpacket_darwin.go
@@ -6,28 +6,29 @@ package collectors
import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-logger"
)
type AfpacketSniffer struct {
- done chan bool
- exit chan bool
- loggers []dnsutils.Worker
- config *pkgconfig.Config
- logger *logger.Logger
- name string
+ done chan bool
+ exit chan bool
+ defaultRoutes []pkgutils.Worker
+ config *pkgconfig.Config
+ logger *logger.Logger
+ name string
}
// workaround for macos, not yet supported
-func NewAfpacketSniffer(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *AfpacketSniffer {
+func NewAfpacketSniffer(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *AfpacketSniffer {
logger.Info("[%s] AFPACKET sniffer - enabled", name)
s := &AfpacketSniffer{
- done: make(chan bool),
- exit: make(chan bool),
- config: config,
- loggers: loggers,
- logger: logger,
- name: name,
+ done: make(chan bool),
+ exit: make(chan bool),
+ config: config,
+ defaultRoutes: loggers,
+ logger: logger,
+ name: name,
}
s.ReadConfig()
return s
@@ -35,8 +36,16 @@ func NewAfpacketSniffer(loggers []dnsutils.Worker, config *pkgconfig.Config, log
func (c *AfpacketSniffer) GetName() string { return c.name }
-func (c *AfpacketSniffer) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *AfpacketSniffer) AddDroppedRoute(wrk pkgutils.Worker) {
+ // TODO
+}
+
+func (c *AfpacketSniffer) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.defaultRoutes = append(c.defaultRoutes, wrk)
+}
+
+func (c *AfpacketSniffer) SetLoggers(loggers []pkgutils.Worker) {
+ c.defaultRoutes = loggers
}
func (c *AfpacketSniffer) LogInfo(msg string, v ...interface{}) {
@@ -47,19 +56,15 @@ func (c *AfpacketSniffer) LogError(msg string, v ...interface{}) {
c.logger.Error("["+c.name+"] collector dns sniffer - "+msg, v...)
}
-func (c *AfpacketSniffer) Loggers() []chan dnsutils.DNSMessage {
- channels := []chan dnsutils.DNSMessage{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
- }
- return channels
+func (c *AfpacketSniffer) Loggers() ([]chan dnsutils.DNSMessage, []string) {
+ return pkgutils.GetRoutes(c.defaultRoutes)
}
func (c *AfpacketSniffer) ReadConfig() {}
func (c *AfpacketSniffer) ReloadConfig(config *pkgconfig.Config) {}
-func (c *AfpacketSniffer) Channel() chan dnsutils.DNSMessage {
+func (c *AfpacketSniffer) GetInputChannel() chan dnsutils.DNSMessage {
return nil
}
diff --git a/collectors/sniffer_afpacket_freebsd.go b/collectors/sniffer_afpacket_freebsd.go
index d3c70271..d5850333 100644
--- a/collectors/sniffer_afpacket_freebsd.go
+++ b/collectors/sniffer_afpacket_freebsd.go
@@ -6,28 +6,29 @@ package collectors
import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-logger"
)
type AfpacketSniffer struct {
- done chan bool
- exit chan bool
- loggers []dnsutils.Worker
- config *pkgconfig.Config
- logger *logger.Logger
- name string
+ done chan bool
+ exit chan bool
+ defaultRoutes []pkgutils.Worker
+ config *pkgconfig.Config
+ logger *logger.Logger
+ name string
}
// workaround for freebsd, not yet supported
-func NewAfpacketSniffer(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *AfpacketSniffer {
+func NewAfpacketSniffer(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *AfpacketSniffer {
logger.Info("[%s] AFPACKET sniffer - enabled", name)
s := &AfpacketSniffer{
- done: make(chan bool),
- exit: make(chan bool),
- config: config,
- loggers: loggers,
- logger: logger,
- name: name,
+ done: make(chan bool),
+ exit: make(chan bool),
+ config: config,
+ defaultRoutes: loggers,
+ logger: logger,
+ name: name,
}
s.ReadConfig()
return s
@@ -35,8 +36,16 @@ func NewAfpacketSniffer(loggers []dnsutils.Worker, config *pkgconfig.Config, log
func (c *AfpacketSniffer) GetName() string { return c.name }
-func (c *AfpacketSniffer) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *AfpacketSniffer) AddDroppedRoute(wrk pkgutils.Worker) {
+ // TODO
+}
+
+func (c *AfpacketSniffer) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.defaultRoutes = append(c.defaultRoutes, wrk)
+}
+
+func (c *AfpacketSniffer) SetLoggers(loggers []pkgutils.Worker) {
+ c.defaultRoutes = loggers
}
func (c *AfpacketSniffer) LogInfo(msg string, v ...interface{}) {
@@ -47,19 +56,15 @@ func (c *AfpacketSniffer) LogError(msg string, v ...interface{}) {
c.logger.Error("["+c.name+"] AFPACKET sniffer - "+msg, v...)
}
-func (c *AfpacketSniffer) Loggers() []chan dnsutils.DNSMessage {
- channels := []chan dnsutils.DNSMessage{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
- }
- return channels
+func (c *AfpacketSniffer) Loggers() ([]chan dnsutils.DNSMessage, []string) {
+ return pkgutils.GetRoutes(c.defaultRoutes)
}
func (c *AfpacketSniffer) ReadConfig() {}
func (c *AfpacketSniffer) ReloadConfig(config *pkgconfig.Config) {}
-func (c *AfpacketSniffer) Channel() chan dnsutils.DNSMessage {
+func (c *AfpacketSniffer) GetInputChannel() chan dnsutils.DNSMessage {
return nil
}
diff --git a/collectors/sniffer_afpacket_test.go b/collectors/sniffer_afpacket_test.go
index 11d34217..0a938bf8 100644
--- a/collectors/sniffer_afpacket_test.go
+++ b/collectors/sniffer_afpacket_test.go
@@ -11,12 +11,13 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/loggers"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-logger"
)
func TestAfpacketSnifferRun(t *testing.T) {
g := loggers.NewFakeLogger()
- c := NewAfpacketSniffer([]dnsutils.Worker{g}, pkgconfig.GetFakeConfig(), logger.New(false), "test")
+ c := NewAfpacketSniffer([]pkgutils.Worker{g}, pkgconfig.GetFakeConfig(), logger.New(false), "test")
if err := c.Listen(); err != nil {
log.Fatal("collector sniffer listening error: ", err)
}
@@ -27,7 +28,7 @@ func TestAfpacketSnifferRun(t *testing.T) {
// waiting message in channel
for {
- msg := <-g.Channel()
+ msg := <-g.GetInputChannel()
if msg.DNSTap.Operation == dnsutils.DNSTapClientQuery && msg.DNS.Qname == "dns.collector" {
break
}
diff --git a/collectors/sniffer_afpacket_windows.go b/collectors/sniffer_afpacket_windows.go
index 19c300ed..d24b8606 100644
--- a/collectors/sniffer_afpacket_windows.go
+++ b/collectors/sniffer_afpacket_windows.go
@@ -6,28 +6,29 @@ package collectors
import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-logger"
)
type AfpacketSniffer struct {
- done chan bool
- exit chan bool
- loggers []dnsutils.Worker
- config *pkgconfig.Config
- logger *logger.Logger
- name string
+ done chan bool
+ exit chan bool
+ defaultRoutes []pkgutils.Worker
+ config *pkgconfig.Config
+ logger *logger.Logger
+ name string
}
// workaround for macos, not yet supported
-func NewAfpacketSniffer(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *AfpacketSniffer {
+func NewAfpacketSniffer(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *AfpacketSniffer {
logger.Info("[%s] AFPACKET sniffer - enabled", name)
s := &AfpacketSniffer{
- done: make(chan bool),
- exit: make(chan bool),
- config: config,
- loggers: loggers,
- logger: logger,
- name: name,
+ done: make(chan bool),
+ exit: make(chan bool),
+ config: config,
+ defaultRoutes: loggers,
+ logger: logger,
+ name: name,
}
s.ReadConfig()
return s
@@ -35,8 +36,16 @@ func NewAfpacketSniffer(loggers []dnsutils.Worker, config *pkgconfig.Config, log
func (c *AfpacketSniffer) GetName() string { return c.name }
-func (c *AfpacketSniffer) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *AfpacketSniffer) AddDroppedRoute(wrk pkgutils.Worker) {
+ // TODO
+}
+
+func (c *AfpacketSniffer) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.defaultRoutes = append(c.defaultRoutes, wrk)
+}
+
+func (c *AfpacketSniffer) SetLoggers(loggers []pkgutils.Worker) {
+ c.defaultRoutes = loggers
}
func (c *AfpacketSniffer) LogInfo(msg string, v ...interface{}) {
@@ -47,19 +56,15 @@ func (c *AfpacketSniffer) LogError(msg string, v ...interface{}) {
c.logger.Error("["+c.name+"] AFPACKET sniffer - "+msg, v...)
}
-func (c *AfpacketSniffer) Loggers() []chan dnsutils.DNSMessage {
- channels := []chan dnsutils.DNSMessage{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
- }
- return channels
+func (c *AfpacketSniffer) Loggers() ([]chan dnsutils.DNSMessage, []string) {
+ return pkgutils.GetRoutes(c.defaultRoutes)
}
func (c *AfpacketSniffer) ReadConfig() {}
func (c *AfpacketSniffer) ReloadConfig(config *pkgconfig.Config) {}
-func (c *AfpacketSniffer) Channel() chan dnsutils.DNSMessage {
+func (c *AfpacketSniffer) GetInputChannel() chan dnsutils.DNSMessage {
return nil
}
diff --git a/collectors/sniffer_xdp.go b/collectors/sniffer_xdp.go
index c8baf7b8..cb5e9771 100644
--- a/collectors/sniffer_xdp.go
+++ b/collectors/sniffer_xdp.go
@@ -16,6 +16,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/processors"
"github.com/dmachard/go-dnscollector/xdp"
"github.com/dmachard/go-logger"
@@ -42,26 +43,27 @@ func ConvertIP6(ip [4]uint32) net.IP {
}
type XDPSniffer struct {
- done chan bool
- exit chan bool
- identity string
- loggers []dnsutils.Worker
- config *pkgconfig.Config
- configChan chan *pkgconfig.Config
- logger *logger.Logger
- name string
+ done chan bool
+ exit chan bool
+ identity string
+ defaultRoutes []pkgutils.Worker
+ droppedRoutes []pkgutils.Worker
+ config *pkgconfig.Config
+ configChan chan *pkgconfig.Config
+ logger *logger.Logger
+ name string
}
-func NewXDPSniffer(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *XDPSniffer {
+func NewXDPSniffer(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *XDPSniffer {
logger.Info("[%s] collector=xdp - enabled", name)
s := &XDPSniffer{
- done: make(chan bool),
- exit: make(chan bool),
- config: config,
- configChan: make(chan *pkgconfig.Config),
- loggers: loggers,
- logger: logger,
- name: name,
+ done: make(chan bool),
+ exit: make(chan bool),
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ defaultRoutes: loggers,
+ logger: logger,
+ name: name,
}
s.ReadConfig()
return s
@@ -77,18 +79,20 @@ func (c *XDPSniffer) LogError(msg string, v ...interface{}) {
func (c *XDPSniffer) GetName() string { return c.name }
-func (c *XDPSniffer) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *XDPSniffer) AddDroppedRoute(wrk pkgutils.Worker) {
+ c.droppedRoutes = append(c.droppedRoutes, wrk)
+}
+
+func (c *XDPSniffer) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.defaultRoutes = append(c.defaultRoutes, wrk)
+}
+
+func (c *XDPSniffer) SetLoggers(loggers []pkgutils.Worker) {
+ c.defaultRoutes = loggers
}
func (c *XDPSniffer) Loggers() ([]chan dnsutils.DNSMessage, []string) {
- channels := []chan dnsutils.DNSMessage{}
- names := []string{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
- names = append(names, p.GetName())
- }
- return channels, names
+ return pkgutils.GetRoutes(c.defaultRoutes)
}
func (c *XDPSniffer) ReadConfig() {
@@ -100,7 +104,7 @@ func (c *XDPSniffer) ReloadConfig(config *pkgconfig.Config) {
c.configChan <- config
}
-func (c *XDPSniffer) Channel() chan dnsutils.DNSMessage {
+func (c *XDPSniffer) GetInputChannel() chan dnsutils.DNSMessage {
return nil
}
@@ -118,8 +122,13 @@ func (c *XDPSniffer) Stop() {
func (c *XDPSniffer) Run() {
c.LogInfo("starting collector...")
- dnsProcessor := processors.NewDNSProcessor(c.config, c.logger, c.name, c.config.Collectors.XdpLiveCapture.ChannelBufferSize)
- go dnsProcessor.Run(c.Loggers())
+ dnsProcessor := processors.NewDNSProcessor(
+ c.config,
+ c.logger,
+ c.name,
+ c.config.Collectors.XdpLiveCapture.ChannelBufferSize,
+ )
+ go dnsProcessor.Run(c.defaultRoutes, c.droppedRoutes)
iface, err := net.InterfaceByName(c.config.Collectors.XdpLiveCapture.Device)
if err != nil {
diff --git a/collectors/sniffer_xdp_windows.go b/collectors/sniffer_xdp_windows.go
index b7118f88..492eb9db 100644
--- a/collectors/sniffer_xdp_windows.go
+++ b/collectors/sniffer_xdp_windows.go
@@ -6,28 +6,29 @@ package collectors
import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-logger"
)
type XDPSniffer struct {
- done chan bool
- exit chan bool
- identity string
- loggers []dnsutils.Worker
- config *pkgconfig.Config
- logger *logger.Logger
- name string
+ done chan bool
+ exit chan bool
+ identity string
+ defaultRoutes []pkgutils.Worker
+ config *pkgconfig.Config
+ logger *logger.Logger
+ name string
}
-func NewXDPSniffer(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *XDPSniffer {
+func NewXDPSniffer(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *XDPSniffer {
logger.Info("[%s] XDP collector enabled", name)
s := &XDPSniffer{
- done: make(chan bool),
- exit: make(chan bool),
- config: config,
- loggers: loggers,
- logger: logger,
- name: name,
+ done: make(chan bool),
+ exit: make(chan bool),
+ config: config,
+ defaultRoutes: loggers,
+ logger: logger,
+ name: name,
}
s.ReadConfig()
return s
@@ -43,23 +44,27 @@ func (c *XDPSniffer) LogError(msg string, v ...interface{}) {
func (c *XDPSniffer) GetName() string { return c.name }
-func (c *XDPSniffer) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *XDPSniffer) AddDroppedRoute(wrk pkgutils.Worker) {
+ // TODO
}
-func (c *XDPSniffer) Loggers() []chan dnsutils.DNSMessage {
- channels := []chan dnsutils.DNSMessage{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
- }
- return channels
+func (c *XDPSniffer) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.defaultRoutes = append(c.defaultRoutes, wrk)
+}
+
+func (c *XDPSniffer) SetLoggers(loggers []pkgutils.Worker) {
+ c.defaultRoutes = loggers
+}
+
+func (c *XDPSniffer) Loggers() ([]chan dnsutils.DNSMessage, []string) {
+ return pkgutils.GetRoutes(c.defaultRoutes)
}
func (c *XDPSniffer) ReadConfig() {}
func (c *XDPSniffer) ReloadConfig(config *pkgconfig.Config) {}
-func (c *XDPSniffer) Channel() chan dnsutils.DNSMessage {
+func (c *XDPSniffer) GetInputChannel() chan dnsutils.DNSMessage {
return nil
}
diff --git a/collectors/tzsp.go b/collectors/tzsp.go
index 91bf836e..cea1a6cd 100644
--- a/collectors/tzsp.go
+++ b/collectors/tzsp.go
@@ -14,6 +14,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/processors"
"github.com/dmachard/go-logger"
"github.com/google/gopacket"
@@ -22,25 +23,26 @@ import (
)
type TZSPSniffer struct {
- done chan bool
- exit chan bool
- listen net.UDPConn
- loggers []dnsutils.Worker
- config *pkgconfig.Config
- logger *logger.Logger
- name string
- identity string
+ done chan bool
+ exit chan bool
+ listen net.UDPConn
+ defaultRoutes []pkgutils.Worker
+ droppedRoutes []pkgutils.Worker
+ config *pkgconfig.Config
+ logger *logger.Logger
+ name string
+ identity string
}
-func NewTZSP(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *TZSPSniffer {
+func NewTZSP(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *TZSPSniffer {
logger.Info("[%s] collector=tzsp - enabled", name)
s := &TZSPSniffer{
- done: make(chan bool),
- exit: make(chan bool),
- config: config,
- loggers: loggers,
- logger: logger,
- name: name,
+ done: make(chan bool),
+ exit: make(chan bool),
+ config: config,
+ defaultRoutes: loggers,
+ logger: logger,
+ name: name,
}
s.ReadConfig()
return s
@@ -48,18 +50,20 @@ func NewTZSP(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger
func (c *TZSPSniffer) GetName() string { return c.name }
-func (c *TZSPSniffer) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *TZSPSniffer) AddDroppedRoute(wrk pkgutils.Worker) {
+ c.droppedRoutes = append(c.droppedRoutes, wrk)
+}
+
+func (c *TZSPSniffer) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.defaultRoutes = append(c.defaultRoutes, wrk)
+}
+
+func (c *TZSPSniffer) SetLoggers(loggers []pkgutils.Worker) {
+ c.defaultRoutes = loggers
}
func (c *TZSPSniffer) Loggers() ([]chan dnsutils.DNSMessage, []string) {
- channels := []chan dnsutils.DNSMessage{}
- names := []string{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
- names = append(names, p.GetName())
- }
- return channels, names
+ return pkgutils.GetRoutes(c.defaultRoutes)
}
func (c *TZSPSniffer) LogInfo(msg string, v ...interface{}) {
@@ -106,7 +110,7 @@ func (c *TZSPSniffer) Listen() error {
return nil
}
-func (c *TZSPSniffer) Channel() chan dnsutils.DNSMessage {
+func (c *TZSPSniffer) GetInputChannel() chan dnsutils.DNSMessage {
return nil
}
@@ -129,7 +133,7 @@ func (c *TZSPSniffer) Run() {
}
dnsProcessor := processors.NewDNSProcessor(c.config, c.logger, c.name, c.config.Collectors.Tzsp.ChannelBufferSize)
- go dnsProcessor.Run(c.Loggers())
+ go dnsProcessor.Run(c.defaultRoutes, c.droppedRoutes)
go func() {
buf := make([]byte, 65536)
diff --git a/collectors/tzsp_darwin.go b/collectors/tzsp_darwin.go
index 934ebc39..ccfe88d1 100644
--- a/collectors/tzsp_darwin.go
+++ b/collectors/tzsp_darwin.go
@@ -6,28 +6,29 @@ package collectors
import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-logger"
)
type TZSPSniffer struct {
- done chan bool
- exit chan bool
- loggers []dnsutils.Worker
- config *pkgconfig.Config
- logger *logger.Logger
- name string
+ done chan bool
+ exit chan bool
+ defaultRoutes []pkgutils.Worker
+ config *pkgconfig.Config
+ logger *logger.Logger
+ name string
}
// workaround for macos, not yet supported
-func NewTZSP(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *AfpacketSniffer {
+func NewTZSP(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *TZSPSniffer {
logger.Info("[%s] tzsp collector - enabled", name)
- s := &AfpacketSniffer{
- done: make(chan bool),
- exit: make(chan bool),
- config: config,
- loggers: loggers,
- logger: logger,
- name: name,
+ s := &TZSPSniffer{
+ done: make(chan bool),
+ exit: make(chan bool),
+ config: config,
+ defaultRoutes: loggers,
+ logger: logger,
+ name: name,
}
s.ReadConfig()
return s
@@ -35,8 +36,16 @@ func NewTZSP(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger
func (c *TZSPSniffer) GetName() string { return c.name }
-func (c *TZSPSniffer) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *TZSPSniffer) AddDroppedRoute(wrk pkgutils.Worker) {
+ // TODO
+}
+
+func (c *TZSPSniffer) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.defaultRoutes = append(c.defaultRoutes, wrk)
+}
+
+func (c *TZSPSniffer) SetLoggers(loggers []pkgutils.Worker) {
+ c.defaultRoutes = loggers
}
func (c *TZSPSniffer) LogInfo(msg string, v ...interface{}) {
@@ -47,19 +56,15 @@ func (c *TZSPSniffer) LogError(msg string, v ...interface{}) {
c.logger.Error("["+c.name+"] tzsp collector - "+msg, v...)
}
-func (c *TZSPSniffer) Loggers() []chan dnsutils.DNSMessage {
- channels := []chan dnsutils.DNSMessage{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
- }
- return channels
+func (c *TZSPSniffer) Loggers() ([]chan dnsutils.DNSMessage, []string) {
+ return pkgutils.GetRoutes(c.defaultRoutes)
}
func (c *TZSPSniffer) ReadConfig() {}
func (c *TZSPSniffer) ReloadConfig(config *pkgconfig.Config) {}
-func (c *TZSPSniffer) Channel() chan dnsutils.DNSMessage {
+func (c *TZSPSniffer) GetInputChannel() chan dnsutils.DNSMessage {
return nil
}
diff --git a/collectors/tzsp_freebsd.go b/collectors/tzsp_freebsd.go
index a122b584..b047a725 100644
--- a/collectors/tzsp_freebsd.go
+++ b/collectors/tzsp_freebsd.go
@@ -6,28 +6,29 @@ package collectors
import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-logger"
)
type TZSPSniffer struct {
- done chan bool
- exit chan bool
- loggers []dnsutils.Worker
- config *pkgconfig.Config
- logger *logger.Logger
- name string
+ done chan bool
+ exit chan bool
+ defaultRoutes []pkgutils.Worker
+ config *pkgconfig.Config
+ logger *logger.Logger
+ name string
}
// workaround for macos, not yet supported
-func NewTZSP(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *AfpacketSniffer {
+func NewTZSP(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *TZSPSniffer {
logger.Info("[%s] tzsp collector - enabled", name)
- s := &AfpacketSniffer{
- done: make(chan bool),
- exit: make(chan bool),
- config: config,
- loggers: loggers,
- logger: logger,
- name: name,
+ s := &TZSPSniffer{
+ done: make(chan bool),
+ exit: make(chan bool),
+ config: config,
+ defaultRoutes: loggers,
+ logger: logger,
+ name: name,
}
s.ReadConfig()
return s
@@ -35,8 +36,16 @@ func NewTZSP(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger
func (c *TZSPSniffer) GetName() string { return c.name }
-func (c *TZSPSniffer) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *TZSPSniffer) AddDroppedRoute(wrk pkgutils.Worker) {
+ // TODO
+}
+
+func (c *TZSPSniffer) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.defaultRoutes = append(c.defaultRoutes, wrk)
+}
+
+func (c *TZSPSniffer) SetLoggers(loggers []pkgutils.Worker) {
+ c.defaultRoutes = loggers
}
func (c *TZSPSniffer) LogInfo(msg string, v ...interface{}) {
@@ -47,19 +56,15 @@ func (c *TZSPSniffer) LogError(msg string, v ...interface{}) {
c.logger.Error("["+c.name+"] tzsp collector - "+msg, v...)
}
-func (c *TZSPSniffer) Loggers() []chan dnsutils.DNSMessage {
- channels := []chan dnsutils.DNSMessage{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
- }
- return channels
+func (c *TZSPSniffer) Loggers() ([]chan dnsutils.DNSMessage, []string) {
+ return pkgutils.GetRoutes(c.defaultRoutes)
}
func (c *TZSPSniffer) ReadConfig() {}
func (c *TZSPSniffer) ReloadConfig(config *pkgconfig.Config) {}
-func (c *TZSPSniffer) Channel() chan dnsutils.DNSMessage {
+func (c *TZSPSniffer) GetInputChannel() chan dnsutils.DNSMessage {
return nil
}
diff --git a/collectors/tzsp_windows.go b/collectors/tzsp_windows.go
index da9e24ab..9225c822 100644
--- a/collectors/tzsp_windows.go
+++ b/collectors/tzsp_windows.go
@@ -6,28 +6,29 @@ package collectors
import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-logger"
)
type TZSPSniffer struct {
- done chan bool
- exit chan bool
- loggers []dnsutils.Worker
- config *pkgconfig.Config
- logger *logger.Logger
- name string
+ done chan bool
+ exit chan bool
+ defaultRoutes []pkgutils.Worker
+ config *pkgconfig.Config
+ logger *logger.Logger
+ name string
}
// workaround for macos, not yet supported
-func NewTZSP(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *AfpacketSniffer {
+func NewTZSP(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *TZSPSniffer {
logger.Info("[%s] tzsp collector - enabled", name)
- s := &AfpacketSniffer{
- done: make(chan bool),
- exit: make(chan bool),
- config: config,
- loggers: loggers,
- logger: logger,
- name: name,
+ s := &TZSPSniffer{
+ done: make(chan bool),
+ exit: make(chan bool),
+ config: config,
+ defaultRoutes: loggers,
+ logger: logger,
+ name: name,
}
s.ReadConfig()
return s
@@ -35,8 +36,16 @@ func NewTZSP(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger
func (c *TZSPSniffer) GetName() string { return c.name }
-func (c *TZSPSniffer) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *TZSPSniffer) AddDroppedRoute(wrk pkgutils.Worker) {
+ // TODO
+}
+
+func (c *TZSPSniffer) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.defaultRoutes = append(c.defaultRoutes, wrk)
+}
+
+func (c *TZSPSniffer) SetLoggers(loggers []pkgutils.Worker) {
+ c.defaultRoutes = loggers
}
func (c *TZSPSniffer) LogInfo(msg string, v ...interface{}) {
@@ -47,19 +56,15 @@ func (c *TZSPSniffer) LogError(msg string, v ...interface{}) {
c.logger.Error("["+c.name+"] tzsp collector - "+msg, v...)
}
-func (c *TZSPSniffer) Loggers() []chan dnsutils.DNSMessage {
- channels := []chan dnsutils.DNSMessage{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
- }
- return channels
+func (c *TZSPSniffer) Loggers() ([]chan dnsutils.DNSMessage, []string) {
+ return pkgutils.GetRoutes(c.defaultRoutes)
}
func (c *TZSPSniffer) ReadConfig() {}
func (c *TZSPSniffer) ReloadConfig(config *pkgconfig.Config) {}
-func (c *TZSPSniffer) Channel() chan dnsutils.DNSMessage {
+func (c *TZSPSniffer) GetInputChannel() chan dnsutils.DNSMessage {
return nil
}
diff --git a/config.yml b/config.yml
index 3a7eb913..fbe356e9 100644
--- a/config.yml
+++ b/config.yml
@@ -82,6 +82,38 @@ multiplexer:
- from: [ tap ]
to: [ console ]
+# /!\ experimental, pipelling running mode /!\
+# pipelines:
+# - name: main-input
+# dnstap:
+# listen-ip: 0.0.0.0
+# listen-port: 6000
+# routing-policy:
+# default: [ filter ]
+
+# - name: filter
+# dnsmessage:
+# matching:
+# include:
+# dns.qname: "^.*\\.google\\.com$"
+# transforms:
+# atags:
+# tags: [ "google"]
+# routing-policy:
+# dropped: [ outputfile ]
+# default: [ console ]
+
+# - name: console
+# stdout:
+# mode: text
+
+# - name: outputfile
+# logfile:
+# file-path: "/tmp/dnstap.log"
+# max-size: 1000
+# max-files: 10
+# mode: flat-json
+
################################################
# list of supported collectors
################################################
diff --git a/dnscollector.go b/dnscollector.go
index 93340959..77aef849 100644
--- a/dnscollector.go
+++ b/dnscollector.go
@@ -9,6 +9,8 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkglinker"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-logger"
"github.com/natefinch/lumberjack"
"github.com/prometheus/common/version"
@@ -86,8 +88,14 @@ func main() {
// create logger
logger := logger.New(true)
- // load config
- config, err := pkgconfig.LoadConfig(configPath)
+ // get DNSMessage flat model
+ dmRef, err := dnsutils.GetFlatDNSMessage()
+ if err != nil {
+ fmt.Printf("config error: unable to get DNSMessage reference %v\n", err)
+ os.Exit(1)
+ }
+
+ config, err := pkgconfig.LoadConfig(configPath, dmRef)
if err != nil {
fmt.Printf("config error: %v\n", err)
os.Exit(1)
@@ -99,13 +107,24 @@ func main() {
logger.Info("main - starting dns-collector...")
// init active collectors and loggers
- mapLoggers := make(map[string]dnsutils.Worker)
- mapCollectors := make(map[string]dnsutils.Worker)
+ mapLoggers := make(map[string]pkgutils.Worker)
+ mapCollectors := make(map[string]pkgutils.Worker)
- // enable multiplexer mode
- if IsMuxEnabled(config) {
+ // running mode,
+ // multiplexer ?
+ if pkglinker.IsMuxEnabled(config) {
logger.Info("main - multiplexer mode enabled")
- InitMultiplexer(mapLoggers, mapCollectors, config, logger)
+ pkglinker.InitMultiplexer(mapLoggers, mapCollectors, config, logger)
+ }
+
+ // or pipeline ?
+ if len(config.Pipelines) > 0 {
+ logger.Info("main - pipelines mode enabled")
+ err := pkglinker.InitPipelines(mapLoggers, mapCollectors, config, logger)
+ if err != nil {
+ logger.Error("main - %s", err.Error())
+ os.Exit(1)
+ }
}
// Handle Ctrl-C with SIG TERM and SIGHUP
@@ -122,15 +141,15 @@ func main() {
logger.Info("main - SIGHUP received")
// read config
- err := pkgconfig.ReloadConfig(configPath, config)
+ err := pkgconfig.ReloadConfig(configPath, config, dmRef)
if err != nil {
panic(fmt.Sprintf("main - reload config error: %v", err))
}
// reload logger and multiplexer
InitLogger(logger, config)
- if IsMuxEnabled(config) {
- ReloadMultiplexer(mapLoggers, mapCollectors, config, logger)
+ if pkglinker.IsMuxEnabled(config) {
+ pkglinker.ReloadMultiplexer(mapLoggers, mapCollectors, config, logger)
}
case <-sigTerm:
diff --git a/dnsutils/constant.go b/dnsutils/constant.go
index 5e91c3a4..13d8b43b 100644
--- a/dnsutils/constant.go
+++ b/dnsutils/constant.go
@@ -16,4 +16,12 @@ const (
DNSTapClientQuery = "CLIENT_QUERY"
DNSTapIdentityTest = "test_id"
+
+ MatchingModeInclude = "include"
+ MatchingOpGreaterThan = "greater-than"
+ MatchingOpLowerThan = "lower-than"
+ MatchingOpSource = "match-source"
+ MatchingOpSourceKind = "source-kind"
+ MatchingKindString = "string_list"
+ MatchingKindRegexp = "regexp_list"
)
diff --git a/dnsutils/message.go b/dnsutils/message.go
index d4279c75..8a13b850 100644
--- a/dnsutils/message.go
+++ b/dnsutils/message.go
@@ -9,6 +9,7 @@ import (
"fmt"
"log"
"net"
+ "reflect"
"regexp"
"strconv"
"strings"
@@ -207,6 +208,10 @@ type TransformML struct {
UncommonQtypes int `json:"uncommon-qtypes" msgpack:"uncommon-qtypes"`
}
+type TransformATags struct {
+ Tags []string `json:"tags" msgpack:"tags"`
+}
+
type DNSMessage struct {
NetworkInfo DNSNetInfo `json:"network" msgpack:"network"`
DNS DNS `json:"dns" msgpack:"dns"`
@@ -220,6 +225,7 @@ type DNSMessage struct {
Reducer *TransformReducer `json:"reducer,omitempty" msgpack:"reducer"`
MachineLearning *TransformML `json:"ml,omitempty" msgpack:"ml"`
Filtering *TransformFiltering `json:"filtering,omitempty" msgpack:"filtering"`
+ ATags *TransformATags `json:"atags,omitempty" msgpack:"atags"`
}
func (dm *DNSMessage) Init() {
@@ -262,6 +268,18 @@ func (dm *DNSMessage) Init() {
}
}
+func (dm *DNSMessage) InitTransforms() {
+ dm.ATags = &TransformATags{}
+ dm.Filtering = &TransformFiltering{}
+ dm.MachineLearning = &TransformML{}
+ dm.Reducer = &TransformReducer{}
+ dm.Extracted = &TransformExtracted{}
+ dm.PublicSuffix = &TransformPublicSuffix{}
+ dm.Suspicious = &TransformSuspicious{}
+ dm.PowerDNS = &PowerDNS{}
+ dm.Geo = &TransformDNSGeo{}
+}
+
func (dm *DNSMessage) handleGeoIPDirectives(directives []string, s *strings.Builder) {
if dm.Geo == nil {
s.WriteString("-")
@@ -839,6 +857,475 @@ func (dm *DNSMessage) Flatten() (ret map[string]interface{}, err error) {
return
}
+func (dm *DNSMessage) Matching(matching map[string]interface{}) (error, bool) {
+ if len(matching) == 0 {
+ return nil, false
+ }
+
+ dmValue := reflect.ValueOf(dm)
+
+ if dmValue.Kind() == reflect.Ptr {
+ dmValue = dmValue.Elem()
+ }
+
+ var isMatch = true
+
+ for nestedKeys, value := range matching {
+ realValue, found := getFieldByJSONTag(dmValue, nestedKeys)
+ if !found {
+ return nil, false
+ }
+
+ expectedValue := reflect.ValueOf(value)
+ // fmt.Println(nestedKeys, realValue, realValue.Kind(), expectedValue.Kind())
+
+ switch expectedValue.Kind() {
+ // integer
+ case reflect.Int:
+ if match, _ := matchUserInteger(realValue, expectedValue); !match {
+ return nil, false
+ }
+
+ // string
+ case reflect.String:
+ if match, _ := matchUserPattern(realValue, expectedValue); !match {
+ return nil, false
+ }
+
+ // bool
+ case reflect.Bool:
+ if match, _ := matchUserBoolean(realValue, expectedValue); !match {
+ return nil, false
+ }
+
+ // map
+ case reflect.Map:
+ if match, _ := matchUserMap(realValue, expectedValue); !match {
+ return nil, false
+ }
+
+ // list/slice
+ case reflect.Slice:
+ if match, _ := matchUserSlice(realValue, expectedValue); !match {
+ return nil, false
+ }
+
+ // other user types
+ default:
+ return fmt.Errorf("unsupported type value: %s", expectedValue.Kind()), false
+ }
+
+ }
+
+ return nil, isMatch
+}
+
+// map can be provided by user in the config
+// dns.qname:
+// match-source: "file://./testsdata/filtering_keep_domains_regex.txt"
+// source-kind: "regexp_list"
+func matchUserMap(realValue, expectedValue reflect.Value) (bool, error) {
+ for _, opKey := range expectedValue.MapKeys() {
+ opValue := expectedValue.MapIndex(opKey)
+ opName := opKey.Interface().(string)
+
+ switch opName {
+ // Integer great than ?
+ case MatchingOpGreaterThan:
+ if _, ok := opValue.Interface().(int); !ok {
+ return false, fmt.Errorf("integer is expected for greater-than operator")
+ }
+
+ // If realValue is a slice
+ if realValue.Kind() == reflect.Slice {
+ for i := 0; i < realValue.Len(); i++ {
+ elemValue := realValue.Index(i)
+
+ // Check if the element is a int
+ if _, ok := elemValue.Interface().(int); !ok {
+ continue
+ }
+
+ // Check for match
+ if elemValue.Interface().(int) > opValue.Interface().(int) {
+ return true, nil
+ }
+ }
+ return false, nil
+ }
+
+ if realValue.Kind() != reflect.Int {
+ return false, nil
+ }
+ if realValue.Interface().(int) > opValue.Interface().(int) {
+ return true, nil
+ }
+ return false, nil
+
+ // Integer lower than ?
+ case MatchingOpLowerThan:
+ if _, ok := opValue.Interface().(int); !ok {
+ return false, fmt.Errorf("integer is expected for lower-than operator")
+ }
+
+ // If realValue is a slice
+ if realValue.Kind() == reflect.Slice {
+ for i := 0; i < realValue.Len(); i++ {
+ elemValue := realValue.Index(i)
+
+ // Check if the element is a int
+ if _, ok := elemValue.Interface().(int); !ok {
+ continue
+ }
+
+ // Check for match
+ if elemValue.Interface().(int) < opValue.Interface().(int) {
+ return true, nil
+ }
+ }
+ return false, nil
+ }
+
+ if realValue.Kind() != reflect.Int {
+ return false, nil
+ }
+ if realValue.Interface().(int) < opValue.Interface().(int) {
+ return true, nil
+ }
+ return false, nil
+
+ // Ignore these operators
+ case MatchingOpSource, MatchingOpSourceKind:
+ continue
+
+ // List of pattern
+ case MatchingKindRegexp:
+ patternList := opValue.Interface().([]*regexp.Regexp)
+
+ // If realValue is a slice
+ if realValue.Kind() == reflect.Slice {
+ for i := 0; i < realValue.Len(); i++ {
+ elemValue := realValue.Index(i)
+
+ // Check if the element is a string
+ if _, ok := elemValue.Interface().(string); !ok {
+ continue
+ }
+
+ // Check for a match with the regex pattern
+ for _, pattern := range patternList {
+ if pattern.MatchString(elemValue.Interface().(string)) {
+ return true, nil
+ }
+ }
+ }
+ // No match found in the slice
+ return false, nil
+ }
+
+ if realValue.Kind() != reflect.String {
+ return false, nil
+ }
+ for _, pattern := range patternList {
+ if pattern.MatchString(realValue.Interface().(string)) {
+ return true, nil
+ }
+ }
+ // No match found in pattern list
+ return false, nil
+
+ // List of string
+ case MatchingKindString:
+ stringList := opValue.Interface().([]string)
+
+ // If realValue is a slice
+ if realValue.Kind() == reflect.Slice {
+ for i := 0; i < realValue.Len(); i++ {
+ elemValue := realValue.Index(i)
+
+ // Check if the element is a string
+ if _, ok := elemValue.Interface().(string); !ok {
+ continue
+ }
+
+ // Check for a match with the text
+ for _, textItem := range stringList {
+ if textItem == realValue.Interface().(string) {
+ return true, nil
+ }
+ }
+ }
+ // No match found in the slice
+ return false, nil
+ }
+
+ if realValue.Kind() != reflect.String {
+ return false, nil
+ }
+ for _, textItem := range stringList {
+ if textItem == realValue.Interface().(string) {
+ return true, nil
+ }
+ }
+
+ // No match found in string list
+ return false, nil
+
+ default:
+ return false, fmt.Errorf("invalid operator '%s', ignore it", opKey.Interface().(string))
+ }
+ }
+ return true, nil
+}
+
+// list can be provided by user in the config
+// dns.qname:
+// - ".*\\.github\\.com$"
+// - "^www\\.google\\.com$"
+func matchUserSlice(realValue, expectedValue reflect.Value) (bool, error) {
+ match := false
+ for i := 0; i < expectedValue.Len() && !match; i++ {
+ reflectedSub := reflect.ValueOf(expectedValue.Index(i).Interface())
+
+ switch reflectedSub.Kind() {
+ case reflect.Int:
+ if realValue.Kind() == reflect.Slice {
+ for i := 0; i < realValue.Len(); i++ {
+ elemValue := realValue.Index(i)
+ if _, ok := elemValue.Interface().(int); !ok {
+ continue
+ }
+ if reflectedSub.Interface().(int) == elemValue.Interface().(int) {
+ return true, nil
+ }
+ }
+ }
+
+ if realValue.Kind() != reflect.Int || realValue.Interface().(int) != reflectedSub.Interface().(int) {
+ continue
+ }
+ match = true
+ case reflect.String:
+ pattern := regexp.MustCompile(reflectedSub.Interface().(string))
+ if realValue.Kind() == reflect.Slice {
+ for i := 0; i < realValue.Len() && !match; i++ {
+ elemValue := realValue.Index(i)
+ if _, ok := elemValue.Interface().(string); !ok {
+ continue
+ }
+ // Check for a match with the regex pattern
+ if pattern.MatchString(elemValue.Interface().(string)) {
+ match = true
+ }
+ }
+ }
+
+ if realValue.Kind() != reflect.String {
+ continue
+ }
+
+ if pattern.MatchString(realValue.Interface().(string)) {
+ match = true
+ }
+ }
+ }
+ return match, nil
+}
+
+// boolean can be provided by user in the config
+// dns.flags.qr: true
+func matchUserBoolean(realValue, expectedValue reflect.Value) (bool, error) {
+ // If realValue is a slice
+ if realValue.Kind() == reflect.Slice {
+ for i := 0; i < realValue.Len(); i++ {
+ elemValue := realValue.Index(i)
+
+ // Check if the element is a int
+ if _, ok := elemValue.Interface().(bool); !ok {
+ continue
+ }
+
+ // Check for match
+ if expectedValue.Interface().(bool) == elemValue.Interface().(bool) {
+ return true, nil
+ }
+ }
+ }
+
+ if realValue.Kind() != reflect.Bool {
+ return false, nil
+ }
+
+ if expectedValue.Interface().(bool) != realValue.Interface().(bool) {
+ return false, nil
+ }
+ return true, nil
+}
+
+// integer can be provided by user in the config
+// dns.opcode: 0
+func matchUserInteger(realValue, expectedValue reflect.Value) (bool, error) {
+ // If realValue is a slice
+ if realValue.Kind() == reflect.Slice {
+ for i := 0; i < realValue.Len(); i++ {
+ elemValue := realValue.Index(i)
+
+ // Check if the element is a int
+ if _, ok := elemValue.Interface().(int); !ok {
+ continue
+ }
+
+ // Check for match
+ if expectedValue.Interface().(int) == elemValue.Interface().(int) {
+ return true, nil
+ }
+ }
+ }
+
+ if realValue.Kind() != reflect.Int {
+ return false, nil
+ }
+ if expectedValue.Interface().(int) != realValue.Interface().(int) {
+ return false, nil
+ }
+
+ return true, nil
+}
+
+// regexp can be provided by user in the config
+// dns.qname: "^.*\\.github\\.com$"
+func matchUserPattern(realValue, expectedValue reflect.Value) (bool, error) {
+ pattern := regexp.MustCompile(expectedValue.Interface().(string))
+
+ // If realValue is a slice
+ if realValue.Kind() == reflect.Slice {
+ for i := 0; i < realValue.Len(); i++ {
+ elemValue := realValue.Index(i)
+
+ // Check if the element is a string
+ if _, ok := elemValue.Interface().(string); !ok {
+ continue
+ }
+
+ // Check for a match with the regex pattern
+ if pattern.MatchString(elemValue.Interface().(string)) {
+ return true, nil
+ }
+ }
+ // No match found in the slice
+ return false, nil
+ }
+
+ // If realValue is not a string
+ if realValue.Kind() != reflect.String {
+ return false, nil
+ }
+
+ // Check for a match with the regex pattern
+ if !pattern.MatchString(realValue.String()) {
+ return false, nil
+ }
+
+ // Match found for a single value
+ return true, nil
+}
+
+func getFieldByJSONTag(value reflect.Value, nestedKeys string) (reflect.Value, bool) {
+ listKeys := strings.SplitN(nestedKeys, ".", 2)
+
+ for j, jsonKey := range listKeys {
+ // Iterate over the fields of the structure
+ for i := 0; i < value.NumField(); i++ {
+ field := value.Type().Field(i)
+
+ // Get JSON tag
+ tag := field.Tag.Get("json")
+ tagClean := strings.TrimSuffix(tag, ",omitempty")
+
+ // Check if the JSON tag matches
+ if tagClean == jsonKey {
+ // ptr
+ switch field.Type.Kind() {
+ // integer
+ case reflect.Ptr:
+ if fieldValue, found := getFieldByJSONTag(value.Field(i).Elem(), listKeys[j+1]); found {
+ return fieldValue, true
+ }
+
+ // struct
+ case reflect.Struct:
+ if fieldValue, found := getFieldByJSONTag(value.Field(i), listKeys[j+1]); found {
+ return fieldValue, true
+ }
+
+ // slice if a list
+ case reflect.Slice:
+ if fieldValue, leftKey, found := getSliceElement(value.Field(i), listKeys[j+1]); found {
+ switch field.Type.Kind() {
+ case reflect.Struct:
+ if fieldValue, found := getFieldByJSONTag(fieldValue, leftKey); found {
+ return fieldValue, true
+ }
+ case reflect.Slice:
+ var result []interface{}
+ for i := 0; i < fieldValue.Len(); i++ {
+
+ if fieldValue.Index(i).Kind() == reflect.String || fieldValue.Index(i).Kind() == reflect.Int {
+ result = append(result, fieldValue.Index(i).Interface())
+ } else {
+ if sliceValue, found := getFieldByJSONTag(fieldValue.Index(i), leftKey); found {
+ result = append(result, sliceValue.Interface())
+ }
+ }
+ }
+ // If the list is not empty, return the list
+ if len(result) > 0 {
+ return reflect.ValueOf(result), true
+ }
+ default:
+ return value.Field(i), true
+ }
+ }
+
+ // int, string
+ default:
+ return value.Field(i), true
+ }
+ }
+ }
+ }
+
+ return reflect.Value{}, false
+}
+
+func getSliceElement(sliceValue reflect.Value, nestedKeys string) (reflect.Value, string, bool) {
+ listKeys := strings.SplitN(nestedKeys, ".", 2)
+ leftKeys := ""
+ if len(listKeys) > 1 {
+ leftKeys = listKeys[1]
+ }
+ sliceIndex := listKeys[0]
+
+ if sliceIndex == "*" {
+ return sliceValue, leftKeys, true
+ }
+
+ // Convert the slice index from string to int
+ index, err := strconv.Atoi(sliceIndex)
+ if err != nil {
+ // Handle the error (e.g., invalid index format)
+ return reflect.Value{}, leftKeys, false
+ }
+
+ for i := 0; i < sliceValue.Len(); i++ {
+ if index == i {
+ return sliceValue.Index(i), leftKeys, true
+ }
+ }
+ // If no match is found, return an empty reflect.Value
+ return reflect.Value{}, leftKeys, false
+}
+
func GetFakeDNSMessage() DNSMessage {
dm := DNSMessage{}
dm.Init()
@@ -868,3 +1355,11 @@ func GetFakeDNSMessageWithPayload() DNSMessage {
dm.DNS.Length = len(dnsquestion)
return dm
}
+
+func GetFlatDNSMessage() (ret map[string]interface{}, err error) {
+ dm := DNSMessage{}
+ dm.Init()
+ dm.InitTransforms()
+ ret, err = dm.Flatten()
+ return
+}
diff --git a/dnsutils/worker.go b/dnsutils/worker.go
deleted file mode 100644
index 0d27c8c7..00000000
--- a/dnsutils/worker.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package dnsutils
-
-import "github.com/dmachard/go-dnscollector/pkgconfig"
-
-type Worker interface {
- SetLoggers(loggers []Worker)
- GetName() string
- Stop()
- Run()
- Channel() chan DNSMessage
- ReadConfig()
- ReloadConfig(config *pkgconfig.Config)
-}
diff --git a/docs/development.md b/docs/development.md
index e36aa8d5..ce867c12 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -12,13 +12,28 @@ How to userguides:
## Build and run from source
-Building from source. Use the latest golang available on your target system.
+Building from source
+
+- The very fast way, go to the top of the project and run go command
+
+```bash
+go run .
+```
+
+- Uses the `MakeFile` (prefered way)
+
+```bash
+make
+```
+
+Execute the binary
```bash
-make build
make run
```
+- From the `DockerFile`
+
## Run linters
Install linter
@@ -178,7 +193,7 @@ func NewMyLogger(config *pkgconfig.Config, logger *logger.Logger, name string) *
func (c *MyLogger) GetName() string { return c.name }
-func (c *MyLogger) SetLoggers(loggers []dnsutils.Worker) {}
+func (c *MyLogger) SetLoggers(loggers []pkgutils.Worker) {}
func (o *MyLogger) ReadConfig() {}
@@ -201,7 +216,7 @@ func (o *MyLogger) Stop() {
close(o.done)
}
-func (o *MyLogger) Channel() chan dnsutils.DnsMessage {
+func (o *MyLogger) GetInputChannel() chan dnsutils.DnsMessage {
return o.channel
}
@@ -255,79 +270,145 @@ Create the following file `collectors/mycollector.go` and `collectors/mycollecto
package collectors
import (
- "github.com/dmachard/go-dnscollector/dnsutils"
- "github.com/dmachard/go-logger"
+ "time"
+
+ "github.com/dmachard/go-dnscollector/dnsutils"
+ "github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-logger"
)
-type MyCollector struct {
- done chan bool
- exit chan bool
- loggers []dnsutils.Worker
- config *pkgconfig.Config
- logger *logger.Logger
- name string
+type MyNewCollector struct {
+ doneRun chan bool
+ doneMonitor chan bool
+ stopRun chan bool
+ stopMonitor chan bool
+ loggers []pkgutils.Worker
+ config *pkgconfig.Config
+ configChan chan *pkgconfig.Config
+ logger *logger.Logger
+ name string
+ droppedCount int
+ dropped chan int
}
-// workaround for macos, not yet supported
-func NewMyCollector(loggers []dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *MyCollector {
- logger.Info("[%s] mycollector - enabled", name)
- s := &MyCollector{
- done: make(chan bool),
- exit: make(chan bool),
- config: config,
- loggers: loggers,
- logger: logger,
- name: name,
- }
- s.ReadConfig()
- return s
+func NewNewCollector(loggers []pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger, name string) *Dnstap {
+ logger.Info("[%s] collector=mynewcollector - enabled", name)
+ s := &MyNewCollector{
+ doneRun: make(chan bool),
+ doneMonitor: make(chan bool),
+ stopRun: make(chan bool),
+ stopMonitor: make(chan bool),
+ dropped: make(chan int),
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ loggers: loggers,
+ logger: logger,
+ name: name,
+ }
+ s.ReadConfig()
+ return s
}
-func (c *MyCollector) GetName() string { return c.name }
+func (c *MyNewCollector) GetName() string { return c.name }
-func (c *MyCollector) SetLoggers(loggers []dnsutils.Worker) {
- c.loggers = loggers
+func (c *MyNewCollector) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.loggers = append(c.loggers, wrk)
}
-func (c *MyCollector) LogInfo(msg string, v ...interface{}) {
- c.logger.Info("["+c.name+"] mycollector - "+msg, v...)
+func (c *MyNewCollector) SetLoggers(loggers []pkgutils.Worker) {
+ c.loggers = loggers
}
-func (c *MyCollector) LogError(msg string, v ...interface{}) {
- c.logger.Error("["+c.name+"] mycollector - "+msg, v...)
+func (c *MyNewCollector) Loggers() ([]chan dnsutils.DNSMessage, []string) {
+ channels := []chan dnsutils.DNSMessage{}
+ names := []string{}
+ for _, p := range c.loggers {
+ channels = append(channels,p.GetInputChannel())
+ names = append(names, p.GetName())
+ }
+ return channels, names
}
-func (c *MyCollector) Loggers() []chan dnsutils.DnsMessage {
- channels := []chan dnsutils.DnsMessage{}
- for _, p := range c.loggers {
- channels = append(channels, p.Channel())
- }
- return channels
+func (c *MyNewCollector) ReadConfig() {}
+
+func (c *MyNewCollector) ReloadConfig(config *pkgconfig.Config) {
+ c.LogInfo("reload configuration...")
+ c.configChan <- config
}
-func (c *MyCollector) ReadConfig() {
+func (c *MyNewCollector) LogInfo(msg string, v ...interface{}) {
+ c.logger.Info("["+c.name+"] collector=mynewcollector - "+msg, v...)
}
-func (c *MyCollector) Channel() chan dnsutils.DnsMessage {
- return nil
+func (c *MyNewCollector) LogError(msg string, v ...interface{}) {
+ c.logger.Error("["+c.name+" collector=mynewcollector - "+msg, v...)
}
-func (c *MyCollector) Stop() {
- c.LogInfo("stopping...")
+func (c *MyNewCollector) GetInputChannel() chan dnsutils.DNSMessage {
+ return nil
+}
- // exit to close properly
- c.exit <- true
+func (c *MyNewCollector) Stop() {
+ // stop monitor goroutine
+ c.LogInfo("stopping monitor...")
+ c.stopMonitor <- true
+ <-c.doneMonitor
- // read done channel and block until run is terminated
- <-c.done
- close(c.done)
+ // read done channel and block until run is terminated
+ c.LogInfo("stopping run...")
+ c.stopRun <- true
+ <-c.doneRun
}
-func (c *MyCollector) Run() {
- c.LogInfo("run terminated")
- c.done <- true
+func (c *MyNewCollector) MonitorCollector() {
+ watchInterval := 10 * time.Second
+ bufferFull := time.NewTimer(watchInterval)
+MONITOR_LOOP:
+ for {
+ select {
+ case <-c.dropped:
+ c.droppedCount++
+ case <-c.stopMonitor:
+ close(c.dropped)
+ bufferFull.Stop()
+ c.doneMonitor <- true
+ break MONITOR_LOOP
+ case <-bufferFull.C:
+ if c.droppedCount > 0 {
+ c.LogError("recv buffer is full, %d packet(s) dropped", c.droppedCount)
+ c.droppedCount = 0
+ }
+ bufferFull.Reset(watchInterval)
+ }
+ }
+ c.LogInfo("monitor terminated")
}
+func (c *DNSMessage) Run() {
+ c.LogInfo("starting collector...")
+
+ // start goroutine to count dropped messsages
+ go c.MonitorCollector()
+
+RUN_LOOP:
+ for {
+ select {
+ case <-c.stopRun:
+ c.doneRun <- true
+ break RUN_LOOP
+
+ case cfg := <-c.configChan:
+
+ // save the new config
+ c.config = cfg
+ c.ReadConfig()
+ }
+
+ }
+ c.LogInfo("run terminated")
+}
+
+
```
Update the main file `dnscollector.go`
diff --git a/loggers/dnstapclient.go b/loggers/dnstapclient.go
index 5d493137..305b55ec 100644
--- a/loggers/dnstapclient.go
+++ b/loggers/dnstapclient.go
@@ -10,6 +10,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-framestream"
"github.com/dmachard/go-logger"
@@ -32,11 +33,12 @@ type DnstapSender struct {
transportReady chan bool
transportReconnect chan bool
name string
+ RoutingHandler pkgutils.RoutingHandler
}
func NewDnstapSender(config *pkgconfig.Config, logger *logger.Logger, name string) *DnstapSender {
logger.Info("[%s] logger=dnstap - enabled", name)
- s := &DnstapSender{
+ ds := &DnstapSender{
stopProcess: make(chan bool),
doneProcess: make(chan bool),
stopRun: make(chan bool),
@@ -49,120 +51,131 @@ func NewDnstapSender(config *pkgconfig.Config, logger *logger.Logger, name strin
config: config,
configChan: make(chan *pkgconfig.Config),
name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
}
- s.ReadConfig()
+ ds.ReadConfig()
+ return ds
+}
+
+func (ds *DnstapSender) GetName() string { return ds.name }
- return s
+func (ds *DnstapSender) AddDroppedRoute(wrk pkgutils.Worker) {
+ ds.RoutingHandler.AddDroppedRoute(wrk)
}
-func (c *DnstapSender) GetName() string { return c.name }
+func (ds *DnstapSender) AddDefaultRoute(wrk pkgutils.Worker) {
+ ds.RoutingHandler.AddDefaultRoute(wrk)
+}
-func (c *DnstapSender) SetLoggers(loggers []dnsutils.Worker) {}
+func (ds *DnstapSender) SetLoggers(loggers []pkgutils.Worker) {}
-func (c *DnstapSender) ReadConfig() {
- c.transport = c.config.Loggers.DNSTap.Transport
+func (ds *DnstapSender) ReadConfig() {
+ ds.transport = ds.config.Loggers.DNSTap.Transport
// begin backward compatibility
- if c.config.Loggers.DNSTap.TLSSupport {
- c.transport = netlib.SocketTLS
+ if ds.config.Loggers.DNSTap.TLSSupport {
+ ds.transport = netlib.SocketTLS
}
- if len(c.config.Loggers.DNSTap.SockPath) > 0 {
- c.transport = netlib.SocketUnix
+ if len(ds.config.Loggers.DNSTap.SockPath) > 0 {
+ ds.transport = netlib.SocketUnix
}
// end
// get hostname or global one
- if c.config.Loggers.DNSTap.ServerID == "" {
- c.config.Loggers.DNSTap.ServerID = c.config.GetServerIdentity()
+ if ds.config.Loggers.DNSTap.ServerID == "" {
+ ds.config.Loggers.DNSTap.ServerID = ds.config.GetServerIdentity()
}
- if !pkgconfig.IsValidTLS(c.config.Loggers.DNSTap.TLSMinVersion) {
- c.logger.Fatal("logger=dnstap - invalid tls min version")
+ if !pkgconfig.IsValidTLS(ds.config.Loggers.DNSTap.TLSMinVersion) {
+ ds.logger.Fatal("logger=dnstap - invalid tls min version")
}
}
-func (c *DnstapSender) ReloadConfig(config *pkgconfig.Config) {
- c.LogInfo("reload configuration!")
- c.configChan <- config
+func (ds *DnstapSender) ReloadConfig(config *pkgconfig.Config) {
+ ds.LogInfo("reload configuration!")
+ ds.configChan <- config
}
-func (c *DnstapSender) LogInfo(msg string, v ...interface{}) {
- c.logger.Info("["+c.name+"] logger=dnstap - "+msg, v...)
+func (ds *DnstapSender) LogInfo(msg string, v ...interface{}) {
+ ds.logger.Info("["+ds.name+"] logger=dnstap - "+msg, v...)
}
-func (c *DnstapSender) LogError(msg string, v ...interface{}) {
- c.logger.Error("["+c.name+"] logger=dnstap - "+msg, v...)
+func (ds *DnstapSender) LogError(msg string, v ...interface{}) {
+ ds.logger.Error("["+ds.name+"] logger=dnstap - "+msg, v...)
}
-func (c *DnstapSender) Channel() chan dnsutils.DNSMessage {
- return c.inputChan
+func (ds *DnstapSender) GetInputChannel() chan dnsutils.DNSMessage {
+ return ds.inputChan
}
-func (c *DnstapSender) Stop() {
- c.LogInfo("stopping to run...")
- c.stopRun <- true
- <-c.doneRun
+func (ds *DnstapSender) Stop() {
+ ds.LogInfo("stopping routing handler...")
+ ds.RoutingHandler.Stop()
- c.LogInfo("stopping to process...")
- c.stopProcess <- true
- <-c.doneProcess
+ ds.LogInfo("stopping to run...")
+ ds.stopRun <- true
+ <-ds.doneRun
+
+ ds.LogInfo("stopping to process...")
+ ds.stopProcess <- true
+ <-ds.doneProcess
}
-func (c *DnstapSender) Disconnect() {
- if c.transportConn != nil {
+func (ds *DnstapSender) Disconnect() {
+ if ds.transportConn != nil {
// reset framestream and ignore errors
- c.LogInfo("closing framestream")
- c.fs.ResetSender()
+ ds.LogInfo("closing framestream")
+ ds.fs.ResetSender()
// closing tcp
- c.LogInfo("closing tcp connection")
- c.transportConn.Close()
- c.LogInfo("closed")
+ ds.LogInfo("closing tcp connection")
+ ds.transportConn.Close()
+ ds.LogInfo("closed")
}
}
-func (c *DnstapSender) ConnectToRemote() {
+func (ds *DnstapSender) ConnectToRemote() {
for {
- if c.transportConn != nil {
- c.transportConn.Close()
- c.transportConn = nil
+ if ds.transportConn != nil {
+ ds.transportConn.Close()
+ ds.transportConn = nil
}
address := net.JoinHostPort(
- c.config.Loggers.DNSTap.RemoteAddress,
- strconv.Itoa(c.config.Loggers.DNSTap.RemotePort),
+ ds.config.Loggers.DNSTap.RemoteAddress,
+ strconv.Itoa(ds.config.Loggers.DNSTap.RemotePort),
)
- connTimeout := time.Duration(c.config.Loggers.DNSTap.ConnectTimeout) * time.Second
+ connTimeout := time.Duration(ds.config.Loggers.DNSTap.ConnectTimeout) * time.Second
// make the connection
var conn net.Conn
var err error
- switch c.transport {
+ switch ds.transport {
case netlib.SocketUnix:
- address = c.config.Loggers.DNSTap.RemoteAddress
- if len(c.config.Loggers.DNSTap.SockPath) > 0 {
- address = c.config.Loggers.DNSTap.SockPath
+ address = ds.config.Loggers.DNSTap.RemoteAddress
+ if len(ds.config.Loggers.DNSTap.SockPath) > 0 {
+ address = ds.config.Loggers.DNSTap.SockPath
}
- c.LogInfo("connecting to %s://%s", c.transport, address)
- conn, err = net.DialTimeout(c.transport, address, connTimeout)
+ ds.LogInfo("connecting to %s://%s", ds.transport, address)
+ conn, err = net.DialTimeout(ds.transport, address, connTimeout)
case netlib.SocketTCP:
- c.LogInfo("connecting to %s://%s", c.transport, address)
- conn, err = net.DialTimeout(c.transport, address, connTimeout)
+ ds.LogInfo("connecting to %s://%s", ds.transport, address)
+ conn, err = net.DialTimeout(ds.transport, address, connTimeout)
case netlib.SocketTLS:
- c.LogInfo("connecting to %s://%s", c.transport, address)
+ ds.LogInfo("connecting to %s://%s", ds.transport, address)
var tlsConfig *tls.Config
tlsOptions := pkgconfig.TLSOptions{
- InsecureSkipVerify: c.config.Loggers.DNSTap.TLSInsecure,
- MinVersion: c.config.Loggers.DNSTap.TLSMinVersion,
- CAFile: c.config.Loggers.DNSTap.CAFile,
- CertFile: c.config.Loggers.DNSTap.CertFile,
- KeyFile: c.config.Loggers.DNSTap.KeyFile,
+ InsecureSkipVerify: ds.config.Loggers.DNSTap.TLSInsecure,
+ MinVersion: ds.config.Loggers.DNSTap.TLSMinVersion,
+ CAFile: ds.config.Loggers.DNSTap.CAFile,
+ CertFile: ds.config.Loggers.DNSTap.CertFile,
+ KeyFile: ds.config.Loggers.DNSTap.KeyFile,
}
tlsConfig, err = pkgconfig.TLSClientConfig(tlsOptions)
@@ -171,28 +184,28 @@ func (c *DnstapSender) ConnectToRemote() {
conn, err = tls.DialWithDialer(dialer, netlib.SocketTCP, address, tlsConfig)
}
default:
- c.logger.Fatal("logger=dnstap - invalid transport:", c.transport)
+ ds.logger.Fatal("logger=dnstap - invalid transport:", ds.transport)
}
// something is wrong during connection ?
if err != nil {
- c.LogError("%s", err)
- c.LogInfo("retry to connect in %d seconds", c.config.Loggers.DNSTap.RetryInterval)
- time.Sleep(time.Duration(c.config.Loggers.DNSTap.RetryInterval) * time.Second)
+ ds.LogError("%s", err)
+ ds.LogInfo("retry to connect in %d seconds", ds.config.Loggers.DNSTap.RetryInterval)
+ time.Sleep(time.Duration(ds.config.Loggers.DNSTap.RetryInterval) * time.Second)
continue
}
- c.transportConn = conn
+ ds.transportConn = conn
// block until framestream is ready
- c.transportReady <- true
+ ds.transportReady <- true
// block until an error occured, need to reconnect
- c.transportReconnect <- true
+ ds.transportReconnect <- true
}
}
-func (c *DnstapSender) FlushBuffer(buf *[]dnsutils.DNSMessage) {
+func (ds *DnstapSender) FlushBuffer(buf *[]dnsutils.DNSMessage) {
var data []byte
var err error
@@ -200,23 +213,23 @@ func (c *DnstapSender) FlushBuffer(buf *[]dnsutils.DNSMessage) {
for _, dm := range *buf {
// update identity ?
- if c.config.Loggers.DNSTap.OverwriteIdentity {
- dm.DNSTap.Identity = c.config.Loggers.DNSTap.ServerID
+ if ds.config.Loggers.DNSTap.OverwriteIdentity {
+ dm.DNSTap.Identity = ds.config.Loggers.DNSTap.ServerID
}
// encode dns message to dnstap protobuf binary
data, err = dm.ToDNSTap()
if err != nil {
- c.LogError("failed to encode to DNStap protobuf: %s", err)
+ ds.LogError("failed to encode to DNStap protobuf: %s", err)
continue
}
// send the frame
frame.Write(data)
- if err := c.fs.SendFrame(frame); err != nil {
- c.LogError("send frame error %s", err)
- c.fsReady = false
- <-c.transportReconnect
+ if err := ds.fs.SendFrame(frame); err != nil {
+ ds.LogError("send frame error %s", err)
+ ds.fsReady = false
+ <-ds.transportReconnect
break
}
}
@@ -225,105 +238,123 @@ func (c *DnstapSender) FlushBuffer(buf *[]dnsutils.DNSMessage) {
*buf = nil
}
-func (c *DnstapSender) Run() {
- c.LogInfo("running in background...")
+func (ds *DnstapSender) Run() {
+ ds.LogInfo("running in background...")
+
+ // prepare next channels
+ defaultRoutes, defaultNames := ds.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := ds.RoutingHandler.GetDroppedRoutes()
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
- listChannel = append(listChannel, c.outputChan)
- subprocessors := transformers.NewTransforms(&c.config.OutgoingTransformers, c.logger, c.name, listChannel, 0)
+ listChannel = append(listChannel, ds.outputChan)
+ subprocessors := transformers.NewTransforms(&ds.config.OutgoingTransformers, ds.logger, ds.name, listChannel, 0)
// goroutine to process transformed dns messages
- go c.Process()
+ go ds.Process()
// init remote conn
- go c.ConnectToRemote()
+ go ds.ConnectToRemote()
// loop to process incoming messages
RUN_LOOP:
for {
select {
- case <-c.stopRun:
+ case <-ds.stopRun:
// cleanup transformers
subprocessors.Reset()
- c.doneRun <- true
+ ds.doneRun <- true
break RUN_LOOP
- case cfg, opened := <-c.configChan:
+ case cfg, opened := <-ds.configChan:
if !opened {
return
}
- c.config = cfg
- c.ReadConfig()
+ ds.config = cfg
+ ds.ReadConfig()
subprocessors.ReloadConfig(&cfg.OutgoingTransformers)
- case dm, opened := <-c.inputChan:
+ case dm, opened := <-ds.inputChan:
if !opened {
- c.LogInfo("input channel closed!")
+ ds.LogInfo("input channel closed!")
return
}
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ ds.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ ds.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
- c.outputChan <- dm
+ ds.outputChan <- dm
}
}
- c.LogInfo("run terminated")
+ ds.LogInfo("run terminated")
}
-func (c *DnstapSender) Process() {
+func (ds *DnstapSender) Process() {
// init buffer
bufferDm := []dnsutils.DNSMessage{}
// init flust timer for buffer
- flushInterval := time.Duration(c.config.Loggers.DNSTap.FlushInterval) * time.Second
+ flushInterval := time.Duration(ds.config.Loggers.DNSTap.FlushInterval) * time.Second
flushTimer := time.NewTimer(flushInterval)
- c.LogInfo("ready to process")
+ // nextStanzaBufferInterval := 10 * time.Second
+ // nextStanzaBufferFull := time.NewTimer(nextStanzaBufferInterval)
+
+ ds.LogInfo("ready to process")
PROCESS_LOOP:
for {
select {
- case <-c.stopProcess:
+ case <-ds.stopProcess:
// closing remote connection if exist
- c.Disconnect()
+ ds.Disconnect()
- c.doneProcess <- true
+ ds.doneProcess <- true
break PROCESS_LOOP
- // init framestream
- case <-c.transportReady:
- c.LogInfo("transport connected with success")
+ // case stanzaName := <-ds.dropped:
+ // if _, ok := ds.droppedCount[stanzaName]; !ok {
+ // ds.droppedCount[stanzaName] = 1
+ // } else {
+ // ds.droppedCount[stanzaName]++
+ // }
+
+ // init framestream
+ case <-ds.transportReady:
+ ds.LogInfo("transport connected with success")
// frame stream library
- r := bufio.NewReader(c.transportConn)
- w := bufio.NewWriter(c.transportConn)
- c.fs = framestream.NewFstrm(r, w, c.transportConn, 5*time.Second, []byte("protobuf:dnstap.Dnstap"), true)
+ r := bufio.NewReader(ds.transportConn)
+ w := bufio.NewWriter(ds.transportConn)
+ ds.fs = framestream.NewFstrm(r, w, ds.transportConn, 5*time.Second, []byte("protobuf:dnstap.Dnstap"), true)
// init framestream protocol
- if err := c.fs.InitSender(); err != nil {
- c.LogError("sender protocol initialization error %s", err)
- c.fsReady = false
- c.transportConn.Close()
- <-c.transportReconnect
+ if err := ds.fs.InitSender(); err != nil {
+ ds.LogError("sender protocol initialization error %s", err)
+ ds.fsReady = false
+ ds.transportConn.Close()
+ <-ds.transportReconnect
} else {
- c.fsReady = true
- c.LogInfo("framestream initialized with success")
+ ds.fsReady = true
+ ds.LogInfo("framestream initialized with success")
}
// incoming dns message to process
- case dm, opened := <-c.outputChan:
+ case dm, opened := <-ds.outputChan:
if !opened {
- c.LogInfo("output channel closed!")
+ ds.LogInfo("output channel closed!")
return
}
// drop dns message if the connection is not ready to avoid memory leak or
// to block the channel
- if !c.fsReady {
+ if !ds.fsReady {
continue
}
@@ -331,20 +362,29 @@ PROCESS_LOOP:
bufferDm = append(bufferDm, dm)
// buffer is full ?
- if len(bufferDm) >= c.config.Loggers.DNSTap.BufferSize {
- c.FlushBuffer(&bufferDm)
+ if len(bufferDm) >= ds.config.Loggers.DNSTap.BufferSize {
+ ds.FlushBuffer(&bufferDm)
}
// flush the buffer
case <-flushTimer.C:
// force to flush the buffer
if len(bufferDm) > 0 {
- c.FlushBuffer(&bufferDm)
+ ds.FlushBuffer(&bufferDm)
}
// restart timer
flushTimer.Reset(flushInterval)
+
+ // case <-nextStanzaBufferFull.C:
+ // for v, k := range ds.droppedCount {
+ // if k > 0 {
+ // ds.LogError("stanza[%s] buffer is full, %d packet(s) dropped", v, k)
+ // ds.droppedCount[v] = 0
+ // }
+ // }
+ // nextStanzaBufferFull.Reset(nextStanzaBufferInterval)
}
}
- c.LogInfo("processing terminated")
+ ds.LogInfo("processing terminated")
}
diff --git a/loggers/dnstapclient_test.go b/loggers/dnstapclient_test.go
index 195764eb..f67b1623 100644
--- a/loggers/dnstapclient_test.go
+++ b/loggers/dnstapclient_test.go
@@ -70,7 +70,7 @@ func Test_DnstapClient(t *testing.T) {
// send fake dns message to logger
dm := dnsutils.GetFakeDNSMessage()
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// receive frame on server side ?, timeout 5s
fs, err := fsSvr.RecvFrame(true)
diff --git a/loggers/elasticsearch.go b/loggers/elasticsearch.go
index 735aacec..9da57d1b 100644
--- a/loggers/elasticsearch.go
+++ b/loggers/elasticsearch.go
@@ -8,6 +8,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
@@ -16,132 +17,153 @@ import (
)
type ElasticSearchClient struct {
- stopProcess chan bool
- doneProcess chan bool
- stopRun chan bool
- doneRun chan bool
- inputChan chan dnsutils.DNSMessage
- outputChan chan dnsutils.DNSMessage
- config *pkgconfig.Config
- configChan chan *pkgconfig.Config
- logger *logger.Logger
- name string
- server string
- index string
- bulkURL string
+ stopProcess chan bool
+ doneProcess chan bool
+ stopRun chan bool
+ doneRun chan bool
+ inputChan chan dnsutils.DNSMessage
+ outputChan chan dnsutils.DNSMessage
+ config *pkgconfig.Config
+ configChan chan *pkgconfig.Config
+ logger *logger.Logger
+ name string
+ server string
+ index string
+ bulkURL string
+ RoutingHandler pkgutils.RoutingHandler
}
func NewElasticSearchClient(config *pkgconfig.Config, console *logger.Logger, name string) *ElasticSearchClient {
console.Info("[%s] logger=elasticsearch - enabled", name)
- c := &ElasticSearchClient{
- stopProcess: make(chan bool),
- doneProcess: make(chan bool),
- stopRun: make(chan bool),
- doneRun: make(chan bool),
- inputChan: make(chan dnsutils.DNSMessage, config.Loggers.ElasticSearchClient.ChannelBufferSize),
- outputChan: make(chan dnsutils.DNSMessage, config.Loggers.ElasticSearchClient.ChannelBufferSize),
- logger: console,
- config: config,
- configChan: make(chan *pkgconfig.Config),
- name: name,
+ ec := &ElasticSearchClient{
+ stopProcess: make(chan bool),
+ doneProcess: make(chan bool),
+ stopRun: make(chan bool),
+ doneRun: make(chan bool),
+ inputChan: make(chan dnsutils.DNSMessage, config.Loggers.ElasticSearchClient.ChannelBufferSize),
+ outputChan: make(chan dnsutils.DNSMessage, config.Loggers.ElasticSearchClient.ChannelBufferSize),
+ logger: console,
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, console, name),
}
- c.ReadConfig()
- return c
+ ec.ReadConfig()
+ return ec
}
-func (c *ElasticSearchClient) GetName() string { return c.name }
+func (ec *ElasticSearchClient) GetName() string { return ec.name }
-func (c *ElasticSearchClient) SetLoggers(loggers []dnsutils.Worker) {}
+func (ec *ElasticSearchClient) AddDroppedRoute(wrk pkgutils.Worker) {
+ ec.RoutingHandler.AddDroppedRoute(wrk)
+}
+
+func (ec *ElasticSearchClient) AddDefaultRoute(wrk pkgutils.Worker) {
+ ec.RoutingHandler.AddDefaultRoute(wrk)
+}
+
+func (ec *ElasticSearchClient) SetLoggers(loggers []pkgutils.Worker) {}
-func (c *ElasticSearchClient) ReadConfig() {
- c.server = c.config.Loggers.ElasticSearchClient.Server
- c.index = c.config.Loggers.ElasticSearchClient.Index
+func (ec *ElasticSearchClient) ReadConfig() {
+ ec.server = ec.config.Loggers.ElasticSearchClient.Server
+ ec.index = ec.config.Loggers.ElasticSearchClient.Index
- u, err := url.Parse(c.server)
+ u, err := url.Parse(ec.server)
if err != nil {
- c.LogError(err.Error())
+ ec.LogError(err.Error())
}
- u.Path = path.Join(u.Path, c.index, "_bulk")
- c.bulkURL = u.String()
+ u.Path = path.Join(u.Path, ec.index, "_bulk")
+ ec.bulkURL = u.String()
}
-func (c *ElasticSearchClient) ReloadConfig(config *pkgconfig.Config) {
- c.LogInfo("reload configuration!")
- c.configChan <- config
+func (ec *ElasticSearchClient) ReloadConfig(config *pkgconfig.Config) {
+ ec.LogInfo("reload configuration!")
+ ec.configChan <- config
}
-func (c *ElasticSearchClient) Channel() chan dnsutils.DNSMessage {
- return c.inputChan
+func (ec *ElasticSearchClient) GetInputChannel() chan dnsutils.DNSMessage {
+ return ec.inputChan
}
-func (c *ElasticSearchClient) LogInfo(msg string, v ...interface{}) {
- c.logger.Info("["+c.name+"] logger=elasticsearch - "+msg, v...)
+func (ec *ElasticSearchClient) LogInfo(msg string, v ...interface{}) {
+ ec.logger.Info("["+ec.name+"] logger=elasticsearch - "+msg, v...)
}
-func (c *ElasticSearchClient) LogError(msg string, v ...interface{}) {
- c.logger.Error("["+c.name+"] logger=elasticsearch - "+msg, v...)
+func (ec *ElasticSearchClient) LogError(msg string, v ...interface{}) {
+ ec.logger.Error("["+ec.name+"] logger=elasticsearch - "+msg, v...)
}
-func (c *ElasticSearchClient) Stop() {
- c.LogInfo("stopping to run...")
- c.stopRun <- true
- <-c.doneRun
+func (ec *ElasticSearchClient) Stop() {
+ ec.LogInfo("stopping routing handler...")
+ ec.RoutingHandler.Stop()
- c.LogInfo("stopping to process...")
- c.stopProcess <- true
- <-c.doneProcess
+ ec.LogInfo("stopping to run...")
+ ec.stopRun <- true
+ <-ec.doneRun
+
+ ec.LogInfo("stopping to process...")
+ ec.stopProcess <- true
+ <-ec.doneProcess
}
-func (c *ElasticSearchClient) Run() {
- c.LogInfo("running in background...")
+func (ec *ElasticSearchClient) Run() {
+ ec.LogInfo("running in background...")
+
+ // prepare next channels
+ defaultRoutes, defaultNames := ec.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := ec.RoutingHandler.GetDroppedRoutes()
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
- listChannel = append(listChannel, c.outputChan)
- subprocessors := transformers.NewTransforms(&c.config.OutgoingTransformers, c.logger, c.name, listChannel, 0)
+ listChannel = append(listChannel, ec.outputChan)
+ subprocessors := transformers.NewTransforms(&ec.config.OutgoingTransformers, ec.logger, ec.name, listChannel, 0)
// goroutine to process transformed dns messages
- go c.Process()
+ go ec.Process()
// loop to process incoming messages
RUN_LOOP:
for {
select {
- case <-c.stopRun:
+ case <-ec.stopRun:
// cleanup transformers
subprocessors.Reset()
- c.doneRun <- true
+ ec.doneRun <- true
break RUN_LOOP
- case cfg, opened := <-c.configChan:
+ case cfg, opened := <-ec.configChan:
if !opened {
return
}
- c.config = cfg
- c.ReadConfig()
+ ec.config = cfg
+ ec.ReadConfig()
subprocessors.ReloadConfig(&cfg.OutgoingTransformers)
- case dm, opened := <-c.inputChan:
+ case dm, opened := <-ec.inputChan:
if !opened {
- c.LogInfo("input channel closed!")
+ ec.LogInfo("input channel closed!")
return
}
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ ec.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ ec.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
- c.outputChan <- dm
+ ec.outputChan <- dm
}
}
- c.LogInfo("run terminated")
+ ec.LogInfo("run terminated")
}
-func (c *ElasticSearchClient) FlushBuffer(buf *[]dnsutils.DNSMessage) {
+func (ec *ElasticSearchClient) FlushBuffer(buf *[]dnsutils.DNSMessage) {
buffer := new(bytes.Buffer)
for _, dm := range *buf {
@@ -150,42 +172,42 @@ func (c *ElasticSearchClient) FlushBuffer(buf *[]dnsutils.DNSMessage) {
// encode
flat, err := dm.Flatten()
if err != nil {
- c.LogError("flattening DNS message failed: %e", err)
+ ec.LogError("flattening DNS message failed: %e", err)
}
json.NewEncoder(buffer).Encode(flat)
}
- req, _ := http.NewRequest("POST", c.bulkURL, buffer)
+ req, _ := http.NewRequest("POST", ec.bulkURL, buffer)
req.Header.Set("Content-Type", "application/json")
client := &http.Client{
Timeout: 5 * time.Second,
}
_, err := client.Do(req)
if err != nil {
- c.LogError(err.Error())
+ ec.LogError(err.Error())
}
*buf = nil
}
-func (c *ElasticSearchClient) Process() {
+func (ec *ElasticSearchClient) Process() {
bufferDm := []dnsutils.DNSMessage{}
- c.LogInfo("ready to process")
+ ec.LogInfo("ready to process")
- flushInterval := time.Duration(c.config.Loggers.ElasticSearchClient.FlushInterval) * time.Second
+ flushInterval := time.Duration(ec.config.Loggers.ElasticSearchClient.FlushInterval) * time.Second
flushTimer := time.NewTimer(flushInterval)
PROCESS_LOOP:
for {
select {
- case <-c.stopProcess:
- c.doneProcess <- true
+ case <-ec.stopProcess:
+ ec.doneProcess <- true
break PROCESS_LOOP
// incoming dns message to process
- case dm, opened := <-c.outputChan:
+ case dm, opened := <-ec.outputChan:
if !opened {
- c.LogInfo("output channel closed!")
+ ec.LogInfo("output channel closed!")
return
}
@@ -193,18 +215,18 @@ PROCESS_LOOP:
bufferDm = append(bufferDm, dm)
// buffer is full ?
- if len(bufferDm) >= c.config.Loggers.ElasticSearchClient.BulkSize {
- c.FlushBuffer(&bufferDm)
+ if len(bufferDm) >= ec.config.Loggers.ElasticSearchClient.BulkSize {
+ ec.FlushBuffer(&bufferDm)
}
// flush the buffer
case <-flushTimer.C:
if len(bufferDm) > 0 {
- c.FlushBuffer(&bufferDm)
+ ec.FlushBuffer(&bufferDm)
}
// restart timer
flushTimer.Reset(flushInterval)
}
}
- c.LogInfo("processing terminated")
+ ec.LogInfo("processing terminated")
}
diff --git a/loggers/elasticsearch_test.go b/loggers/elasticsearch_test.go
index 101eef88..a2f141c4 100644
--- a/loggers/elasticsearch_test.go
+++ b/loggers/elasticsearch_test.go
@@ -47,7 +47,7 @@ func Test_ElasticSearchClient(t *testing.T) {
dm := dnsutils.GetFakeDNSMessage()
for i := 0; i < tc.inputSize; i++ {
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
}
for i := 0; i < tc.inputSize/tc.bulkSize; i++ {
@@ -130,7 +130,7 @@ func Test_ElasticSearchClientFlushINterval(t *testing.T) {
dm := dnsutils.GetFakeDNSMessage()
for i := 0; i < tc.inputSize; i++ {
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
}
conn, err := fakeRcvr.Accept()
diff --git a/loggers/fakelogger.go b/loggers/fakelogger.go
index ea2411c4..970c963d 100644
--- a/loggers/fakelogger.go
+++ b/loggers/fakelogger.go
@@ -3,6 +3,7 @@ package loggers
import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
)
type FakeLogger struct {
@@ -22,7 +23,11 @@ func NewFakeLogger() *FakeLogger {
func (c *FakeLogger) GetName() string { return c.name }
-func (c *FakeLogger) SetLoggers(loggers []dnsutils.Worker) {}
+func (c *FakeLogger) AddDefaultRoute(wrk pkgutils.Worker) {}
+
+func (c *FakeLogger) AddDroppedRoute(wrk pkgutils.Worker) {}
+
+func (c *FakeLogger) SetLoggers(loggers []pkgutils.Worker) {}
func (c *FakeLogger) ReadConfig() {}
@@ -30,7 +35,7 @@ func (c *FakeLogger) ReloadConfig(config *pkgconfig.Config) {}
func (c *FakeLogger) Stop() {}
-func (c *FakeLogger) Channel() chan dnsutils.DNSMessage {
+func (c *FakeLogger) GetInputChannel() chan dnsutils.DNSMessage {
return c.inputChan
}
diff --git a/loggers/falco.go b/loggers/falco.go
index dd634d3c..32375da3 100644
--- a/loggers/falco.go
+++ b/loggers/falco.go
@@ -8,160 +8,182 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
)
type FalcoClient struct {
- stopProcess chan bool
- doneProcess chan bool
- stopRun chan bool
- doneRun chan bool
- inputChan chan dnsutils.DNSMessage
- outputChan chan dnsutils.DNSMessage
- config *pkgconfig.Config
- configChan chan *pkgconfig.Config
- logger *logger.Logger
- name string
- url string
+ stopProcess chan bool
+ doneProcess chan bool
+ stopRun chan bool
+ doneRun chan bool
+ inputChan chan dnsutils.DNSMessage
+ outputChan chan dnsutils.DNSMessage
+ config *pkgconfig.Config
+ configChan chan *pkgconfig.Config
+ logger *logger.Logger
+ name string
+ url string
+ RoutingHandler pkgutils.RoutingHandler
}
func NewFalcoClient(config *pkgconfig.Config, console *logger.Logger, name string) *FalcoClient {
console.Info("[%s] logger=falco - enabled", name)
- f := &FalcoClient{
- stopProcess: make(chan bool),
- doneProcess: make(chan bool),
- stopRun: make(chan bool),
- doneRun: make(chan bool),
- inputChan: make(chan dnsutils.DNSMessage, config.Loggers.FalcoClient.ChannelBufferSize),
- outputChan: make(chan dnsutils.DNSMessage, config.Loggers.FalcoClient.ChannelBufferSize),
- logger: console,
- config: config,
- configChan: make(chan *pkgconfig.Config),
- name: name,
+ fc := &FalcoClient{
+ stopProcess: make(chan bool),
+ doneProcess: make(chan bool),
+ stopRun: make(chan bool),
+ doneRun: make(chan bool),
+ inputChan: make(chan dnsutils.DNSMessage, config.Loggers.FalcoClient.ChannelBufferSize),
+ outputChan: make(chan dnsutils.DNSMessage, config.Loggers.FalcoClient.ChannelBufferSize),
+ logger: console,
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, console, name),
}
- f.ReadConfig()
- return f
+ fc.ReadConfig()
+ return fc
}
-func (f *FalcoClient) GetName() string { return f.name }
+func (fc *FalcoClient) GetName() string { return fc.name }
-func (f *FalcoClient) SetLoggers(loggers []dnsutils.Worker) {}
+func (fc *FalcoClient) AddDroppedRoute(wrk pkgutils.Worker) {
+ fc.RoutingHandler.AddDroppedRoute(wrk)
+}
+
+func (fc *FalcoClient) AddDefaultRoute(wrk pkgutils.Worker) {
+ fc.RoutingHandler.AddDefaultRoute(wrk)
+}
+
+func (fc *FalcoClient) SetLoggers(loggers []pkgutils.Worker) {}
-func (f *FalcoClient) ReadConfig() {
- f.url = f.config.Loggers.FalcoClient.URL
+func (fc *FalcoClient) ReadConfig() {
+ fc.url = fc.config.Loggers.FalcoClient.URL
}
-func (f *FalcoClient) ReloadConfig(config *pkgconfig.Config) {
- f.LogInfo("reload configuration!")
- f.configChan <- config
+func (fc *FalcoClient) ReloadConfig(config *pkgconfig.Config) {
+ fc.LogInfo("reload configuration!")
+ fc.configChan <- config
}
-func (f *FalcoClient) Channel() chan dnsutils.DNSMessage {
- return f.inputChan
+func (fc *FalcoClient) GetInputChannel() chan dnsutils.DNSMessage {
+ return fc.inputChan
}
-func (f *FalcoClient) LogInfo(msg string, v ...interface{}) {
- f.logger.Info("["+f.name+"] logger=falco - "+msg, v...)
+func (fc *FalcoClient) LogInfo(msg string, v ...interface{}) {
+ fc.logger.Info("["+fc.name+"] logger=falco - "+msg, v...)
}
-func (f *FalcoClient) LogError(msg string, v ...interface{}) {
- f.logger.Error("["+f.name+"] logger=falco - "+msg, v...)
+func (fc *FalcoClient) LogError(msg string, v ...interface{}) {
+ fc.logger.Error("["+fc.name+"] logger=falco - "+msg, v...)
}
-func (f *FalcoClient) Stop() {
- f.LogInfo("stopping to run...")
- f.stopRun <- true
- <-f.doneRun
+func (fc *FalcoClient) Stop() {
+ fc.LogInfo("stopping routing handler...")
+ fc.RoutingHandler.Stop()
- f.LogInfo("stopping to process...")
- f.stopProcess <- true
- <-f.doneProcess
+ fc.LogInfo("stopping to run...")
+ fc.stopRun <- true
+ <-fc.doneRun
+
+ fc.LogInfo("stopping to process...")
+ fc.stopProcess <- true
+ <-fc.doneProcess
}
-func (f *FalcoClient) Run() {
- f.LogInfo("running in background...")
+func (fc *FalcoClient) Run() {
+ fc.LogInfo("running in background...")
+
+ // prepare next channels
+ defaultRoutes, defaultNames := fc.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := fc.RoutingHandler.GetDroppedRoutes()
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
- listChannel = append(listChannel, f.outputChan)
- subprocessors := transformers.NewTransforms(&f.config.OutgoingTransformers, f.logger, f.name, listChannel, 0)
+ listChannel = append(listChannel, fc.outputChan)
+ subprocessors := transformers.NewTransforms(&fc.config.OutgoingTransformers, fc.logger, fc.name, listChannel, 0)
// goroutine to process transformed dns messages
- go f.Process()
+ go fc.Process()
// loop to process incoming messages
RUN_LOOP:
for {
select {
- case <-f.stopRun:
+ case <-fc.stopRun:
// cleanup transformers
subprocessors.Reset()
- f.doneRun <- true
+ fc.doneRun <- true
break RUN_LOOP
- case cfg, opened := <-f.configChan:
+ case cfg, opened := <-fc.configChan:
if !opened {
return
}
- f.config = cfg
- f.ReadConfig()
+ fc.config = cfg
+ fc.ReadConfig()
subprocessors.ReloadConfig(&cfg.OutgoingTransformers)
- case dm, opened := <-f.inputChan:
+ case dm, opened := <-fc.inputChan:
if !opened {
- f.LogInfo("input channel closed!")
+ fc.LogInfo("input channel closed!")
return
}
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ fc.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ fc.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
- f.outputChan <- dm
+ fc.outputChan <- dm
}
}
- f.LogInfo("run terminated")
+ fc.LogInfo("run terminated")
}
-func (f *FalcoClient) Process() {
+func (fc *FalcoClient) Process() {
buffer := new(bytes.Buffer)
- f.LogInfo("ready to process")
+ fc.LogInfo("ready to process")
PROCESS_LOOP:
for {
select {
- case <-f.stopProcess:
- f.doneProcess <- true
+ case <-fc.stopProcess:
+ fc.doneProcess <- true
break PROCESS_LOOP
// incoming dns message to process
- case dm, opened := <-f.outputChan:
+ case dm, opened := <-fc.outputChan:
if !opened {
- f.LogInfo("output channel closed!")
+ fc.LogInfo("output channel closed!")
return
}
// encode
json.NewEncoder(buffer).Encode(dm)
- req, _ := http.NewRequest("POST", f.url, buffer)
+ req, _ := http.NewRequest("POST", fc.url, buffer)
req.Header.Set("Content-Type", "application/json")
client := &http.Client{
Timeout: 5 * time.Second,
}
_, err := client.Do(req)
if err != nil {
- f.LogError(err.Error())
+ fc.LogError(err.Error())
}
// finally reset the buffer for next iter
buffer.Reset()
}
}
- f.LogInfo("processing terminated")
+ fc.LogInfo("processing terminated")
}
diff --git a/loggers/falco_test.go b/loggers/falco_test.go
index 88ea3d8b..77056812 100644
--- a/loggers/falco_test.go
+++ b/loggers/falco_test.go
@@ -39,7 +39,7 @@ func Test_FalcoClient(t *testing.T) {
go g.Run()
dm := dnsutils.GetFakeDNSMessage()
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// accept conn
conn, err := fakeRcvr.Accept()
diff --git a/loggers/fluentd.go b/loggers/fluentd.go
index 9144f2f8..1f24314a 100644
--- a/loggers/fluentd.go
+++ b/loggers/fluentd.go
@@ -11,6 +11,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
"github.com/vmihailenco/msgpack"
@@ -34,11 +35,12 @@ type FluentdClient struct {
transportReconnect chan bool
writerReady bool
name string
+ RoutingHandler pkgutils.RoutingHandler
}
func NewFluentdClient(config *pkgconfig.Config, logger *logger.Logger, name string) *FluentdClient {
logger.Info("[%s] logger=fluentd - enabled", name)
- s := &FluentdClient{
+ fc := &FluentdClient{
stopProcess: make(chan bool),
doneProcess: make(chan bool),
stopRun: make(chan bool),
@@ -53,130 +55,140 @@ func NewFluentdClient(config *pkgconfig.Config, logger *logger.Logger, name stri
config: config,
configChan: make(chan *pkgconfig.Config),
name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
}
- s.ReadConfig()
+ fc.ReadConfig()
+ return fc
+}
+
+func (fc *FluentdClient) GetName() string { return fc.name }
- return s
+func (fc *FluentdClient) AddDroppedRoute(wrk pkgutils.Worker) {
+ fc.RoutingHandler.AddDroppedRoute(wrk)
}
-func (c *FluentdClient) GetName() string { return c.name }
+func (fc *FluentdClient) AddDefaultRoute(wrk pkgutils.Worker) {
+ fc.RoutingHandler.AddDefaultRoute(wrk)
+}
-func (c *FluentdClient) SetLoggers(loggers []dnsutils.Worker) {}
+func (fc *FluentdClient) SetLoggers(loggers []pkgutils.Worker) {}
-func (c *FluentdClient) ReadConfig() {
- c.transport = c.config.Loggers.Fluentd.Transport
+func (fc *FluentdClient) ReadConfig() {
+ fc.transport = fc.config.Loggers.Fluentd.Transport
// begin backward compatibility
- if c.config.Loggers.Fluentd.TLSSupport {
- c.transport = netlib.SocketTLS
+ if fc.config.Loggers.Fluentd.TLSSupport {
+ fc.transport = netlib.SocketTLS
}
- if len(c.config.Loggers.Fluentd.SockPath) > 0 {
- c.transport = netlib.SocketUnix
+ if len(fc.config.Loggers.Fluentd.SockPath) > 0 {
+ fc.transport = netlib.SocketUnix
}
- // end
}
-func (c *FluentdClient) ReloadConfig(config *pkgconfig.Config) {
- c.LogInfo("reload configuration!")
- c.configChan <- config
+func (fc *FluentdClient) ReloadConfig(config *pkgconfig.Config) {
+ fc.LogInfo("reload configuration!")
+ fc.configChan <- config
}
-func (c *FluentdClient) LogInfo(msg string, v ...interface{}) {
- c.logger.Info("["+c.name+"] logger=fluentd - "+msg, v...)
+func (fc *FluentdClient) LogInfo(msg string, v ...interface{}) {
+ fc.logger.Info("["+fc.name+"] logger=fluentd - "+msg, v...)
}
-func (c *FluentdClient) LogError(msg string, v ...interface{}) {
- c.logger.Error("["+c.name+"] logger=fluentd - "+msg, v...)
+func (fc *FluentdClient) LogError(msg string, v ...interface{}) {
+ fc.logger.Error("["+fc.name+"] logger=fluentd - "+msg, v...)
}
-func (c *FluentdClient) Channel() chan dnsutils.DNSMessage {
- return c.inputChan
+func (fc *FluentdClient) GetInputChannel() chan dnsutils.DNSMessage {
+ return fc.inputChan
}
-func (c *FluentdClient) Stop() {
- c.LogInfo("stopping to run...")
- c.stopRun <- true
- <-c.doneRun
+func (fc *FluentdClient) Stop() {
+ fc.LogInfo("stopping routing handler...")
+ fc.RoutingHandler.Stop()
- c.LogInfo("stopping to read...")
- c.stopRead <- true
- <-c.doneRead
+ fc.LogInfo("stopping to run...")
+ fc.stopRun <- true
+ <-fc.doneRun
- c.LogInfo("stopping to process...")
- c.stopProcess <- true
- <-c.doneProcess
+ fc.LogInfo("stopping to read...")
+ fc.stopRead <- true
+ <-fc.doneRead
+
+ fc.LogInfo("stopping to process...")
+ fc.stopProcess <- true
+ <-fc.doneProcess
}
-func (c *FluentdClient) Disconnect() {
- if c.transportConn != nil {
- c.LogInfo("closing tcp connection")
- c.transportConn.Close()
+func (fc *FluentdClient) Disconnect() {
+ if fc.transportConn != nil {
+ fc.LogInfo("closing tcp connection")
+ fc.transportConn.Close()
}
}
-func (c *FluentdClient) ReadFromConnection() {
+func (fc *FluentdClient) ReadFromConnection() {
buffer := make([]byte, 4096)
go func() {
for {
- _, err := c.transportConn.Read(buffer)
+ _, err := fc.transportConn.Read(buffer)
if err != nil {
if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
- c.LogInfo("read from connection terminated")
+ fc.LogInfo("read from connection terminated")
break
}
- c.LogError("Error on reading: %s", err.Error())
+ fc.LogError("Error on reading: %s", err.Error())
}
// We just discard the data
}
}()
// block goroutine until receive true event in stopRead channel
- <-c.stopRead
- c.doneRead <- true
+ <-fc.stopRead
+ fc.doneRead <- true
- c.LogInfo("read goroutine terminated")
+ fc.LogInfo("read goroutine terminated")
}
-func (c *FluentdClient) ConnectToRemote() {
+func (fc *FluentdClient) ConnectToRemote() {
for {
- if c.transportConn != nil {
- c.transportConn.Close()
- c.transportConn = nil
+ if fc.transportConn != nil {
+ fc.transportConn.Close()
+ fc.transportConn = nil
}
- address := c.config.Loggers.Fluentd.RemoteAddress + ":" + strconv.Itoa(c.config.Loggers.Fluentd.RemotePort)
- connTimeout := time.Duration(c.config.Loggers.Fluentd.ConnectTimeout) * time.Second
+ address := fc.config.Loggers.Fluentd.RemoteAddress + ":" + strconv.Itoa(fc.config.Loggers.Fluentd.RemotePort)
+ connTimeout := time.Duration(fc.config.Loggers.Fluentd.ConnectTimeout) * time.Second
// make the connection
var conn net.Conn
var err error
- switch c.transport {
+ switch fc.transport {
case netlib.SocketUnix:
- address = c.config.Loggers.Fluentd.RemoteAddress
- if len(c.config.Loggers.Fluentd.SockPath) > 0 {
- address = c.config.Loggers.Fluentd.SockPath
+ address = fc.config.Loggers.Fluentd.RemoteAddress
+ if len(fc.config.Loggers.Fluentd.SockPath) > 0 {
+ address = fc.config.Loggers.Fluentd.SockPath
}
- c.LogInfo("connecting to %s://%s", c.transport, address)
- conn, err = net.DialTimeout(c.transport, address, connTimeout)
+ fc.LogInfo("connecting to %s://%s", fc.transport, address)
+ conn, err = net.DialTimeout(fc.transport, address, connTimeout)
case netlib.SocketTCP:
- c.LogInfo("connecting to %s://%s", c.transport, address)
- conn, err = net.DialTimeout(c.transport, address, connTimeout)
+ fc.LogInfo("connecting to %s://%s", fc.transport, address)
+ conn, err = net.DialTimeout(fc.transport, address, connTimeout)
case netlib.SocketTLS:
- c.LogInfo("connecting to %s://%s", c.transport, address)
+ fc.LogInfo("connecting to %s://%s", fc.transport, address)
var tlsConfig *tls.Config
tlsOptions := pkgconfig.TLSOptions{
- InsecureSkipVerify: c.config.Loggers.Fluentd.TLSInsecure,
- MinVersion: c.config.Loggers.Fluentd.TLSMinVersion,
- CAFile: c.config.Loggers.Fluentd.CAFile,
- CertFile: c.config.Loggers.Fluentd.CertFile,
- KeyFile: c.config.Loggers.Fluentd.KeyFile,
+ InsecureSkipVerify: fc.config.Loggers.Fluentd.TLSInsecure,
+ MinVersion: fc.config.Loggers.Fluentd.TLSMinVersion,
+ CAFile: fc.config.Loggers.Fluentd.CAFile,
+ CertFile: fc.config.Loggers.Fluentd.CertFile,
+ KeyFile: fc.config.Loggers.Fluentd.KeyFile,
}
tlsConfig, err = pkgconfig.TLSClientConfig(tlsOptions)
@@ -185,37 +197,37 @@ func (c *FluentdClient) ConnectToRemote() {
conn, err = tls.DialWithDialer(dialer, netlib.SocketTCP, address, tlsConfig)
}
default:
- c.logger.Fatal("logger=fluent - invalid transport:", c.transport)
+ fc.logger.Fatal("logger=fluent - invalid transport:", fc.transport)
}
// something is wrong during connection ?
if err != nil {
- c.LogError("connect error: %s", err)
- c.LogInfo("retry to connect in %d seconds", c.config.Loggers.Fluentd.RetryInterval)
- time.Sleep(time.Duration(c.config.Loggers.Fluentd.RetryInterval) * time.Second)
+ fc.LogError("connect error: %s", err)
+ fc.LogInfo("retry to connect in %d seconds", fc.config.Loggers.Fluentd.RetryInterval)
+ time.Sleep(time.Duration(fc.config.Loggers.Fluentd.RetryInterval) * time.Second)
continue
}
- c.transportConn = conn
+ fc.transportConn = conn
// block until framestream is ready
- c.transportReady <- true
+ fc.transportReady <- true
// block until an error occured, need to reconnect
- c.transportReconnect <- true
+ fc.transportReconnect <- true
}
}
-func (c *FluentdClient) FlushBuffer(buf *[]dnsutils.DNSMessage) {
+func (fc *FluentdClient) FlushBuffer(buf *[]dnsutils.DNSMessage) {
- tag, _ := msgpack.Marshal(c.config.Loggers.Fluentd.Tag)
+ tag, _ := msgpack.Marshal(fc.config.Loggers.Fluentd.Tag)
for _, dm := range *buf {
// prepare event
tm, _ := msgpack.Marshal(dm.DNSTap.TimeSec)
record, err := msgpack.Marshal(dm)
if err != nil {
- c.LogError("msgpack error:", err.Error())
+ fc.LogError("msgpack error:", err.Error())
continue
}
@@ -229,104 +241,112 @@ func (c *FluentdClient) FlushBuffer(buf *[]dnsutils.DNSMessage) {
encoded = append(encoded, record...)
// write event message
- _, err = c.transportConn.Write(encoded)
+ _, err = fc.transportConn.Write(encoded)
// flusth the buffer
if err != nil {
- c.LogError("send transport error", err.Error())
- c.writerReady = false
- <-c.transportReconnect
+ fc.LogError("send transport error", err.Error())
+ fc.writerReady = false
+ <-fc.transportReconnect
break
}
}
}
-func (c *FluentdClient) Run() {
- c.LogInfo("running in background...")
+func (fc *FluentdClient) Run() {
+ fc.LogInfo("running in background...")
+
+ // prepare next channels
+ defaultRoutes, defaultNames := fc.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := fc.RoutingHandler.GetDroppedRoutes()
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
- listChannel = append(listChannel, c.outputChan)
- subprocessors := transformers.NewTransforms(&c.config.OutgoingTransformers, c.logger, c.name, listChannel, 0)
+ listChannel = append(listChannel, fc.outputChan)
+ subprocessors := transformers.NewTransforms(&fc.config.OutgoingTransformers, fc.logger, fc.name, listChannel, 0)
// goroutine to process transformed dns messages
- go c.Process()
+ go fc.Process()
// init remote conn
- go c.ConnectToRemote()
+ go fc.ConnectToRemote()
// loop to process incoming messages
RUN_LOOP:
for {
select {
- case <-c.stopRun:
+ case <-fc.stopRun:
// cleanup transformers
subprocessors.Reset()
- c.doneRun <- true
+ fc.doneRun <- true
break RUN_LOOP
- case cfg, opened := <-c.configChan:
+ case cfg, opened := <-fc.configChan:
if !opened {
return
}
- c.config = cfg
- c.ReadConfig()
+ fc.config = cfg
+ fc.ReadConfig()
subprocessors.ReloadConfig(&cfg.OutgoingTransformers)
- case dm, opened := <-c.inputChan:
+ case dm, opened := <-fc.inputChan:
if !opened {
- c.LogInfo("input channel closed!")
+ fc.LogInfo("input channel closed!")
return
}
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ fc.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ fc.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
- c.outputChan <- dm
+ fc.outputChan <- dm
}
}
- c.LogInfo("run terminated")
+ fc.LogInfo("run terminated")
}
-func (c *FluentdClient) Process() {
+func (fc *FluentdClient) Process() {
// init buffer
bufferDm := []dnsutils.DNSMessage{}
// init flust timer for buffer
- flushInterval := time.Duration(c.config.Loggers.Fluentd.FlushInterval) * time.Second
+ flushInterval := time.Duration(fc.config.Loggers.Fluentd.FlushInterval) * time.Second
flushTimer := time.NewTimer(flushInterval)
- c.LogInfo("ready to process")
+ fc.LogInfo("ready to process")
PROCESS_LOOP:
for {
select {
- case <-c.stopProcess:
- c.doneProcess <- true
+ case <-fc.stopProcess:
+ fc.doneProcess <- true
break PROCESS_LOOP
- case <-c.transportReady:
- c.LogInfo("connected")
- c.writerReady = true
+ case <-fc.transportReady:
+ fc.LogInfo("connected")
+ fc.writerReady = true
// read from the connection until we stop
- go c.ReadFromConnection()
+ go fc.ReadFromConnection()
// incoming dns message to process
- case dm, opened := <-c.outputChan:
+ case dm, opened := <-fc.outputChan:
if !opened {
- c.LogInfo("output channel closed!")
+ fc.LogInfo("output channel closed!")
return
}
// drop dns message if the connection is not ready to avoid memory leak or
// to block the channel
- if !c.writerReady {
+ if !fc.writerReady {
continue
}
@@ -334,24 +354,24 @@ PROCESS_LOOP:
bufferDm = append(bufferDm, dm)
// buffer is full ?
- if len(bufferDm) >= c.config.Loggers.Fluentd.BufferSize {
- c.FlushBuffer(&bufferDm)
+ if len(bufferDm) >= fc.config.Loggers.Fluentd.BufferSize {
+ fc.FlushBuffer(&bufferDm)
}
// flush the buffer
case <-flushTimer.C:
- if !c.writerReady {
+ if !fc.writerReady {
bufferDm = nil
continue
}
if len(bufferDm) > 0 {
- c.FlushBuffer(&bufferDm)
+ fc.FlushBuffer(&bufferDm)
}
// restart timer
flushTimer.Reset(flushInterval)
}
}
- c.LogInfo("processing terminated")
+ fc.LogInfo("processing terminated")
}
diff --git a/loggers/fluentd_test.go b/loggers/fluentd_test.go
index b8e6ff48..1afe8a7d 100644
--- a/loggers/fluentd_test.go
+++ b/loggers/fluentd_test.go
@@ -50,7 +50,7 @@ func Test_FluentdClient(t *testing.T) {
// send fake dns message to logger
time.Sleep(time.Second)
dm := dnsutils.GetFakeDNSMessage()
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// read data on fake server side
buf := make([]byte, 4096)
diff --git a/loggers/influxdb.go b/loggers/influxdb.go
index 4b3eabc3..f3ebb89a 100644
--- a/loggers/influxdb.go
+++ b/loggers/influxdb.go
@@ -5,6 +5,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
@@ -13,173 +14,194 @@ import (
)
type InfluxDBClient struct {
- stopProcess chan bool
- doneProcess chan bool
- stopRun chan bool
- doneRun chan bool
- inputChan chan dnsutils.DNSMessage
- outputChan chan dnsutils.DNSMessage
- config *pkgconfig.Config
- configChan chan *pkgconfig.Config
- logger *logger.Logger
- influxdbConn influxdb2.Client
- writeAPI api.WriteAPI
- name string
+ stopProcess chan bool
+ doneProcess chan bool
+ stopRun chan bool
+ doneRun chan bool
+ inputChan chan dnsutils.DNSMessage
+ outputChan chan dnsutils.DNSMessage
+ config *pkgconfig.Config
+ configChan chan *pkgconfig.Config
+ logger *logger.Logger
+ influxdbConn influxdb2.Client
+ writeAPI api.WriteAPI
+ name string
+ RoutingHandler pkgutils.RoutingHandler
}
func NewInfluxDBClient(config *pkgconfig.Config, logger *logger.Logger, name string) *InfluxDBClient {
logger.Info("[%s] logger=influxdb - enabled", name)
- s := &InfluxDBClient{
- stopProcess: make(chan bool),
- doneProcess: make(chan bool),
- stopRun: make(chan bool),
- doneRun: make(chan bool),
- inputChan: make(chan dnsutils.DNSMessage, config.Loggers.InfluxDB.ChannelBufferSize),
- outputChan: make(chan dnsutils.DNSMessage, config.Loggers.InfluxDB.ChannelBufferSize),
- logger: logger,
- config: config,
- configChan: make(chan *pkgconfig.Config),
- name: name,
+ ic := &InfluxDBClient{
+ stopProcess: make(chan bool),
+ doneProcess: make(chan bool),
+ stopRun: make(chan bool),
+ doneRun: make(chan bool),
+ inputChan: make(chan dnsutils.DNSMessage, config.Loggers.InfluxDB.ChannelBufferSize),
+ outputChan: make(chan dnsutils.DNSMessage, config.Loggers.InfluxDB.ChannelBufferSize),
+ logger: logger,
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
}
- s.ReadConfig()
+ ic.ReadConfig()
- return s
+ return ic
}
-func (i *InfluxDBClient) GetName() string { return i.name }
+func (ic *InfluxDBClient) GetName() string { return ic.name }
-func (i *InfluxDBClient) SetLoggers(loggers []dnsutils.Worker) {}
+func (ic *InfluxDBClient) AddDroppedRoute(wrk pkgutils.Worker) {
+ ic.RoutingHandler.AddDroppedRoute(wrk)
+}
+
+func (ic *InfluxDBClient) AddDefaultRoute(wrk pkgutils.Worker) {
+ ic.RoutingHandler.AddDefaultRoute(wrk)
+}
+
+func (ic *InfluxDBClient) SetLoggers(loggers []pkgutils.Worker) {}
-func (i *InfluxDBClient) ReadConfig() {}
+func (ic *InfluxDBClient) ReadConfig() {}
-func (i *InfluxDBClient) ReloadConfig(config *pkgconfig.Config) {
- i.LogInfo("reload configuration!")
- i.configChan <- config
+func (ic *InfluxDBClient) ReloadConfig(config *pkgconfig.Config) {
+ ic.LogInfo("reload configuration!")
+ ic.configChan <- config
}
-func (i *InfluxDBClient) LogInfo(msg string, v ...interface{}) {
- i.logger.Info("["+i.name+"] logger=influxdb - "+msg, v...)
+func (ic *InfluxDBClient) LogInfo(msg string, v ...interface{}) {
+ ic.logger.Info("["+ic.name+"] logger=influxdb - "+msg, v...)
}
-func (i *InfluxDBClient) LogError(msg string, v ...interface{}) {
- i.logger.Error("["+i.name+"] logger=influxdb - "+msg, v...)
+func (ic *InfluxDBClient) LogError(msg string, v ...interface{}) {
+ ic.logger.Error("["+ic.name+"] logger=influxdb - "+msg, v...)
}
-func (i *InfluxDBClient) Channel() chan dnsutils.DNSMessage {
- return i.inputChan
+func (ic *InfluxDBClient) GetInputChannel() chan dnsutils.DNSMessage {
+ return ic.inputChan
}
-func (i *InfluxDBClient) Stop() {
- i.LogInfo("stopping to run...")
- i.stopRun <- true
- <-i.doneRun
+func (ic *InfluxDBClient) Stop() {
+ ic.LogInfo("stopping routing handler...")
+ ic.RoutingHandler.Stop()
- i.LogInfo("stopping to process...")
- i.stopProcess <- true
- <-i.doneProcess
+ ic.LogInfo("stopping to run...")
+ ic.stopRun <- true
+ <-ic.doneRun
+
+ ic.LogInfo("stopping to process...")
+ ic.stopProcess <- true
+ <-ic.doneProcess
}
-func (i *InfluxDBClient) Run() {
- i.LogInfo("running in background...")
+func (ic *InfluxDBClient) Run() {
+ ic.LogInfo("running in background...")
+
+ // prepare next channels
+ defaultRoutes, defaultNames := ic.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := ic.RoutingHandler.GetDroppedRoutes()
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
- listChannel = append(listChannel, i.outputChan)
- subprocessors := transformers.NewTransforms(&i.config.OutgoingTransformers, i.logger, i.name, listChannel, 0)
+ listChannel = append(listChannel, ic.outputChan)
+ subprocessors := transformers.NewTransforms(&ic.config.OutgoingTransformers, ic.logger, ic.name, listChannel, 0)
// goroutine to process transformed dns messages
- go i.Process()
+ go ic.Process()
// loop to process incoming messages
RUN_LOOP:
for {
select {
- case <-i.stopRun:
+ case <-ic.stopRun:
// cleanup transformers
subprocessors.Reset()
- i.doneRun <- true
+ ic.doneRun <- true
break RUN_LOOP
- case cfg, opened := <-i.configChan:
+ case cfg, opened := <-ic.configChan:
if !opened {
return
}
- i.config = cfg
- i.ReadConfig()
+ ic.config = cfg
+ ic.ReadConfig()
subprocessors.ReloadConfig(&cfg.OutgoingTransformers)
- case dm, opened := <-i.inputChan:
+ case dm, opened := <-ic.inputChan:
if !opened {
- i.LogInfo("input channel closed!")
+ ic.LogInfo("input channel closed!")
return
}
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ ic.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ ic.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
- i.outputChan <- dm
+ ic.outputChan <- dm
}
}
- i.LogInfo("run terminated")
+ ic.LogInfo("run terminated")
}
-func (i *InfluxDBClient) Process() {
+func (ic *InfluxDBClient) Process() {
// prepare options for influxdb
opts := influxdb2.DefaultOptions()
opts.SetUseGZip(true)
- if i.config.Loggers.InfluxDB.TLSSupport {
+ if ic.config.Loggers.InfluxDB.TLSSupport {
tlsOptions := pkgconfig.TLSOptions{
- InsecureSkipVerify: i.config.Loggers.InfluxDB.TLSInsecure,
- MinVersion: i.config.Loggers.InfluxDB.TLSMinVersion,
- CAFile: i.config.Loggers.InfluxDB.CAFile,
- CertFile: i.config.Loggers.InfluxDB.CertFile,
- KeyFile: i.config.Loggers.InfluxDB.KeyFile,
+ InsecureSkipVerify: ic.config.Loggers.InfluxDB.TLSInsecure,
+ MinVersion: ic.config.Loggers.InfluxDB.TLSMinVersion,
+ CAFile: ic.config.Loggers.InfluxDB.CAFile,
+ CertFile: ic.config.Loggers.InfluxDB.CertFile,
+ KeyFile: ic.config.Loggers.InfluxDB.KeyFile,
}
tlsConfig, err := pkgconfig.TLSClientConfig(tlsOptions)
if err != nil {
- i.logger.Fatal("logger=influxdb - tls config failed:", err)
+ ic.logger.Fatal("logger=influxdb - tls config failed:", err)
}
opts.SetTLSConfig(tlsConfig)
}
// init the client
influxClient := influxdb2.NewClientWithOptions(
- i.config.Loggers.InfluxDB.ServerURL,
- i.config.Loggers.InfluxDB.AuthToken,
+ ic.config.Loggers.InfluxDB.ServerURL,
+ ic.config.Loggers.InfluxDB.AuthToken,
opts,
)
writeAPI := influxClient.WriteAPI(
- i.config.Loggers.InfluxDB.Organization,
- i.config.Loggers.InfluxDB.Bucket,
+ ic.config.Loggers.InfluxDB.Organization,
+ ic.config.Loggers.InfluxDB.Bucket,
)
- i.influxdbConn = influxClient
- i.writeAPI = writeAPI
+ ic.influxdbConn = influxClient
+ ic.writeAPI = writeAPI
- i.LogInfo("ready to process")
+ ic.LogInfo("ready to process")
PROCESS_LOOP:
for {
select {
- case <-i.stopProcess:
+ case <-ic.stopProcess:
// Force all unwritten data to be sent
- i.writeAPI.Flush()
+ ic.writeAPI.Flush()
// Ensures background processes finishes
- i.influxdbConn.Close()
- i.doneProcess <- true
+ ic.influxdbConn.Close()
+ ic.doneProcess <- true
break PROCESS_LOOP
// incoming dns message to process
- case dm, opened := <-i.outputChan:
+ case dm, opened := <-ic.outputChan:
if !opened {
- i.LogInfo("output channel closed!")
+ ic.LogInfo("output channel closed!")
return
}
@@ -195,8 +217,8 @@ PROCESS_LOOP:
SetTime(time.Unix(int64(dm.DNSTap.TimeSec), int64(dm.DNSTap.TimeNsec)))
// write asynchronously
- i.writeAPI.WritePoint(p)
+ ic.writeAPI.WritePoint(p)
}
}
- i.LogInfo("processing terminated")
+ ic.LogInfo("processing terminated")
}
diff --git a/loggers/influxdb_test.go b/loggers/influxdb_test.go
index eb0e346f..78233cbf 100644
--- a/loggers/influxdb_test.go
+++ b/loggers/influxdb_test.go
@@ -29,7 +29,7 @@ func Test_InfluxDB(t *testing.T) {
// send fake dns message to logger
dm := dnsutils.GetFakeDNSMessage()
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// accept conn
conn, err := fakeRcvr.Accept()
diff --git a/loggers/kafkaproducer.go b/loggers/kafkaproducer.go
index b11e3ec8..89d5ecb3 100644
--- a/loggers/kafkaproducer.go
+++ b/loggers/kafkaproducer.go
@@ -11,6 +11,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
"github.com/segmentio/kafka-go"
@@ -36,6 +37,7 @@ type KafkaProducer struct {
kafkaReconnect chan bool
kafkaConnected bool
compressCodec compress.Codec
+ RoutingHandler pkgutils.RoutingHandler
}
func NewKafkaProducer(config *pkgconfig.Config, logger *logger.Logger, name string) *KafkaProducer {
@@ -53,16 +55,24 @@ func NewKafkaProducer(config *pkgconfig.Config, logger *logger.Logger, name stri
kafkaReady: make(chan bool),
kafkaReconnect: make(chan bool),
name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
}
k.ReadConfig()
-
return k
}
func (k *KafkaProducer) GetName() string { return k.name }
-func (k *KafkaProducer) SetLoggers(loggers []dnsutils.Worker) {}
+func (k *KafkaProducer) AddDroppedRoute(wrk pkgutils.Worker) {
+ k.RoutingHandler.AddDroppedRoute(wrk)
+}
+
+func (k *KafkaProducer) AddDefaultRoute(wrk pkgutils.Worker) {
+ k.RoutingHandler.AddDefaultRoute(wrk)
+}
+
+func (k *KafkaProducer) SetLoggers(loggers []pkgutils.Worker) {}
func (k *KafkaProducer) ReadConfig() {
if len(k.config.Loggers.RedisPub.TextFormat) > 0 {
@@ -102,11 +112,14 @@ func (k *KafkaProducer) LogError(msg string, v ...interface{}) {
k.logger.Error("["+k.name+"] logger=kafka - "+msg, v...)
}
-func (k *KafkaProducer) Channel() chan dnsutils.DNSMessage {
+func (k *KafkaProducer) GetInputChannel() chan dnsutils.DNSMessage {
return k.inputChan
}
func (k *KafkaProducer) Stop() {
+ k.LogInfo("stopping routing handler...")
+ k.RoutingHandler.Stop()
+
k.LogInfo("stopping to run...")
k.stopRun <- true
<-k.doneRun
@@ -253,6 +266,10 @@ func (k *KafkaProducer) FlushBuffer(buf *[]dnsutils.DNSMessage) {
func (k *KafkaProducer) Run() {
k.LogInfo("running in background...")
+ // prepare next channels
+ defaultRoutes, defaultNames := k.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := k.RoutingHandler.GetDroppedRoutes()
+
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
listChannel = append(listChannel, k.outputChan)
@@ -289,9 +306,13 @@ RUN_LOOP:
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ k.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ k.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
k.outputChan <- dm
}
diff --git a/loggers/kafkaproducer_test.go b/loggers/kafkaproducer_test.go
index 23a538ce..f333afd4 100644
--- a/loggers/kafkaproducer_test.go
+++ b/loggers/kafkaproducer_test.go
@@ -90,7 +90,7 @@ func Test_KafkaProducer(t *testing.T) {
// send fake dns message to logger
dm := dnsutils.GetFakeDNSMessage()
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// just wait
time.Sleep(1 * time.Second)
diff --git a/loggers/logfile.go b/loggers/logfile.go
index 211cac59..f573b715 100644
--- a/loggers/logfile.go
+++ b/loggers/logfile.go
@@ -18,6 +18,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
"github.com/google/gopacket"
@@ -66,88 +67,101 @@ type LogFile struct {
commpressTimer *time.Timer
textFormat []string
name string
+ RoutingHandler pkgutils.RoutingHandler
}
func NewLogFile(config *pkgconfig.Config, logger *logger.Logger, name string) *LogFile {
logger.Info("[%s] logger=file - enabled", name)
- l := &LogFile{
- stopProcess: make(chan bool),
- doneProcess: make(chan bool),
- stopRun: make(chan bool),
- doneRun: make(chan bool),
- inputChan: make(chan dnsutils.DNSMessage, config.Loggers.LogFile.ChannelBufferSize),
- outputChan: make(chan dnsutils.DNSMessage, config.Loggers.LogFile.ChannelBufferSize),
- config: config,
- configChan: make(chan *pkgconfig.Config),
- logger: logger,
- name: name,
+ lf := &LogFile{
+ stopProcess: make(chan bool),
+ doneProcess: make(chan bool),
+ stopRun: make(chan bool),
+ doneRun: make(chan bool),
+ inputChan: make(chan dnsutils.DNSMessage, config.Loggers.LogFile.ChannelBufferSize),
+ outputChan: make(chan dnsutils.DNSMessage, config.Loggers.LogFile.ChannelBufferSize),
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ logger: logger,
+ name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
}
- l.ReadConfig()
+ lf.ReadConfig()
- if err := l.OpenFile(); err != nil {
- l.logger.Fatal("["+name+"] logger=file - unable to open output file:", err)
+ if err := lf.OpenFile(); err != nil {
+ lf.logger.Fatal("["+name+"] logger=file - unable to open output file:", err)
}
- return l
+ return lf
}
-func (l *LogFile) GetName() string { return l.name }
+func (lf *LogFile) GetName() string { return lf.name }
-func (l *LogFile) SetLoggers(loggers []dnsutils.Worker) {}
+func (lf *LogFile) AddDroppedRoute(wrk pkgutils.Worker) {
+ lf.RoutingHandler.AddDroppedRoute(wrk)
+}
+
+func (lf *LogFile) AddDefaultRoute(wrk pkgutils.Worker) {
+ lf.RoutingHandler.AddDefaultRoute(wrk)
+}
+
+func (lf *LogFile) SetLoggers(loggers []pkgutils.Worker) {}
-func (l *LogFile) Channel() chan dnsutils.DNSMessage {
- return l.inputChan
+func (lf *LogFile) GetInputChannel() chan dnsutils.DNSMessage {
+ return lf.inputChan
}
-func (l *LogFile) ReadConfig() {
- if !IsValidMode(l.config.Loggers.LogFile.Mode) {
- l.logger.Fatal("["+l.name+"] logger=file - invalid mode: ", l.config.Loggers.LogFile.Mode)
+func (lf *LogFile) ReadConfig() {
+ if !IsValidMode(lf.config.Loggers.LogFile.Mode) {
+ lf.logger.Fatal("["+lf.name+"] logger=file - invalid mode: ", lf.config.Loggers.LogFile.Mode)
}
- l.fileDir = filepath.Dir(l.config.Loggers.LogFile.FilePath)
- l.fileName = filepath.Base(l.config.Loggers.LogFile.FilePath)
- l.fileExt = filepath.Ext(l.fileName)
- l.filePrefix = strings.TrimSuffix(l.fileName, l.fileExt)
+ lf.fileDir = filepath.Dir(lf.config.Loggers.LogFile.FilePath)
+ lf.fileName = filepath.Base(lf.config.Loggers.LogFile.FilePath)
+ lf.fileExt = filepath.Ext(lf.fileName)
+ lf.filePrefix = strings.TrimSuffix(lf.fileName, lf.fileExt)
- if len(l.config.Loggers.LogFile.TextFormat) > 0 {
- l.textFormat = strings.Fields(l.config.Loggers.LogFile.TextFormat)
+ if len(lf.config.Loggers.LogFile.TextFormat) > 0 {
+ lf.textFormat = strings.Fields(lf.config.Loggers.LogFile.TextFormat)
} else {
- l.textFormat = strings.Fields(l.config.Global.TextFormat)
+ lf.textFormat = strings.Fields(lf.config.Global.TextFormat)
}
- l.LogInfo("running in mode: %s", l.config.Loggers.LogFile.Mode)
+ lf.LogInfo("running in mode: %s", lf.config.Loggers.LogFile.Mode)
}
-func (l *LogFile) ReloadConfig(config *pkgconfig.Config) {
- l.LogInfo("reload configuration!")
- l.configChan <- config
+func (lf *LogFile) ReloadConfig(config *pkgconfig.Config) {
+ lf.LogInfo("reload configuration!")
+ lf.configChan <- config
}
-func (l *LogFile) LogInfo(msg string, v ...interface{}) {
- l.logger.Info("["+l.name+"] logger=file - "+msg, v...)
+func (lf *LogFile) LogInfo(msg string, v ...interface{}) {
+ lf.logger.Info("["+lf.name+"] logger=file - "+msg, v...)
}
-func (l *LogFile) LogError(msg string, v ...interface{}) {
- l.logger.Error("["+l.name+"] logger=file - "+msg, v...)
+func (lf *LogFile) LogError(msg string, v ...interface{}) {
+ lf.logger.Error("["+lf.name+"] logger=file - "+msg, v...)
}
-func (l *LogFile) Stop() {
- l.LogInfo("stopping to run...")
- l.stopRun <- true
- <-l.doneRun
+func (lf *LogFile) Stop() {
+ lf.LogInfo("stopping routing handler...")
+ lf.RoutingHandler.Stop()
- l.LogInfo("stopping to process...")
- l.stopProcess <- true
- <-l.doneProcess
+ lf.LogInfo("stopping to run...")
+ lf.stopRun <- true
+ <-lf.doneRun
+
+ lf.LogInfo("stopping to process...")
+ lf.stopProcess <- true
+ <-lf.doneProcess
}
-func (l *LogFile) Cleanup() error {
- if l.config.Loggers.LogFile.MaxFiles == 0 {
+func (lf *LogFile) Cleanup() error {
+ if lf.config.Loggers.LogFile.MaxFiles == 0 {
return nil
}
// remove old files ? keep only max files number
- entries, err := os.ReadDir(l.fileDir)
+ entries, err := os.ReadDir(lf.fileDir)
if err != nil {
return err
}
@@ -159,7 +173,7 @@ func (l *LogFile) Cleanup() error {
}
// extract timestamp from filename
- re := regexp.MustCompile(`^` + l.filePrefix + `-(?P<ts>\d+)` + l.fileExt)
+ re := regexp.MustCompile(`^` + lf.filePrefix + `-(?P<ts>\d+)` + lf.fileExt)
matches := re.FindStringSubmatch(entry.Name())
if len(matches) == 0 {
@@ -177,13 +191,13 @@ func (l *LogFile) Cleanup() error {
sort.Ints(logFiles)
// too much log files ?
- diffNB := len(logFiles) - l.config.Loggers.LogFile.MaxFiles
+ diffNB := len(logFiles) - lf.config.Loggers.LogFile.MaxFiles
if diffNB > 0 {
for i := 0; i < diffNB; i++ {
- filename := fmt.Sprintf("%s-%d%s", l.filePrefix, logFiles[i], l.fileExt)
- f := filepath.Join(l.fileDir, filename)
+ filename := fmt.Sprintf("%s-%d%s", lf.filePrefix, logFiles[i], lf.fileExt)
+ f := filepath.Join(lf.fileDir, filename)
if _, err := os.Stat(f); os.IsNotExist(err) {
- f = filepath.Join(l.fileDir, filename+compressSuffix)
+ f = filepath.Join(lf.fileDir, filename+compressSuffix)
}
// ignore errors on deletion
@@ -194,55 +208,55 @@ func (l *LogFile) Cleanup() error {
return nil
}
-func (l *LogFile) OpenFile() error {
+func (lf *LogFile) OpenFile() error {
- fd, err := os.OpenFile(l.config.Loggers.LogFile.FilePath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
+ fd, err := os.OpenFile(lf.config.Loggers.LogFile.FilePath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
return err
}
- l.fileFd = fd
+ lf.fileFd = fd
- fileinfo, err := os.Stat(l.config.Loggers.LogFile.FilePath)
+ fileinfo, err := os.Stat(lf.config.Loggers.LogFile.FilePath)
if err != nil {
return err
}
- l.fileSize = fileinfo.Size()
+ lf.fileSize = fileinfo.Size()
- switch l.config.Loggers.LogFile.Mode {
+ switch lf.config.Loggers.LogFile.Mode {
case pkgconfig.ModeText, pkgconfig.ModeJSON, pkgconfig.ModeFlatJSON:
bufferSize := 4096
- l.writerPlain = bufio.NewWriterSize(fd, bufferSize)
+ lf.writerPlain = bufio.NewWriterSize(fd, bufferSize)
case pkgconfig.ModePCAP:
- l.writerPcap = pcapgo.NewWriter(fd)
- if l.fileSize == 0 {
- if err := l.writerPcap.WriteFileHeader(65536, layers.LinkTypeEthernet); err != nil {
+ lf.writerPcap = pcapgo.NewWriter(fd)
+ if lf.fileSize == 0 {
+ if err := lf.writerPcap.WriteFileHeader(65536, layers.LinkTypeEthernet); err != nil {
return err
}
}
case pkgconfig.ModeDNSTap:
fsOptions := &framestream.EncoderOptions{ContentType: []byte("protobuf:dnstap.Dnstap"), Bidirectional: false}
- l.writerDnstap, err = framestream.NewEncoder(fd, fsOptions)
+ lf.writerDnstap, err = framestream.NewEncoder(fd, fsOptions)
if err != nil {
return err
}
}
- l.LogInfo("file opened with success: %s", l.config.Loggers.LogFile.FilePath)
+ lf.LogInfo("file opened with success: %s", lf.config.Loggers.LogFile.FilePath)
return nil
}
-func (l *LogFile) GetMaxSize() int64 {
- return int64(1024*1024) * int64(l.config.Loggers.LogFile.MaxSize)
+func (lf *LogFile) GetMaxSize() int64 {
+ return int64(1024*1024) * int64(lf.config.Loggers.LogFile.MaxSize)
}
-func (l *LogFile) CompressFile() {
- entries, err := os.ReadDir(l.fileDir)
+func (lf *LogFile) CompressFile() {
+ entries, err := os.ReadDir(lf.fileDir)
if err != nil {
- l.LogError("unable to list all files: %s", err)
+ lf.LogError("unable to list all files: %s", err)
return
}
@@ -252,155 +266,155 @@ func (l *LogFile) CompressFile() {
continue
}
- matched, _ := regexp.MatchString(`^`+l.filePrefix+`-\d+`+l.fileExt+`$`, entry.Name())
+ matched, _ := regexp.MatchString(`^`+lf.filePrefix+`-\d+`+lf.fileExt+`$`, entry.Name())
if matched {
- src := filepath.Join(l.fileDir, entry.Name())
- dst := filepath.Join(l.fileDir, entry.Name()+compressSuffix)
+ src := filepath.Join(lf.fileDir, entry.Name())
+ dst := filepath.Join(lf.fileDir, entry.Name()+compressSuffix)
- fl, err := os.Open(src)
+ fd, err := os.Open(src)
if err != nil {
- l.LogError("compress - failed to open file: ", err)
+ lf.LogError("compress - failed to open file: ", err)
continue
}
- defer fl.Close()
+ defer fd.Close()
fi, err := os.Stat(src)
if err != nil {
- l.LogError("compress - failed to stat file: ", err)
+ lf.LogError("compress - failed to stat file: ", err)
continue
}
gzf, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, fi.Mode())
if err != nil {
- l.LogError("compress - failed to open compressed file: ", err)
+ lf.LogError("compress - failed to open compressed file: ", err)
continue
}
defer gzf.Close()
gz := gzip.NewWriter(gzf)
- if _, err := io.Copy(gz, fl); err != nil {
- l.LogError("compress - failed to compress file: ", err)
+ if _, err := io.Copy(gz, fd); err != nil {
+ lf.LogError("compress - failed to compress file: ", err)
os.Remove(dst)
continue
}
if err := gz.Close(); err != nil {
- l.LogError("compress - failed to close gz writer: ", err)
+ lf.LogError("compress - failed to close gz writer: ", err)
os.Remove(dst)
continue
}
if err := gzf.Close(); err != nil {
- l.LogError("compress - failed to close gz file: ", err)
+ lf.LogError("compress - failed to close gz file: ", err)
os.Remove(dst)
continue
}
- if err := fl.Close(); err != nil {
- l.LogError("compress - failed to close log file: ", err)
+ if err := fd.Close(); err != nil {
+ lf.LogError("compress - failed to close log file: ", err)
os.Remove(dst)
continue
}
if err := os.Remove(src); err != nil {
- l.LogError("compress - failed to remove log file: ", err)
+ lf.LogError("compress - failed to remove log file: ", err)
os.Remove(dst)
continue
}
// post rotate command?
- l.CompressPostRotateCommand(dst)
+ lf.CompressPostRotateCommand(dst)
}
}
- l.commpressTimer.Reset(time.Duration(l.config.Loggers.LogFile.CompressInterval) * time.Second)
+ lf.commpressTimer.Reset(time.Duration(lf.config.Loggers.LogFile.CompressInterval) * time.Second)
}
-func (l *LogFile) PostRotateCommand(filename string) {
- if len(l.config.Loggers.LogFile.PostRotateCommand) > 0 {
- l.LogInfo("execute postrotate command: %s", filename)
- _, err := exec.Command(l.config.Loggers.LogFile.PostRotateCommand, filename).Output()
+func (lf *LogFile) PostRotateCommand(filename string) {
+ if len(lf.config.Loggers.LogFile.PostRotateCommand) > 0 {
+ lf.LogInfo("execute postrotate command: %s", filename)
+ _, err := exec.Command(lf.config.Loggers.LogFile.PostRotateCommand, filename).Output()
if err != nil {
- l.LogError("postrotate command error: %s", err)
- } else if l.config.Loggers.LogFile.PostRotateDelete {
+ lf.LogError("postrotate command error: %s", err)
+ } else if lf.config.Loggers.LogFile.PostRotateDelete {
os.Remove(filename)
}
}
}
-func (l *LogFile) CompressPostRotateCommand(filename string) {
- if len(l.config.Loggers.LogFile.CompressPostCommand) > 0 {
+func (lf *LogFile) CompressPostRotateCommand(filename string) {
+ if len(lf.config.Loggers.LogFile.CompressPostCommand) > 0 {
- l.LogInfo("execute compress postrotate command: %s", filename)
- _, err := exec.Command(l.config.Loggers.LogFile.CompressPostCommand, filename).Output()
+ lf.LogInfo("execute compress postrotate command: %s", filename)
+ _, err := exec.Command(lf.config.Loggers.LogFile.CompressPostCommand, filename).Output()
if err != nil {
- l.LogError("compress - postcommand error: %s", err)
+ lf.LogError("compress - postcommand error: %s", err)
}
}
}
-func (l *LogFile) FlushWriters() {
- switch l.config.Loggers.LogFile.Mode {
+func (lf *LogFile) FlushWriters() {
+ switch lf.config.Loggers.LogFile.Mode {
case pkgconfig.ModeText, pkgconfig.ModeJSON, pkgconfig.ModeFlatJSON:
- l.writerPlain.Flush()
+ lf.writerPlain.Flush()
case pkgconfig.ModeDNSTap:
- l.writerDnstap.Flush()
+ lf.writerDnstap.Flush()
}
}
-func (l *LogFile) RotateFile() error {
+func (lf *LogFile) RotateFile() error {
// close writer and existing file
- l.FlushWriters()
+ lf.FlushWriters()
- if l.config.Loggers.LogFile.Mode == pkgconfig.ModeDNSTap {
- l.writerDnstap.Close()
+ if lf.config.Loggers.LogFile.Mode == pkgconfig.ModeDNSTap {
+ lf.writerDnstap.Close()
}
- if err := l.fileFd.Close(); err != nil {
+ if err := lf.fileFd.Close(); err != nil {
return err
}
// Rename current log file
- bfpath := filepath.Join(l.fileDir, fmt.Sprintf("%s-%d%s", l.filePrefix, time.Now().UnixNano(), l.fileExt))
- err := os.Rename(l.config.Loggers.LogFile.FilePath, bfpath)
+ bfpath := filepath.Join(lf.fileDir, fmt.Sprintf("%s-%d%s", lf.filePrefix, time.Now().UnixNano(), lf.fileExt))
+ err := os.Rename(lf.config.Loggers.LogFile.FilePath, bfpath)
if err != nil {
return err
}
// post rotate command?
- l.PostRotateCommand(bfpath)
+ lf.PostRotateCommand(bfpath)
// keep only max files
- err = l.Cleanup()
+ err = lf.Cleanup()
if err != nil {
- l.LogError("unable to cleanup log files: %s", err)
+ lf.LogError("unable to cleanup log files: %s", err)
return err
}
// re-create new one
- if err := l.OpenFile(); err != nil {
- l.LogError("unable to re-create file: %s", err)
+ if err := lf.OpenFile(); err != nil {
+ lf.LogError("unable to re-create file: %s", err)
return err
}
return nil
}
-func (l *LogFile) WriteToPcap(dm dnsutils.DNSMessage, pkt []gopacket.SerializableLayer) {
+func (lf *LogFile) WriteToPcap(dm dnsutils.DNSMessage, pkt []gopacket.SerializableLayer) {
// create the packet with the layers
buf := gopacket.NewSerializeBuffer()
opts := gopacket.SerializeOptions{
FixLengths: true,
ComputeChecksums: true,
}
- for _, l := range pkt {
- l.SerializeTo(buf, opts)
+ for _, layer := range pkt {
+ layer.SerializeTo(buf, opts)
}
// rotate pcap file ?
bufSize := len(buf.Bytes())
- if (l.fileSize + int64(bufSize)) > l.GetMaxSize() {
- if err := l.RotateFile(); err != nil {
- l.LogError("failed to rotate file: %s", err)
+ if (lf.fileSize + int64(bufSize)) > lf.GetMaxSize() {
+ if err := lf.RotateFile(); err != nil {
+ lf.LogError("failed to rotate file: %s", err)
return
}
}
@@ -411,198 +425,206 @@ func (l *LogFile) WriteToPcap(dm dnsutils.DNSMessage, pkt []gopacket.Serializabl
Length: bufSize,
}
- l.writerPcap.WritePacket(ci, buf.Bytes())
+ lf.writerPcap.WritePacket(ci, buf.Bytes())
// increase size file
- l.fileSize += int64(bufSize)
+ lf.fileSize += int64(bufSize)
}
-func (l *LogFile) WriteToPlain(data []byte) {
+func (lf *LogFile) WriteToPlain(data []byte) {
dataSize := int64(len(data))
// rotate file ?
- if (l.fileSize + dataSize) > l.GetMaxSize() {
- if err := l.RotateFile(); err != nil {
- l.LogError("failed to rotate file: %s", err)
+ if (lf.fileSize + dataSize) > lf.GetMaxSize() {
+ if err := lf.RotateFile(); err != nil {
+ lf.LogError("failed to rotate file: %s", err)
return
}
}
// write log to file
- n, _ := l.writerPlain.Write(data)
+ n, _ := lf.writerPlain.Write(data)
// increase size file
- l.fileSize += int64(n)
+ lf.fileSize += int64(n)
}
-func (l *LogFile) WriteToDnstap(data []byte) {
+func (lf *LogFile) WriteToDnstap(data []byte) {
dataSize := int64(len(data))
// rotate file ?
- if (l.fileSize + dataSize) > l.GetMaxSize() {
- if err := l.RotateFile(); err != nil {
- l.LogError("failed to rotate file: %s", err)
+ if (lf.fileSize + dataSize) > lf.GetMaxSize() {
+ if err := lf.RotateFile(); err != nil {
+ lf.LogError("failed to rotate file: %s", err)
return
}
}
// write log to file
- n, _ := l.writerDnstap.Write(data)
+ n, _ := lf.writerDnstap.Write(data)
// increase size file
- l.fileSize += int64(n)
+ lf.fileSize += int64(n)
}
-func (l *LogFile) Run() {
- l.LogInfo("running in background...")
+func (lf *LogFile) Run() {
+ lf.LogInfo("running in background...")
+
+ // prepare next channels
+ defaultRoutes, defaultNames := lf.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := lf.RoutingHandler.GetDroppedRoutes()
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
- listChannel = append(listChannel, l.outputChan)
- subprocessors := transformers.NewTransforms(&l.config.OutgoingTransformers, l.logger, l.name, listChannel, 0)
+ listChannel = append(listChannel, lf.outputChan)
+ subprocessors := transformers.NewTransforms(&lf.config.OutgoingTransformers, lf.logger, lf.name, listChannel, 0)
// goroutine to process transformed dns messages
- go l.Process()
+ go lf.Process()
// loop to process incoming messages
RUN_LOOP:
for {
select {
- case <-l.stopRun:
+ case <-lf.stopRun:
// cleanup transformers
subprocessors.Reset()
- l.doneRun <- true
+ lf.doneRun <- true
break RUN_LOOP
- case cfg, opened := <-l.configChan:
+ case cfg, opened := <-lf.configChan:
if !opened {
return
}
- l.config = cfg
- l.ReadConfig()
+ lf.config = cfg
+ lf.ReadConfig()
subprocessors.ReloadConfig(&cfg.OutgoingTransformers)
- case dm, opened := <-l.inputChan:
+ case dm, opened := <-lf.inputChan:
if !opened {
- l.LogInfo("input channel closed!")
+ lf.LogInfo("input channel closed!")
return
}
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ lf.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ lf.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
- l.outputChan <- dm
+ lf.outputChan <- dm
}
}
- l.LogInfo("run terminated")
+ lf.LogInfo("run terminated")
}
-func (l *LogFile) Process() {
+func (lf *LogFile) Process() {
// prepare some timers
- flushInterval := time.Duration(l.config.Loggers.LogFile.FlushInterval) * time.Second
+ flushInterval := time.Duration(lf.config.Loggers.LogFile.FlushInterval) * time.Second
flushTimer := time.NewTimer(flushInterval)
- l.commpressTimer = time.NewTimer(time.Duration(l.config.Loggers.LogFile.CompressInterval) * time.Second)
+ lf.commpressTimer = time.NewTimer(time.Duration(lf.config.Loggers.LogFile.CompressInterval) * time.Second)
buffer := new(bytes.Buffer)
var data []byte
var err error
- l.LogInfo("ready to process")
+ lf.LogInfo("ready to process")
PROCESS_LOOP:
for {
select {
- case <-l.stopProcess:
+ case <-lf.stopProcess:
// stop timer
flushTimer.Stop()
- l.commpressTimer.Stop()
+ lf.commpressTimer.Stop()
// flush writer
- l.FlushWriters()
+ lf.FlushWriters()
// closing file
- l.LogInfo("closing log file")
- if l.config.Loggers.LogFile.Mode == pkgconfig.ModeDNSTap {
- l.writerDnstap.Close()
+ lf.LogInfo("closing log file")
+ if lf.config.Loggers.LogFile.Mode == pkgconfig.ModeDNSTap {
+ lf.writerDnstap.Close()
}
- l.fileFd.Close()
+ lf.fileFd.Close()
- l.doneProcess <- true
+ lf.doneProcess <- true
break PROCESS_LOOP
- case dm, opened := <-l.outputChan:
+ case dm, opened := <-lf.outputChan:
if !opened {
- l.LogInfo("output channel closed!")
+ lf.LogInfo("output channel closed!")
return
}
// write to file
- switch l.config.Loggers.LogFile.Mode {
+ switch lf.config.Loggers.LogFile.Mode {
// with basic text mode
case pkgconfig.ModeText:
- l.WriteToPlain(dm.Bytes(l.textFormat,
- l.config.Global.TextFormatDelimiter,
- l.config.Global.TextFormatBoundary))
+ lf.WriteToPlain(dm.Bytes(lf.textFormat,
+ lf.config.Global.TextFormatDelimiter,
+ lf.config.Global.TextFormatBoundary))
var delimiter bytes.Buffer
delimiter.WriteString("\n")
- l.WriteToPlain(delimiter.Bytes())
+ lf.WriteToPlain(delimiter.Bytes())
// with json mode
case pkgconfig.ModeFlatJSON:
flat, err := dm.Flatten()
if err != nil {
- l.LogError("flattening DNS message failed: %e", err)
+ lf.LogError("flattening DNS message failed: %e", err)
}
json.NewEncoder(buffer).Encode(flat)
- l.WriteToPlain(buffer.Bytes())
+ lf.WriteToPlain(buffer.Bytes())
buffer.Reset()
// with json mode
case pkgconfig.ModeJSON:
json.NewEncoder(buffer).Encode(dm)
- l.WriteToPlain(buffer.Bytes())
+ lf.WriteToPlain(buffer.Bytes())
buffer.Reset()
// with dnstap mode
case pkgconfig.ModeDNSTap:
data, err = dm.ToDNSTap()
if err != nil {
- l.LogError("failed to encode to DNStap protobuf: %s", err)
+ lf.LogError("failed to encode to DNStap protobuf: %s", err)
continue
}
- l.WriteToDnstap(data)
+ lf.WriteToDnstap(data)
// with pcap mode
case pkgconfig.ModePCAP:
pkt, err := dm.ToPacketLayer()
if err != nil {
- l.LogError("failed to encode to packet layer: %s", err)
+ lf.LogError("failed to encode to packet layer: %s", err)
continue
}
// write the packet
- l.WriteToPcap(dm, pkt)
+ lf.WriteToPcap(dm, pkt)
}
case <-flushTimer.C:
// flush writer
- l.FlushWriters()
+ lf.FlushWriters()
// reset flush timer and buffer
buffer.Reset()
flushTimer.Reset(flushInterval)
- case <-l.commpressTimer.C:
- if l.config.Loggers.LogFile.Compress {
- l.CompressFile()
+ case <-lf.commpressTimer.C:
+ if lf.config.Loggers.LogFile.Compress {
+ lf.CompressFile()
}
}
}
- l.LogInfo("processing terminated")
+ lf.LogInfo("processing terminated")
}
diff --git a/loggers/logfile_test.go b/loggers/logfile_test.go
index 6522f4c7..e1bb9660 100644
--- a/loggers/logfile_test.go
+++ b/loggers/logfile_test.go
@@ -60,7 +60,7 @@ func Test_LogFileText(t *testing.T) {
// send fake dns message to logger
dm := dnsutils.GetFakeDNSMessage()
dm.DNSTap.Identity = dnsutils.DNSTapIdentityTest
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
time.Sleep(time.Second)
g.Stop()
diff --git a/loggers/lokiclient.go b/loggers/lokiclient.go
index 57fbac03..b4815fcd 100644
--- a/loggers/lokiclient.go
+++ b/loggers/lokiclient.go
@@ -15,6 +15,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
"github.com/gogo/protobuf/proto"
@@ -73,46 +74,55 @@ func (o *LokiStream) Encode2Proto() ([]byte, error) {
}
type LokiClient struct {
- stopProcess chan bool
- doneProcess chan bool
- stopRun chan bool
- doneRun chan bool
- inputChan chan dnsutils.DNSMessage
- outputChan chan dnsutils.DNSMessage
- config *pkgconfig.Config
- configChan chan *pkgconfig.Config
- logger *logger.Logger
- httpclient *http.Client
- textFormat []string
- streams map[string]*LokiStream
- name string
+ stopProcess chan bool
+ doneProcess chan bool
+ stopRun chan bool
+ doneRun chan bool
+ inputChan chan dnsutils.DNSMessage
+ outputChan chan dnsutils.DNSMessage
+ config *pkgconfig.Config
+ configChan chan *pkgconfig.Config
+ logger *logger.Logger
+ httpclient *http.Client
+ textFormat []string
+ streams map[string]*LokiStream
+ name string
+ RoutingHandler pkgutils.RoutingHandler
}
func NewLokiClient(config *pkgconfig.Config, logger *logger.Logger, name string) *LokiClient {
logger.Info("[%s] logger=loki - enabled", name)
s := &LokiClient{
- stopProcess: make(chan bool),
- doneProcess: make(chan bool),
- stopRun: make(chan bool),
- doneRun: make(chan bool),
- inputChan: make(chan dnsutils.DNSMessage, config.Loggers.LokiClient.ChannelBufferSize),
- outputChan: make(chan dnsutils.DNSMessage, config.Loggers.LokiClient.ChannelBufferSize),
- logger: logger,
- config: config,
- configChan: make(chan *pkgconfig.Config),
- streams: make(map[string]*LokiStream),
- name: name,
+ stopProcess: make(chan bool),
+ doneProcess: make(chan bool),
+ stopRun: make(chan bool),
+ doneRun: make(chan bool),
+ inputChan: make(chan dnsutils.DNSMessage, config.Loggers.LokiClient.ChannelBufferSize),
+ outputChan: make(chan dnsutils.DNSMessage, config.Loggers.LokiClient.ChannelBufferSize),
+ logger: logger,
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ streams: make(map[string]*LokiStream),
+ name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
}
s.ReadConfig()
-
return s
}
func (c *LokiClient) GetName() string { return c.name }
-func (c *LokiClient) SetLoggers(loggers []dnsutils.Worker) {}
+func (c *LokiClient) AddDroppedRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDroppedRoute(wrk)
+}
+
+func (c *LokiClient) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDefaultRoute(wrk)
+}
+
+func (c *LokiClient) SetLoggers(loggers []pkgutils.Worker) {}
func (c *LokiClient) ReadConfig() {
if len(c.config.Loggers.LokiClient.TextFormat) > 0 {
@@ -176,11 +186,14 @@ func (c *LokiClient) LogError(msg string, v ...interface{}) {
c.logger.Error("["+c.name+"] logger=loki - "+msg, v...)
}
-func (c *LokiClient) Channel() chan dnsutils.DNSMessage {
+func (c *LokiClient) GetInputChannel() chan dnsutils.DNSMessage {
return c.inputChan
}
func (c *LokiClient) Stop() {
+ c.LogInfo("stopping routing handler...")
+ c.RoutingHandler.Stop()
+
c.LogInfo("stopping to run...")
c.stopRun <- true
<-c.doneRun
@@ -193,6 +206,10 @@ func (c *LokiClient) Stop() {
func (c *LokiClient) Run() {
c.LogInfo("running in background...")
+ // prepare next channels
+ defaultRoutes, defaultNames := c.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := c.RoutingHandler.GetDroppedRoutes()
+
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
listChannel = append(listChannel, c.outputChan)
@@ -229,9 +246,13 @@ RUN_LOOP:
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ c.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ c.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
c.outputChan <- dm
}
diff --git a/loggers/lokiclient_test.go b/loggers/lokiclient_test.go
index b54fb8b7..0203e809 100644
--- a/loggers/lokiclient_test.go
+++ b/loggers/lokiclient_test.go
@@ -57,7 +57,7 @@ func Test_LokiClientRun(t *testing.T) {
// send fake dns message to logger
dm := dnsutils.GetFakeDNSMessage()
dm.DNSTap.Identity = dnsutils.DNSTapIdentityTest
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// accept conn
conn, err := fakeRcvr.Accept()
@@ -160,7 +160,7 @@ func Test_LokiClientRelabel(t *testing.T) {
// send fake dns message to logger
dm := dnsutils.GetFakeDNSMessage()
dm.DNSTap.Identity = dnsutils.DNSTapIdentityTest
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// accept conn
conn, err := fakeRcvr.Accept()
diff --git a/loggers/prometheus.go b/loggers/prometheus.go
index f4af353d..db5ea579 100644
--- a/loggers/prometheus.go
+++ b/loggers/prometheus.go
@@ -16,6 +16,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
"github.com/dmachard/go-topmap"
@@ -241,7 +242,8 @@ type Prometheus struct {
histogramQnamesLength *prometheus.HistogramVec
histogramLatencies *prometheus.HistogramVec
- name string
+ name string
+ RoutingHandler pkgutils.RoutingHandler
}
func newPrometheusCounterSet(p *Prometheus, labels prometheus.Labels) *PrometheusCountersSet {
@@ -759,18 +761,19 @@ func CreateSystemCatalogue(o *Prometheus) ([]string, *PromCounterCatalogueContai
func NewPrometheus(config *pkgconfig.Config, logger *logger.Logger, name string) *Prometheus {
logger.Info("[%s] logger=prometheus - enabled", name)
o := &Prometheus{
- doneAPI: make(chan bool),
- stopProcess: make(chan bool),
- doneProcess: make(chan bool),
- stopRun: make(chan bool),
- doneRun: make(chan bool),
- config: config,
- configChan: make(chan *pkgconfig.Config),
- inputChan: make(chan dnsutils.DNSMessage, config.Loggers.Prometheus.ChannelBufferSize),
- outputChan: make(chan dnsutils.DNSMessage, config.Loggers.Prometheus.ChannelBufferSize),
- logger: logger,
- promRegistry: prometheus.NewPedanticRegistry(),
- name: name,
+ doneAPI: make(chan bool),
+ stopProcess: make(chan bool),
+ doneProcess: make(chan bool),
+ stopRun: make(chan bool),
+ doneRun: make(chan bool),
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ inputChan: make(chan dnsutils.DNSMessage, config.Loggers.Prometheus.ChannelBufferSize),
+ outputChan: make(chan dnsutils.DNSMessage, config.Loggers.Prometheus.ChannelBufferSize),
+ logger: logger,
+ promRegistry: prometheus.NewPedanticRegistry(),
+ name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
}
// This will create a catalogue of counters indexed by fileds requested by config
@@ -812,7 +815,15 @@ func NewPrometheus(config *pkgconfig.Config, logger *logger.Logger, name string)
func (c *Prometheus) GetName() string { return c.name }
-func (c *Prometheus) SetLoggers(loggers []dnsutils.Worker) {}
+func (c *Prometheus) AddDroppedRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDroppedRoute(wrk)
+}
+
+func (c *Prometheus) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDefaultRoute(wrk)
+}
+
+func (c *Prometheus) SetLoggers(loggers []pkgutils.Worker) {}
func (c *Prometheus) InitProm() {
@@ -1113,11 +1124,14 @@ func (c *Prometheus) LogError(msg string, v ...interface{}) {
c.logger.Error("["+c.name+"] logger=prometheus - "+msg, v...)
}
-func (c *Prometheus) Channel() chan dnsutils.DNSMessage {
+func (c *Prometheus) GetInputChannel() chan dnsutils.DNSMessage {
return c.inputChan
}
func (c *Prometheus) Stop() {
+ c.LogInfo("stopping routing handler...")
+ c.RoutingHandler.Stop()
+
c.LogInfo("stopping to run...")
c.stopRun <- true
<-c.doneRun
@@ -1219,6 +1233,10 @@ func (c *Prometheus) ListenAndServe() {
func (c *Prometheus) Run() {
c.LogInfo("running in background...")
+ // prepare next channels
+ defaultRoutes, defaultNames := c.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := c.RoutingHandler.GetDroppedRoutes()
+
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
listChannel = append(listChannel, c.outputChan)
@@ -1257,9 +1275,13 @@ RUN_LOOP:
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ c.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ c.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
c.outputChan <- dm
}
diff --git a/loggers/redispub.go b/loggers/redispub.go
index 37e50885..cbb90c6e 100644
--- a/loggers/redispub.go
+++ b/loggers/redispub.go
@@ -15,6 +15,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
)
@@ -39,6 +40,7 @@ type RedisPub struct {
transportReady chan bool
transportReconnect chan bool
writerReady bool
+ RoutingHandler pkgutils.RoutingHandler
}
func NewRedisPub(config *pkgconfig.Config, logger *logger.Logger, name string) *RedisPub {
@@ -58,6 +60,7 @@ func NewRedisPub(config *pkgconfig.Config, logger *logger.Logger, name string) *
config: config,
configChan: make(chan *pkgconfig.Config),
name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
}
s.ReadConfig()
@@ -67,7 +70,15 @@ func NewRedisPub(config *pkgconfig.Config, logger *logger.Logger, name string) *
func (c *RedisPub) GetName() string { return c.name }
-func (c *RedisPub) SetLoggers(loggers []dnsutils.Worker) {}
+func (c *RedisPub) AddDroppedRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDroppedRoute(wrk)
+}
+
+func (c *RedisPub) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDefaultRoute(wrk)
+}
+
+func (c *RedisPub) SetLoggers(loggers []pkgutils.Worker) {}
func (c *RedisPub) ReadConfig() {
@@ -102,11 +113,14 @@ func (c *RedisPub) LogError(msg string, v ...interface{}) {
c.logger.Error("["+c.name+"] logger=redispub - "+msg, v...)
}
-func (c *RedisPub) Channel() chan dnsutils.DNSMessage {
+func (c *RedisPub) GetInputChannel() chan dnsutils.DNSMessage {
return c.inputChan
}
func (c *RedisPub) Stop() {
+ c.LogInfo("stopping routing handler...")
+ c.RoutingHandler.Stop()
+
c.LogInfo("stopping to run...")
c.stopRun <- true
<-c.doneRun
@@ -269,6 +283,10 @@ func (c *RedisPub) FlushBuffer(buf *[]dnsutils.DNSMessage) {
func (c *RedisPub) Run() {
c.LogInfo("running in background...")
+ // prepare next channels
+ defaultRoutes, defaultNames := c.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := c.RoutingHandler.GetDroppedRoutes()
+
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
listChannel = append(listChannel, c.outputChan)
@@ -305,9 +323,13 @@ RUN_LOOP:
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ c.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ c.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
c.outputChan <- dm
}
diff --git a/loggers/redispub_test.go b/loggers/redispub_test.go
index 6270b5d0..f832a524 100644
--- a/loggers/redispub_test.go
+++ b/loggers/redispub_test.go
@@ -64,7 +64,7 @@ func Test_RedisPubRun(t *testing.T) {
// send fake dns message to logger
dm := dnsutils.GetFakeDNSMessage()
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// read data on server side and decode-it
reader := bufio.NewReader(conn)
diff --git a/loggers/restapi.go b/loggers/restapi.go
index b8c5df4e..d0db230f 100644
--- a/loggers/restapi.go
+++ b/loggers/restapi.go
@@ -11,6 +11,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
"github.com/dmachard/go-topmap"
@@ -45,19 +46,20 @@ type KeyHit struct {
}
type RestAPI struct {
- doneAPI chan bool
- stopProcess chan bool
- doneProcess chan bool
- stopRun chan bool
- doneRun chan bool
- inputChan chan dnsutils.DNSMessage
- outputChan chan dnsutils.DNSMessage
- httpserver net.Listener
- httpmux *http.ServeMux
- config *pkgconfig.Config
- configChan chan *pkgconfig.Config
- logger *logger.Logger
- name string
+ doneAPI chan bool
+ stopProcess chan bool
+ doneProcess chan bool
+ stopRun chan bool
+ doneRun chan bool
+ inputChan chan dnsutils.DNSMessage
+ outputChan chan dnsutils.DNSMessage
+ httpserver net.Listener
+ httpmux *http.ServeMux
+ config *pkgconfig.Config
+ configChan chan *pkgconfig.Config
+ logger *logger.Logger
+ name string
+ RoutingHandler pkgutils.RoutingHandler
HitsStream HitsStream
HitsUniq HitsUniq
@@ -76,17 +78,18 @@ type RestAPI struct {
func NewRestAPI(config *pkgconfig.Config, logger *logger.Logger, name string) *RestAPI {
logger.Info("[%s] logger=restapi - enabled", name)
o := &RestAPI{
- doneAPI: make(chan bool),
- stopProcess: make(chan bool),
- doneProcess: make(chan bool),
- stopRun: make(chan bool),
- doneRun: make(chan bool),
- config: config,
- configChan: make(chan *pkgconfig.Config),
- inputChan: make(chan dnsutils.DNSMessage, config.Loggers.RestAPI.ChannelBufferSize),
- outputChan: make(chan dnsutils.DNSMessage, config.Loggers.RestAPI.ChannelBufferSize),
- logger: logger,
- name: name,
+ doneAPI: make(chan bool),
+ stopProcess: make(chan bool),
+ doneProcess: make(chan bool),
+ stopRun: make(chan bool),
+ doneRun: make(chan bool),
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ inputChan: make(chan dnsutils.DNSMessage, config.Loggers.RestAPI.ChannelBufferSize),
+ outputChan: make(chan dnsutils.DNSMessage, config.Loggers.RestAPI.ChannelBufferSize),
+ logger: logger,
+ name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
HitsStream: HitsStream{
Streams: make(map[string]SearchBy),
@@ -113,7 +116,15 @@ func NewRestAPI(config *pkgconfig.Config, logger *logger.Logger, name string) *R
func (c *RestAPI) GetName() string { return c.name }
-func (c *RestAPI) SetLoggers(loggers []dnsutils.Worker) {}
+func (c *RestAPI) AddDroppedRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDroppedRoute(wrk)
+}
+
+func (c *RestAPI) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDefaultRoute(wrk)
+}
+
+func (c *RestAPI) SetLoggers(loggers []pkgutils.Worker) {}
func (c *RestAPI) ReadConfig() {
if !pkgconfig.IsValidTLS(c.config.Loggers.RestAPI.TLSMinVersion) {
@@ -134,11 +145,14 @@ func (c *RestAPI) LogError(msg string, v ...interface{}) {
c.logger.Error("["+c.name+"] logger=restapi - "+msg, v...)
}
-func (c *RestAPI) Channel() chan dnsutils.DNSMessage {
+func (c *RestAPI) GetInputChannel() chan dnsutils.DNSMessage {
return c.inputChan
}
func (c *RestAPI) Stop() {
+ c.LogInfo("stopping routing handler...")
+ c.RoutingHandler.Stop()
+
c.LogInfo("stopping to run...")
c.stopRun <- true
<-c.doneRun
@@ -697,6 +711,10 @@ func (c *RestAPI) ListenAndServe() {
}
func (c *RestAPI) Run() {
+ // prepare next channels
+ defaultRoutes, defaultNames := c.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := c.RoutingHandler.GetDroppedRoutes()
+
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
listChannel = append(listChannel, c.outputChan)
@@ -736,9 +754,13 @@ RUN_LOOP:
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ c.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ c.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
c.outputChan <- dm
}
diff --git a/loggers/scalyr.go b/loggers/scalyr.go
index 4e495392..daf63efc 100644
--- a/loggers/scalyr.go
+++ b/loggers/scalyr.go
@@ -18,6 +18,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
)
@@ -25,18 +26,20 @@ import (
// ScalyrClient is a client for Scalyr(https://www.dataset.com/)
// This client is using the addEvents endpoint, described here: https://app.scalyr.com/help/api#addEvents
type ScalyrClient struct {
- stopProcess chan bool
- doneProcess chan bool
- stopRun chan bool
- doneRun chan bool
- inputChan chan dnsutils.DNSMessage
- outputChan chan dnsutils.DNSMessage
- logger *logger.Logger
- name string
- config *pkgconfig.Config
- configChan chan *pkgconfig.Config
- mode string
- textFormat []string
+ stopProcess chan bool
+ doneProcess chan bool
+ stopRun chan bool
+ doneRun chan bool
+ inputChan chan dnsutils.DNSMessage
+ outputChan chan dnsutils.DNSMessage
+ logger *logger.Logger
+ name string
+ config *pkgconfig.Config
+ configChan chan *pkgconfig.Config
+ RoutingHandler pkgutils.RoutingHandler
+
+ mode string
+ textFormat []string
session string // Session ID, used by scalyr, see API docs
@@ -54,17 +57,19 @@ type ScalyrClient struct {
func NewScalyrClient(config *pkgconfig.Config, console *logger.Logger, name string) *ScalyrClient {
console.Info("[%s] logger=scalyr - starting", name)
c := &ScalyrClient{
- stopProcess: make(chan bool),
- doneProcess: make(chan bool),
- stopRun: make(chan bool),
- doneRun: make(chan bool),
- inputChan: make(chan dnsutils.DNSMessage, config.Loggers.ScalyrClient.ChannelBufferSize),
- outputChan: make(chan dnsutils.DNSMessage, config.Loggers.ScalyrClient.ChannelBufferSize),
- logger: console,
- name: name,
- config: config,
- configChan: make(chan *pkgconfig.Config),
- mode: pkgconfig.ModeText,
+ stopProcess: make(chan bool),
+ doneProcess: make(chan bool),
+ stopRun: make(chan bool),
+ doneRun: make(chan bool),
+ inputChan: make(chan dnsutils.DNSMessage, config.Loggers.ScalyrClient.ChannelBufferSize),
+ outputChan: make(chan dnsutils.DNSMessage, config.Loggers.ScalyrClient.ChannelBufferSize),
+ logger: console,
+ name: name,
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ RoutingHandler: pkgutils.NewRoutingHandler(config, console, name),
+
+ mode: pkgconfig.ModeText,
endpoint: makeEndpoint("app.scalyr.com"),
flush: time.NewTicker(30 * time.Second),
@@ -153,6 +158,10 @@ func (c *ScalyrClient) ReloadConfig(config *pkgconfig.Config) {
func (c *ScalyrClient) Run() {
c.LogInfo("running in background...")
+ // prepare next channels
+ defaultRoutes, defaultNames := c.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := c.RoutingHandler.GetDroppedRoutes()
+
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
listChannel = append(listChannel, c.outputChan)
@@ -189,9 +198,13 @@ RUN_LOOP:
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ c.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ c.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
c.outputChan <- dm
}
@@ -289,6 +302,9 @@ PROCESS_LOOP:
}
func (c ScalyrClient) Stop() {
+ c.LogInfo("stopping routing handler...")
+ c.RoutingHandler.Stop()
+
c.LogInfo("stopping to run...")
c.stopRun <- true
<-c.doneRun
@@ -437,8 +453,16 @@ type response struct {
func (c *ScalyrClient) GetName() string { return c.name }
-func (c *ScalyrClient) SetLoggers(loggers []dnsutils.Worker) {}
+func (c *ScalyrClient) AddDroppedRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDroppedRoute(wrk)
+}
+
+func (c *ScalyrClient) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDefaultRoute(wrk)
+}
+
+func (c *ScalyrClient) SetLoggers(loggers []pkgutils.Worker) {}
-func (c *ScalyrClient) Channel() chan dnsutils.DNSMessage {
+func (c *ScalyrClient) GetInputChannel() chan dnsutils.DNSMessage {
return c.inputChan
}
diff --git a/loggers/statsd.go b/loggers/statsd.go
index 80a72b97..cc9b93c7 100644
--- a/loggers/statsd.go
+++ b/loggers/statsd.go
@@ -12,6 +12,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
"github.com/dmachard/go-topmap"
@@ -44,16 +45,17 @@ type StreamStats struct {
}
type StatsdClient struct {
- stopProcess chan bool
- doneProcess chan bool
- stopRun chan bool
- doneRun chan bool
- inputChan chan dnsutils.DNSMessage
- outputChan chan dnsutils.DNSMessage
- config *pkgconfig.Config
- configChan chan *pkgconfig.Config
- logger *logger.Logger
- name string
+ stopProcess chan bool
+ doneProcess chan bool
+ stopRun chan bool
+ doneRun chan bool
+ inputChan chan dnsutils.DNSMessage
+ outputChan chan dnsutils.DNSMessage
+ config *pkgconfig.Config
+ configChan chan *pkgconfig.Config
+ logger *logger.Logger
+ name string
+ RoutingHandler pkgutils.RoutingHandler
Stats StreamStats
sync.RWMutex
@@ -63,17 +65,18 @@ func NewStatsdClient(config *pkgconfig.Config, logger *logger.Logger, name strin
logger.Info("[%s] logger=statsd - enabled", name)
s := &StatsdClient{
- stopProcess: make(chan bool),
- doneProcess: make(chan bool),
- stopRun: make(chan bool),
- doneRun: make(chan bool),
- inputChan: make(chan dnsutils.DNSMessage, config.Loggers.Statsd.ChannelBufferSize),
- outputChan: make(chan dnsutils.DNSMessage, config.Loggers.Statsd.ChannelBufferSize),
- logger: logger,
- config: config,
- configChan: make(chan *pkgconfig.Config),
- name: name,
- Stats: StreamStats{Streams: make(map[string]*StatsPerStream)},
+ stopProcess: make(chan bool),
+ doneProcess: make(chan bool),
+ stopRun: make(chan bool),
+ doneRun: make(chan bool),
+ inputChan: make(chan dnsutils.DNSMessage, config.Loggers.Statsd.ChannelBufferSize),
+ outputChan: make(chan dnsutils.DNSMessage, config.Loggers.Statsd.ChannelBufferSize),
+ logger: logger,
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ name: name,
+ Stats: StreamStats{Streams: make(map[string]*StatsPerStream)},
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
}
// check config
@@ -84,7 +87,15 @@ func NewStatsdClient(config *pkgconfig.Config, logger *logger.Logger, name strin
func (c *StatsdClient) GetName() string { return c.name }
-func (c *StatsdClient) SetLoggers(loggers []dnsutils.Worker) {}
+func (c *StatsdClient) AddDroppedRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDroppedRoute(wrk)
+}
+
+func (c *StatsdClient) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDefaultRoute(wrk)
+}
+
+func (c *StatsdClient) SetLoggers(loggers []pkgutils.Worker) {}
func (c *StatsdClient) ReadConfig() {
if !pkgconfig.IsValidTLS(c.config.Loggers.Statsd.TLSMinVersion) {
@@ -105,11 +116,14 @@ func (c *StatsdClient) LogError(msg string, v ...interface{}) {
c.logger.Error("["+c.name+"] logger=statsd - "+msg, v...)
}
-func (c *StatsdClient) Channel() chan dnsutils.DNSMessage {
+func (c *StatsdClient) GetInputChannel() chan dnsutils.DNSMessage {
return c.inputChan
}
func (c *StatsdClient) Stop() {
+ c.LogInfo("stopping routing handler...")
+ c.RoutingHandler.Stop()
+
c.LogInfo("stopping to run...")
c.stopRun <- true
<-c.doneRun
@@ -235,6 +249,10 @@ func (c *StatsdClient) RecordDNSMessage(dm dnsutils.DNSMessage) {
func (c *StatsdClient) Run() {
c.LogInfo("running in background...")
+ // prepare next channels
+ defaultRoutes, defaultNames := c.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := c.RoutingHandler.GetDroppedRoutes()
+
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
listChannel = append(listChannel, c.outputChan)
@@ -271,9 +289,13 @@ RUN_LOOP:
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ c.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ c.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
c.outputChan <- dm
}
diff --git a/loggers/statsd_test.go b/loggers/statsd_test.go
index 6874c41f..91ae02d0 100644
--- a/loggers/statsd_test.go
+++ b/loggers/statsd_test.go
@@ -29,7 +29,7 @@ func TestStatsdRun(t *testing.T) {
// send fake dns message to logger
dm := dnsutils.GetFakeDNSMessage()
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// read data on fake server side
buf := make([]byte, 4096)
diff --git a/loggers/stdout.go b/loggers/stdout.go
index 3d1a407a..e22afa3c 100644
--- a/loggers/stdout.go
+++ b/loggers/stdout.go
@@ -11,6 +11,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
"github.com/google/gopacket"
@@ -31,179 +32,200 @@ func IsStdoutValidMode(mode string) bool {
}
type StdOut struct {
- stopProcess chan bool
- doneProcess chan bool
- stopRun chan bool
- doneRun chan bool
- inputChan chan dnsutils.DNSMessage
- outputChan chan dnsutils.DNSMessage
- textFormat []string
- config *pkgconfig.Config
- configChan chan *pkgconfig.Config
- logger *logger.Logger
- writerText *log.Logger
- writerPcap *pcapgo.Writer
- name string
+ stopProcess chan bool
+ doneProcess chan bool
+ stopRun chan bool
+ doneRun chan bool
+ inputChan chan dnsutils.DNSMessage
+ outputChan chan dnsutils.DNSMessage
+ textFormat []string
+ config *pkgconfig.Config
+ configChan chan *pkgconfig.Config
+ logger *logger.Logger
+ writerText *log.Logger
+ writerPcap *pcapgo.Writer
+ name string
+ RoutingHandler pkgutils.RoutingHandler
}
func NewStdOut(config *pkgconfig.Config, console *logger.Logger, name string) *StdOut {
console.Info("[%s] logger=stdout - enabled", name)
- o := &StdOut{
- stopProcess: make(chan bool),
- doneProcess: make(chan bool),
- stopRun: make(chan bool),
- doneRun: make(chan bool),
- inputChan: make(chan dnsutils.DNSMessage, config.Loggers.Stdout.ChannelBufferSize),
- outputChan: make(chan dnsutils.DNSMessage, config.Loggers.Stdout.ChannelBufferSize),
- logger: console,
- config: config,
- configChan: make(chan *pkgconfig.Config),
- writerText: log.New(os.Stdout, "", 0),
- name: name,
+ so := &StdOut{
+ stopProcess: make(chan bool),
+ doneProcess: make(chan bool),
+ stopRun: make(chan bool),
+ doneRun: make(chan bool),
+ inputChan: make(chan dnsutils.DNSMessage, config.Loggers.Stdout.ChannelBufferSize),
+ outputChan: make(chan dnsutils.DNSMessage, config.Loggers.Stdout.ChannelBufferSize),
+ logger: console,
+ config: config,
+ configChan: make(chan *pkgconfig.Config),
+ writerText: log.New(os.Stdout, "", 0),
+ name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, console, name),
}
- o.ReadConfig()
- return o
+ so.ReadConfig()
+ return so
}
-func (c *StdOut) GetName() string { return c.name }
+func (so *StdOut) GetName() string { return so.name }
-func (c *StdOut) SetLoggers(loggers []dnsutils.Worker) {}
+func (so *StdOut) AddDroppedRoute(wrk pkgutils.Worker) {
+ so.RoutingHandler.AddDroppedRoute(wrk)
+}
+
+func (so *StdOut) AddDefaultRoute(wrk pkgutils.Worker) {
+ so.RoutingHandler.AddDefaultRoute(wrk)
+}
+
+func (so *StdOut) SetLoggers(loggers []pkgutils.Worker) {}
-func (c *StdOut) ReadConfig() {
- if !IsStdoutValidMode(c.config.Loggers.Stdout.Mode) {
- c.logger.Fatal("["+c.name+"] logger=stdout - invalid mode: ", c.config.Loggers.Stdout.Mode)
+func (so *StdOut) ReadConfig() {
+ if !IsStdoutValidMode(so.config.Loggers.Stdout.Mode) {
+ so.logger.Fatal("["+so.name+"] logger=stdout - invalid mode: ", so.config.Loggers.Stdout.Mode)
}
- if len(c.config.Loggers.Stdout.TextFormat) > 0 {
- c.textFormat = strings.Fields(c.config.Loggers.Stdout.TextFormat)
+ if len(so.config.Loggers.Stdout.TextFormat) > 0 {
+ so.textFormat = strings.Fields(so.config.Loggers.Stdout.TextFormat)
} else {
- c.textFormat = strings.Fields(c.config.Global.TextFormat)
+ so.textFormat = strings.Fields(so.config.Global.TextFormat)
}
}
-func (c *StdOut) ReloadConfig(config *pkgconfig.Config) {
- c.LogInfo("reload configuration!")
- c.configChan <- config
+func (so *StdOut) ReloadConfig(config *pkgconfig.Config) {
+ so.LogInfo("reload configuration!")
+ so.configChan <- config
}
-func (c *StdOut) LogInfo(msg string, v ...interface{}) {
- c.logger.Info("["+c.name+"] logger=stdout - "+msg, v...)
+func (so *StdOut) LogInfo(msg string, v ...interface{}) {
+ so.logger.Info("["+so.name+"] logger=stdout - "+msg, v...)
}
-func (c *StdOut) LogError(msg string, v ...interface{}) {
- c.logger.Error("["+c.name+"] logger=stdout - "+msg, v...)
+func (so *StdOut) LogError(msg string, v ...interface{}) {
+ so.logger.Error("["+so.name+"] logger=stdout - "+msg, v...)
}
-func (c *StdOut) SetTextWriter(b *bytes.Buffer) {
- c.writerText = log.New(os.Stdout, "", 0)
- c.writerText.SetOutput(b)
+func (so *StdOut) SetTextWriter(b *bytes.Buffer) {
+ so.writerText = log.New(os.Stdout, "", 0)
+ so.writerText.SetOutput(b)
}
-func (c *StdOut) SetPcapWriter(w io.Writer) {
- c.LogInfo("init pcap writer")
+func (so *StdOut) SetPcapWriter(w io.Writer) {
+ so.LogInfo("init pcap writer")
- c.writerPcap = pcapgo.NewWriter(w)
- if err := c.writerPcap.WriteFileHeader(65536, layers.LinkTypeEthernet); err != nil {
- c.logger.Fatal("["+c.name+"] logger=stdout - pcap init error: %e", err)
+ so.writerPcap = pcapgo.NewWriter(w)
+ if err := so.writerPcap.WriteFileHeader(65536, layers.LinkTypeEthernet); err != nil {
+ so.logger.Fatal("["+so.name+"] logger=stdout - pcap init error: %e", err)
}
}
-func (c *StdOut) Channel() chan dnsutils.DNSMessage {
- return c.inputChan
+func (so *StdOut) GetInputChannel() chan dnsutils.DNSMessage {
+ return so.inputChan
}
-func (c *StdOut) Stop() {
- c.LogInfo("stopping to run...")
- c.stopRun <- true
- <-c.doneRun
+func (so *StdOut) Stop() {
+ so.LogInfo("stopping routing handler...")
+ so.RoutingHandler.Stop()
- c.LogInfo("stopping to process...")
- c.stopProcess <- true
- <-c.doneProcess
+ so.LogInfo("stopping to run...")
+ so.stopRun <- true
+ <-so.doneRun
+
+ so.LogInfo("stopping to process...")
+ so.stopProcess <- true
+ <-so.doneProcess
}
-func (c *StdOut) Run() {
- c.LogInfo("running in background...")
+func (so *StdOut) Run() {
+ so.LogInfo("running in background...")
+
+ // prepare next channels
+ defaultRoutes, defaultNames := so.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := so.RoutingHandler.GetDroppedRoutes()
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
- listChannel = append(listChannel, c.outputChan)
- subprocessors := transformers.NewTransforms(&c.config.OutgoingTransformers, c.logger, c.name, listChannel, 0)
+ listChannel = append(listChannel, so.outputChan)
+ subprocessors := transformers.NewTransforms(&so.config.OutgoingTransformers, so.logger, so.name, listChannel, 0)
// goroutine to process transformed dns messages
- go c.Process()
+ go so.Process()
// loop to process incoming messages
RUN_LOOP:
for {
select {
- case <-c.stopRun:
+ case <-so.stopRun:
// cleanup transformers
subprocessors.Reset()
- c.doneRun <- true
+ so.doneRun <- true
break RUN_LOOP
// new config provided?
- case cfg, opened := <-c.configChan:
+ case cfg, opened := <-so.configChan:
if !opened {
return
}
- c.config = cfg
- c.ReadConfig()
+ so.config = cfg
+ so.ReadConfig()
subprocessors.ReloadConfig(&cfg.OutgoingTransformers)
- case dm, opened := <-c.inputChan:
+ case dm, opened := <-so.inputChan:
if !opened {
- c.LogInfo("run: input channel closed!")
+ so.LogInfo("run: input channel closed!")
return
}
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ so.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ so.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
- c.outputChan <- dm
+ so.outputChan <- dm
}
}
- c.LogInfo("run terminated")
+ so.LogInfo("run terminated")
}
-func (c *StdOut) Process() {
+func (so *StdOut) Process() {
// standard output buffer
buffer := new(bytes.Buffer)
- if c.config.Loggers.Stdout.Mode == pkgconfig.ModePCAP && c.writerPcap == nil {
- c.SetPcapWriter(os.Stdout)
+ if so.config.Loggers.Stdout.Mode == pkgconfig.ModePCAP && so.writerPcap == nil {
+ so.SetPcapWriter(os.Stdout)
}
- c.LogInfo("ready to process")
+ so.LogInfo("ready to process")
PROCESS_LOOP:
for {
select {
- case <-c.stopProcess:
- c.doneProcess <- true
+ case <-so.stopProcess:
+ so.doneProcess <- true
break PROCESS_LOOP
- case dm, opened := <-c.outputChan:
+ case dm, opened := <-so.outputChan:
if !opened {
- c.LogInfo("process: output channel closed!")
+ so.LogInfo("process: output channel closed!")
return
}
- switch c.config.Loggers.Stdout.Mode {
+ switch so.config.Loggers.Stdout.Mode {
case pkgconfig.ModePCAP:
if len(dm.DNS.Payload) == 0 {
- c.LogError("process: no dns payload to encode, drop it")
+ so.LogError("process: no dns payload to encode, drop it")
continue
}
pkt, err := dm.ToPacketLayer()
if err != nil {
- c.LogError("unable to pack layer: %s", err)
+ so.LogError("unable to pack layer: %s", err)
continue
}
@@ -223,28 +245,28 @@ PROCESS_LOOP:
Length: bufSize,
}
- c.writerPcap.WritePacket(ci, buf.Bytes())
+ so.writerPcap.WritePacket(ci, buf.Bytes())
case pkgconfig.ModeText:
- c.writerText.Print(dm.String(c.textFormat,
- c.config.Global.TextFormatDelimiter,
- c.config.Global.TextFormatBoundary))
+ so.writerText.Print(dm.String(so.textFormat,
+ so.config.Global.TextFormatDelimiter,
+ so.config.Global.TextFormatBoundary))
case pkgconfig.ModeJSON:
json.NewEncoder(buffer).Encode(dm)
- c.writerText.Print(buffer.String())
+ so.writerText.Print(buffer.String())
buffer.Reset()
case pkgconfig.ModeFlatJSON:
flat, err := dm.Flatten()
if err != nil {
- c.LogError("process: flattening DNS message failed: %e", err)
+ so.LogError("process: flattening DNS message failed: %e", err)
}
json.NewEncoder(buffer).Encode(flat)
- c.writerText.Print(buffer.String())
+ so.writerText.Print(buffer.String())
buffer.Reset()
}
}
}
- c.LogInfo("processing terminated")
+ so.LogInfo("processing terminated")
}
diff --git a/loggers/stdout_test.go b/loggers/stdout_test.go
index fd701d22..c8c717a2 100644
--- a/loggers/stdout_test.go
+++ b/loggers/stdout_test.go
@@ -77,7 +77,7 @@ func Test_StdoutTextMode(t *testing.T) {
// print dns message to stdout buffer
dm := dnsutils.GetFakeDNSMessage()
dm.DNS.Qname = tc.qname
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// stop logger
time.Sleep(time.Second)
@@ -120,7 +120,7 @@ func Test_StdoutJsonMode(t *testing.T) {
// print dns message to stdout buffer
dm := dnsutils.GetFakeDNSMessage()
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// stop logger
time.Sleep(time.Second)
@@ -151,7 +151,7 @@ func Test_StdoutPcapMode(t *testing.T) {
// send DNSMessage to channel
dm := dnsutils.GetFakeDNSMessageWithPayload()
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// stop logger
time.Sleep(time.Second)
@@ -192,7 +192,7 @@ func Test_StdoutPcapMode_NoDNSPayload(t *testing.T) {
// send DNSMessage to channel
dm := dnsutils.GetFakeDNSMessage()
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// stop logger
time.Sleep(time.Second)
diff --git a/loggers/syslog.go b/loggers/syslog.go
index fa817058..a913fdfa 100644
--- a/loggers/syslog.go
+++ b/loggers/syslog.go
@@ -14,6 +14,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
)
@@ -72,6 +73,7 @@ type Syslog struct {
transportReconnect chan bool
textFormat []string
name string
+ RoutingHandler pkgutils.RoutingHandler
}
func NewSyslog(config *pkgconfig.Config, console *logger.Logger, name string) *Syslog {
@@ -89,6 +91,7 @@ func NewSyslog(config *pkgconfig.Config, console *logger.Logger, name string) *S
config: config,
configChan: make(chan *pkgconfig.Config),
name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, console, name),
}
s.ReadConfig()
return s
@@ -96,7 +99,15 @@ func NewSyslog(config *pkgconfig.Config, console *logger.Logger, name string) *S
func (s *Syslog) GetName() string { return s.name }
-func (s *Syslog) SetLoggers(loggers []dnsutils.Worker) {}
+func (s *Syslog) AddDroppedRoute(wrk pkgutils.Worker) {
+ s.RoutingHandler.AddDroppedRoute(wrk)
+}
+
+func (s *Syslog) AddDefaultRoute(wrk pkgutils.Worker) {
+ s.RoutingHandler.AddDefaultRoute(wrk)
+}
+
+func (s *Syslog) SetLoggers(loggers []pkgutils.Worker) {}
func (s *Syslog) ReadConfig() {
if !pkgconfig.IsValidTLS(s.config.Loggers.Syslog.TLSMinVersion) {
@@ -130,7 +141,7 @@ func (s *Syslog) ReloadConfig(config *pkgconfig.Config) {
s.configChan <- config
}
-func (s *Syslog) Channel() chan dnsutils.DNSMessage {
+func (s *Syslog) GetInputChannel() chan dnsutils.DNSMessage {
return s.inputChan
}
@@ -143,6 +154,9 @@ func (s *Syslog) LogError(msg string, v ...interface{}) {
}
func (s *Syslog) Stop() {
+ s.LogInfo("stopping routing handler...")
+ s.RoutingHandler.Stop()
+
s.LogInfo("stopping to run...")
s.stopRun <- true
<-s.doneRun
@@ -245,6 +259,10 @@ func (s *Syslog) ConnectToRemote() {
func (s *Syslog) Run() {
s.LogInfo("running in background...")
+ // prepare next channels
+ defaultRoutes, defaultNames := s.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := s.RoutingHandler.GetDroppedRoutes()
+
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
listChannel = append(listChannel, s.outputChan)
@@ -285,9 +303,13 @@ RUN_LOOP:
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ s.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ s.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
s.outputChan <- dm
}
diff --git a/loggers/syslog_test.go b/loggers/syslog_test.go
index be3363b1..9e4a2901 100644
--- a/loggers/syslog_test.go
+++ b/loggers/syslog_test.go
@@ -88,7 +88,7 @@ func Test_SyslogRunUdp(t *testing.T) {
// send fake dns message to logger
time.Sleep(time.Second)
dm := dnsutils.GetFakeDNSMessage()
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// read data on fake server side
buf := make([]byte, 4096)
@@ -191,7 +191,7 @@ func Test_SyslogRunTcp(t *testing.T) {
// send fake dns message to logger
time.Sleep(time.Second)
dm := dnsutils.GetFakeDNSMessage()
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// read data on server side and decode-it
reader := bufio.NewReader(conn)
@@ -235,7 +235,7 @@ func Test_SyslogRun_RemoveNullCharacter(t *testing.T) {
time.Sleep(time.Second)
dm := dnsutils.GetFakeDNSMessage()
dm.DNS.Qname = "null\x00char.com"
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// read data on fake server side
buf := make([]byte, (500))
diff --git a/loggers/tcpclient.go b/loggers/tcpclient.go
index 12a47c70..e426830b 100644
--- a/loggers/tcpclient.go
+++ b/loggers/tcpclient.go
@@ -14,6 +14,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
)
@@ -38,6 +39,7 @@ type TCPClient struct {
transportReady chan bool
transportReconnect chan bool
writerReady bool
+ RoutingHandler pkgutils.RoutingHandler
}
func NewTCPClient(config *pkgconfig.Config, logger *logger.Logger, name string) *TCPClient {
@@ -57,6 +59,7 @@ func NewTCPClient(config *pkgconfig.Config, logger *logger.Logger, name string)
config: config,
configChan: make(chan *pkgconfig.Config),
name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
}
s.ReadConfig()
@@ -66,7 +69,15 @@ func NewTCPClient(config *pkgconfig.Config, logger *logger.Logger, name string)
func (c *TCPClient) GetName() string { return c.name }
-func (c *TCPClient) SetLoggers(loggers []dnsutils.Worker) {}
+func (c *TCPClient) AddDroppedRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDroppedRoute(wrk)
+}
+
+func (c *TCPClient) AddDefaultRoute(wrk pkgutils.Worker) {
+ c.RoutingHandler.AddDefaultRoute(wrk)
+}
+
+func (c *TCPClient) SetLoggers(loggers []pkgutils.Worker) {}
func (c *TCPClient) ReadConfig() {
c.transport = c.config.Loggers.TCPClient.Transport
@@ -100,11 +111,14 @@ func (c *TCPClient) LogError(msg string, v ...interface{}) {
c.logger.Error("["+c.name+"] logger=tcpclient - "+msg, v...)
}
-func (c *TCPClient) Channel() chan dnsutils.DNSMessage {
+func (c *TCPClient) GetInputChannel() chan dnsutils.DNSMessage {
return c.inputChan
}
func (c *TCPClient) Stop() {
+ c.LogInfo("stopping routing handler...")
+ c.RoutingHandler.Stop()
+
c.LogInfo("stopping to run...")
c.stopRun <- true
<-c.doneRun
@@ -257,6 +271,10 @@ func (c *TCPClient) FlushBuffer(buf *[]dnsutils.DNSMessage) {
func (c *TCPClient) Run() {
c.LogInfo("running in background...")
+ // prepare next channels
+ defaultRoutes, defaultNames := c.RoutingHandler.GetDefaultRoutes()
+ droppedRoutes, droppedNames := c.RoutingHandler.GetDroppedRoutes()
+
// prepare transforms
listChannel := []chan dnsutils.DNSMessage{}
listChannel = append(listChannel, c.outputChan)
@@ -293,9 +311,13 @@ RUN_LOOP:
// apply tranforms, init dns message with additionnals parts if necessary
subprocessors.InitDNSMessageFormat(&dm)
if subprocessors.ProcessMessage(&dm) == transformers.ReturnDrop {
+ c.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
+ // send to next ?
+ c.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
// send to output channel
c.outputChan <- dm
}
diff --git a/loggers/tcpclient_test.go b/loggers/tcpclient_test.go
index 94e13164..8e2c0371 100644
--- a/loggers/tcpclient_test.go
+++ b/loggers/tcpclient_test.go
@@ -65,7 +65,7 @@ func Test_TcpClientRun(t *testing.T) {
// send fake dns message to logger
dm := dnsutils.GetFakeDNSMessage()
- g.Channel() <- dm
+ g.GetInputChannel() <- dm
// read data on server side and decode-it
reader := bufio.NewReader(conn)
diff --git a/pkgconfig/collectors.go b/pkgconfig/collectors.go
index 094ac805..dcf0a478 100644
--- a/pkgconfig/collectors.go
+++ b/pkgconfig/collectors.go
@@ -1,6 +1,16 @@
package pkgconfig
+import "reflect"
+
type ConfigCollectors struct {
+ DNSMessage struct {
+ Enable bool `yaml:"enable"`
+ ChannelBufferSize int `yaml:"chan-buffer-size"`
+ Matching struct {
+ Include map[string]interface{} `yaml:"include"`
+ Exclude map[string]interface{} `yaml:"exclude"`
+ } `yaml:"matching"`
+ } `yaml:"dnsmessage"`
Tail struct {
Enable bool `yaml:"enable"`
TimeLayout string `yaml:"time-layout"`
@@ -74,6 +84,9 @@ type ConfigCollectors struct {
}
func (c *ConfigCollectors) SetDefault() {
+ c.DNSMessage.Enable = false
+ c.DNSMessage.ChannelBufferSize = 65535
+
c.Tail.Enable = false
c.Tail.TimeLayout = ""
c.Tail.PatternQuery = ""
@@ -135,3 +148,24 @@ func (c *ConfigCollectors) SetDefault() {
c.Tzsp.ListenPort = 10000
c.Tzsp.ChannelBufferSize = 65535
}
+
+func (c *ConfigCollectors) GetTags() (ret []string) {
+ cl := reflect.TypeOf(*c)
+
+ for i := 0; i < cl.NumField(); i++ {
+ field := cl.Field(i)
+ tag := field.Tag.Get("yaml")
+ ret = append(ret, tag)
+ }
+ return ret
+}
+
+func (c *ConfigCollectors) IsValid(name string) bool {
+ tags := c.GetTags()
+ for i := range tags {
+ if name == tags[i] {
+ return true
+ }
+ }
+ return false
+}
diff --git a/pkgconfig/collectors_test.go b/pkgconfig/collectors_test.go
index 4a21cd61..04538339 100644
--- a/pkgconfig/collectors_test.go
+++ b/pkgconfig/collectors_test.go
@@ -6,6 +6,9 @@ func TestConfigCollectorsSetDefault(t *testing.T) {
config := ConfigCollectors{}
config.SetDefault()
+ if config.DNSMessage.Enable != false {
+ t.Errorf("dnsmessage should be disabled")
+ }
if config.Dnstap.Enable != false {
t.Errorf("dnstap should be disabled")
}
diff --git a/pkgconfig/config.go b/pkgconfig/config.go
index b77a2dd3..1ff9f5a0 100644
--- a/pkgconfig/config.go
+++ b/pkgconfig/config.go
@@ -5,6 +5,7 @@ import (
"io"
"os"
"reflect"
+ "regexp"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
@@ -28,6 +29,7 @@ type Config struct {
Loggers ConfigLoggers `yaml:"loggers"`
OutgoingTransformers ConfigTransformers `yaml:"loggers-transformers"`
Multiplexer ConfigMultiplexer `yaml:"multiplexer"`
+ Pipelines []ConfigPipelines `yaml:"pipelines"`
}
func (c *Config) SetDefault() {
@@ -63,7 +65,7 @@ func (c *Config) GetServerIdentity() string {
}
}
-func ReloadConfig(configPath string, config *Config) error {
+func ReloadConfig(configPath string, config *Config, refDNSMessage map[string]interface{}) error {
// Open config file
configFile, err := os.Open(configPath)
if err != nil {
@@ -72,7 +74,7 @@ func ReloadConfig(configPath string, config *Config) error {
defer configFile.Close()
// Check config to detect unknown keywords
- if err := CheckConfig(configPath); err != nil {
+ if err := CheckConfig(configPath, refDNSMessage); err != nil {
return err
}
@@ -86,7 +88,7 @@ func ReloadConfig(configPath string, config *Config) error {
return nil
}
-func LoadConfig(configPath string) (*Config, error) {
+func LoadConfig(configPath string, refDNSMessage map[string]interface{}) (*Config, error) {
// Open config file
configFile, err := os.Open(configPath)
if err != nil {
@@ -95,7 +97,7 @@ func LoadConfig(configPath string) (*Config, error) {
defer configFile.Close()
// Check config to detect unknown keywords
- if err := CheckConfig(configPath); err != nil {
+ if err := CheckConfig(configPath, refDNSMessage); err != nil {
return nil, err
}
@@ -113,14 +115,15 @@ func LoadConfig(configPath string) (*Config, error) {
return config, nil
}
-func CheckConfig(userConfigPath string) error {
+func CheckConfig(userConfigPath string, dmRef map[string]interface{}) error {
// create default config
- // and simulate one route, one collector and one logger
+ // and simulate items in multiplexer and pipelines mode
defaultConfig := &Config{}
defaultConfig.SetDefault()
defaultConfig.Multiplexer.Routes = append(defaultConfig.Multiplexer.Routes, MultiplexRoutes{})
defaultConfig.Multiplexer.Loggers = append(defaultConfig.Multiplexer.Loggers, MultiplexInOut{})
defaultConfig.Multiplexer.Collectors = append(defaultConfig.Multiplexer.Collectors, MultiplexInOut{})
+ defaultConfig.Pipelines = append(defaultConfig.Pipelines, ConfigPipelines{})
// Convert default config to map
// And get unique YAML keys
@@ -130,6 +133,11 @@ func CheckConfig(userConfigPath string) error {
}
defaultKeywords := getUniqueKeywords(defaultConfigMap)
+ // add DNSMessage default keys
+ for k := range dmRef {
+ defaultKeywords[k] = true
+ }
+
// Read user configuration file
// And get unique YAML keys from user config
userConfigMap, err := loadUserConfigToMap(userConfigPath)
@@ -139,7 +147,26 @@ func CheckConfig(userConfigPath string) error {
userKeywords := getUniqueKeywords(userConfigMap)
// Check for unknown keys in user config
+ // ignore dynamic keys as atags.tags.*: google
+
+ // Define regular expressions to match dynamic keys
+ regexPatterns := []string{`\.\*(\.)?`, `\.(\d+)(\.)?`}
+
for key := range userKeywords {
+ // Ignore dynamic keys that contain ".*" or .[digits].
+ matched := false
+ for _, pattern := range regexPatterns {
+ match, _ := regexp.MatchString(pattern, key)
+ if match {
+ matched = true
+ break
+ }
+ }
+ if matched {
+ continue
+ }
+
+ // search in default keywords
if _, ok := defaultKeywords[key]; !ok {
return errors.Errorf("unknown YAML key `%s` in configuration", key)
}
@@ -156,11 +183,13 @@ func CheckConfig(userConfigPath string) error {
func checkKeywordsPosition(nextUserCfg, nextDefCfg map[string]interface{}, defaultConf map[string]interface{}, sectionName string) error {
for k, v := range nextUserCfg {
// Check if the key is present in the default config
- if _, ok := nextDefCfg[k]; !ok {
- if sectionName == "" {
- return errors.Errorf("invalid key `%s` at root", k)
+ if len(nextDefCfg) > 0 {
+ if _, ok := nextDefCfg[k]; !ok {
+ if sectionName == "" {
+ return errors.Errorf("invalid key `%s` at root", k)
+ }
+ return errors.Errorf("invalid key `%s` in section `%s`", k, sectionName)
}
- return errors.Errorf("invalid key `%s` in section `%s`", k, sectionName)
}
// If the value is a map, recursively check for invalid keywords
@@ -181,6 +210,71 @@ func checkKeywordsPosition(nextUserCfg, nextDefCfg map[string]interface{}, defau
}
}
+ // If the value is a slice and we are in the pipelines part
+ if val.Kind() == reflect.Slice && k == "pipelines" {
+ if err := checkPipelinesConfig(val, nextDefCfg[k].([]interface{}), defaultConf, k); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func checkPipelinesConfig(currentVal reflect.Value, currentRef []interface{}, defaultConf map[string]interface{}, k string) error {
+ refLoggers := defaultConf[KeyLoggers].(map[string]interface{})
+ refCollectors := defaultConf[KeyCollectors].(map[string]interface{})
+ refTransforms := defaultConf["collectors-transformers"].(map[string]interface{})
+
+ for pos, item := range currentVal.Interface().([]interface{}) {
+ valReflect := reflect.ValueOf(item)
+ refItem := currentRef[0].(map[string]interface{})
+ if valReflect.Kind() == reflect.Map {
+ for _, key := range valReflect.MapKeys() {
+ strKey := key.Interface().(string)
+ mapVal := valReflect.MapIndex(key)
+
+ if _, ok := refItem[strKey]; !ok {
+ // Check if the key exists in neither loggers nor collectors
+ loggerExists := refLoggers[strKey] != nil
+ collectorExists := refCollectors[strKey] != nil
+ if !loggerExists && !collectorExists {
+ return errors.Errorf("invalid `%s` in `%s` pipelines at position %d", strKey, k, pos)
+ }
+
+ // check logger or collectors
+ if loggerExists || collectorExists {
+ nextSectionName := fmt.Sprintf("%s[%d].%s", k, pos, strKey)
+ refMap := refLoggers
+ if collectorExists {
+ refMap = refCollectors
+ }
+ // Type assertion to check if the value is a map
+ if value, ok := mapVal.Interface().(map[string]interface{}); ok {
+ if err := checkKeywordsPosition(value, refMap[strKey].(map[string]interface{}), defaultConf, nextSectionName); err != nil {
+ return err
+ }
+ } else {
+ return errors.Errorf("invalid `%s` value in `%s` pipelines at position %d", strKey, k, pos)
+ }
+ }
+ }
+
+ // Check transforms section
+ // Type assertion to check if the value is a map
+ if strKey == "transforms" {
+ nextSectionName := fmt.Sprintf("%s.%s", k, strKey)
+ if value, ok := mapVal.Interface().(map[string]interface{}); ok {
+ if err := checkKeywordsPosition(value, refTransforms, defaultConf, nextSectionName); err != nil {
+ return err
+ }
+ } else {
+ return errors.Errorf("invalid `%s` value in `%s` pipelines at position %d", strKey, k, pos)
+ }
+ }
+ }
+ } else {
+ return errors.Errorf("invalid item type in pipelines list: %s", valReflect.Kind())
+ }
}
return nil
}
@@ -214,7 +308,7 @@ func checkMultiplexerConfig(currentVal reflect.Value, currentRef []interface{},
return errors.Errorf("invalid `%s` in `%s` list at position %d", strKey, k, pos)
}
- // check logger
+ // check logger or collectors
if k == KeyLoggers || k == KeyCollectors {
nextSectionName := fmt.Sprintf("%s[%d].%s", k, pos, strKey)
refMap := refLoggers
@@ -246,6 +340,8 @@ func checkMultiplexerConfig(currentVal reflect.Value, currentRef []interface{},
}
}
}
+ } else {
+ return errors.Errorf("invalid item type in multiplexer list: %s", valReflect.Kind())
}
}
return nil
diff --git a/pkgconfig/config_test.go b/pkgconfig/config_test.go
index 0cb0c8c6..87055b1e 100644
--- a/pkgconfig/config_test.go
+++ b/pkgconfig/config_test.go
@@ -61,7 +61,8 @@ multiplexer:
t.Fatal("Error writing to user configuration file:", err)
}
- if err := CheckConfig(userConfigFile.Name()); err != nil {
+ dm := make(map[string]interface{})
+ if err := CheckConfig(userConfigFile.Name(), dm); err != nil {
t.Errorf("failed: Unexpected error: %v", err)
}
}
@@ -88,12 +89,47 @@ multiplexer:
t.Fatal("Error writing to user configuration file:", err)
}
+ dm := make(map[string]interface{})
expectedError := errors.Errorf("unknown YAML key `unknown-key` in configuration")
- if err := CheckConfig(userConfigFile.Name()); err == nil || err.Error() != expectedError.Error() {
+ if err := CheckConfig(userConfigFile.Name(), dm); err == nil || err.Error() != expectedError.Error() {
t.Errorf("Expected error %v, but got %v", expectedError, err)
}
}
+// Ignore dynamic keys
+func TestConfig_CheckConfig_IgnoreDynamicKeys(t *testing.T) {
+ userConfigFile, err := os.CreateTemp("", "user-config.yaml")
+ if err != nil {
+ t.Fatal("Error creating temporary file:", err)
+ }
+ defer os.Remove(userConfigFile.Name())
+ defer userConfigFile.Close()
+
+ userConfigContent := `
+global:
+ trace: false
+pipelines:
+ - name: match
+ dnsmessage:
+ matching:
+ include:
+ atags.tags.*: test
+ atags.tags.2: test
+ dns.resources-records.*: test
+ dns.resources-records.10.rdata: test
+ dns.resources-records.*.ttl: test
+`
+ err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644)
+ if err != nil {
+ t.Fatal("Error writing to user configuration file:", err)
+ }
+
+ dm := make(map[string]interface{})
+ if err := CheckConfig(userConfigFile.Name(), dm); err != nil {
+ t.Errorf("Expected no error, but got %v", err)
+ }
+}
+
// Keywork exist but not at the good position
func TestConfig_CheckConfig_BadKeywordPosition(t *testing.T) {
userConfigFile, err := os.CreateTemp("", "user-config.yaml")
@@ -112,7 +148,9 @@ global:
if err != nil {
t.Fatal("Error writing to user configuration file:", err)
}
- if err := CheckConfig(userConfigFile.Name()); err == nil {
+
+ dm := make(map[string]interface{})
+ if err := CheckConfig(userConfigFile.Name(), dm); err == nil {
t.Errorf("Expected error, but got %v", err)
}
}
@@ -148,7 +186,9 @@ multiplexer:
if err != nil {
t.Fatal("Error writing to user configuration file:", err)
}
- if err := CheckConfig(userConfigFile.Name()); err != nil {
+
+ dm := make(map[string]interface{})
+ if err := CheckConfig(userConfigFile.Name(), dm); err != nil {
t.Errorf("failed: Unexpected error: %v", err)
}
}
@@ -178,7 +218,74 @@ multiplexer:
if err != nil {
t.Fatal("Error writing to user configuration file:", err)
}
- if err := CheckConfig(userConfigFile.Name()); err == nil {
+
+ dm := make(map[string]interface{})
+ if err := CheckConfig(userConfigFile.Name(), dm); err == nil {
+ t.Errorf("Expected error, but got %v", err)
+ }
+}
+
+// Valid pipeline configuration
+func TestConfig_CheckPipelinesConfig_Valid(t *testing.T) {
+ userConfigFile, err := os.CreateTemp("", "user-config.yaml")
+ if err != nil {
+ t.Fatal("Error creating temporary file:", err)
+ }
+ defer os.Remove(userConfigFile.Name())
+ defer userConfigFile.Close()
+
+ userConfigContent := `
+pipelines:
+- name: dnsdist-main
+ dnstap:
+ listen-ip: 0.0.0.0
+ listen-port: 6000
+ routing-policy:
+ default: [ console ]
+
+- name: console
+ stdout:
+ mode: text
+`
+ err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644)
+ if err != nil {
+ t.Fatal("Error writing to user configuration file:", err)
+ }
+
+ dm := make(map[string]interface{})
+ if err := CheckConfig(userConfigFile.Name(), dm); err != nil {
+ t.Errorf("failed: Unexpected error: %v", err)
+ }
+}
+
+// Invalid pipeline configuration
+func TestConfig_CheckPipelinesConfig_Invalid(t *testing.T) {
+ userConfigFile, err := os.CreateTemp("", "user-config.yaml")
+ if err != nil {
+ t.Fatal("Error creating temporary file:", err)
+ }
+ defer os.Remove(userConfigFile.Name())
+ defer userConfigFile.Close()
+
+ userConfigContent := `
+pipelines:
+- name: dnsdist-main
+ dnstap:
+ listen-ip: 0.0.0.0
+ transforms:
+ normalize:
+ qname-lowercase: true
+ routing-policy:
+ default: [ console ]
+`
+
+ err = os.WriteFile(userConfigFile.Name(), []byte(userConfigContent), 0644)
+ if err != nil {
+ t.Fatal("Error writing to user configuration file:", err)
+ }
+
+ dm := make(map[string]interface{})
+ if err := CheckConfig(userConfigFile.Name(), dm); err == nil {
t.Errorf("Expected error, but got %v", err)
}
}
diff --git a/pkgconfig/loggers.go b/pkgconfig/loggers.go
index e96fd3ac..cc9a33c0 100644
--- a/pkgconfig/loggers.go
+++ b/pkgconfig/loggers.go
@@ -1,6 +1,8 @@
package pkgconfig
import (
+ "reflect"
+
"github.com/dmachard/go-dnscollector/netlib"
"github.com/prometheus/prometheus/model/relabel"
)
@@ -539,5 +541,25 @@ func (c *ConfigLoggers) SetDefault() {
c.FalcoClient.Enable = false
c.FalcoClient.URL = "http://127.0.0.1:9200"
c.FalcoClient.ChannelBufferSize = 65535
+}
+
+func (c *ConfigLoggers) GetTags() (ret []string) {
+ cl := reflect.TypeOf(*c)
+
+ for i := 0; i < cl.NumField(); i++ {
+ field := cl.Field(i)
+ tag := field.Tag.Get("yaml")
+ ret = append(ret, tag)
+ }
+ return ret
+}
+func (c *ConfigLoggers) IsValid(name string) bool {
+ tags := c.GetTags()
+ for i := range tags {
+ if name == tags[i] {
+ return true
+ }
+ }
+ return false
}
diff --git a/pkgconfig/pipelines.go b/pkgconfig/pipelines.go
new file mode 100644
index 00000000..126f743a
--- /dev/null
+++ b/pkgconfig/pipelines.go
@@ -0,0 +1,13 @@
+package pkgconfig
+
+type ConfigPipelines struct {
+ Name string `yaml:"name"`
+ Transforms map[string]interface{} `yaml:"transforms"`
+ Params map[string]interface{} `yaml:",inline"`
+ RoutingPolicy PipelinesRouting `yaml:"routing-policy"`
+}
+
+type PipelinesRouting struct {
+ Default []string `yaml:"default,flow"`
+ Dropped []string `yaml:"dropped,flow"`
+}
diff --git a/pkgconfig/transformers.go b/pkgconfig/transformers.go
index d52b9aa0..33461919 100644
--- a/pkgconfig/transformers.go
+++ b/pkgconfig/transformers.go
@@ -22,13 +22,13 @@ type ConfigTransformers struct {
MeasureLatency bool `yaml:"measure-latency"`
UnansweredQueries bool `yaml:"unanswered-queries"`
QueriesTimeout int `yaml:"queries-timeout"`
- }
+ } `yaml:"latency"`
Reducer struct {
Enable bool `yaml:"enable"`
RepetitiveTrafficDetector bool `yaml:"repetitive-traffic-detector"`
QnamePlusOne bool `yaml:"qname-plus-one"`
WatchInterval int `yaml:"watch-interval"`
- }
+ } `yaml:"reducer"`
Filtering struct {
Enable bool `yaml:"enable"`
DropFqdnFile string `yaml:"drop-fqdn-file"`
@@ -67,6 +67,10 @@ type ConfigTransformers struct {
Enable bool `yaml:"enable"`
AddFeatures bool `yaml:"add-features"`
} `yaml:"machine-learning"`
+ ATags struct {
+ Enable bool `yaml:"enable"`
+ Tags []string `yaml:"tags,flow"`
+ } `yaml:"atags"`
}
func (c *ConfigTransformers) SetDefault() {
@@ -125,6 +129,9 @@ func (c *ConfigTransformers) SetDefault() {
c.MachineLearning.Enable = false
c.MachineLearning.AddFeatures = false
+
+ c.ATags.Enable = false
+ c.ATags.Tags = []string{}
}
func GetFakeConfigTransformers() *ConfigTransformers {
diff --git a/multiplexer.go b/pkglinker/multiplexer.go
similarity index 95%
rename from multiplexer.go
rename to pkglinker/multiplexer.go
index 7fc7e0b9..a1202c11 100644
--- a/multiplexer.go
+++ b/pkglinker/multiplexer.go
@@ -1,13 +1,13 @@
-package main
+package pkglinker
import (
"fmt"
"strings"
"github.com/dmachard/go-dnscollector/collectors"
- "github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/loggers"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-logger"
"gopkg.in/yaml.v2"
)
@@ -79,7 +79,7 @@ func GetItemConfig(section string, config *pkgconfig.Config, item pkgconfig.Mult
return subcfg
}
-func InitMultiplexer(mapLoggers map[string]dnsutils.Worker, mapCollectors map[string]dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger) {
+func InitMultiplexer(mapLoggers map[string]pkgutils.Worker, mapCollectors map[string]pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger) {
// checking all routes before to continue
if err := AreRoutesValid(config); err != nil {
@@ -178,7 +178,7 @@ func InitMultiplexer(mapLoggers map[string]dnsutils.Worker, mapCollectors map[st
// here the multiplexer logic
// connect collectors between loggers
for _, route := range config.Multiplexer.Routes {
- var logwrks []dnsutils.Worker
+ var logwrks []pkgutils.Worker
for _, dst := range route.Dst {
if _, ok := mapLoggers[dst]; ok {
logwrks = append(logwrks, mapLoggers[dst])
@@ -199,7 +199,7 @@ func InitMultiplexer(mapLoggers map[string]dnsutils.Worker, mapCollectors map[st
}
}
-func ReloadMultiplexer(mapLoggers map[string]dnsutils.Worker, mapCollectors map[string]dnsutils.Worker, config *pkgconfig.Config, logger *logger.Logger) {
+func ReloadMultiplexer(mapLoggers map[string]pkgutils.Worker, mapCollectors map[string]pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger) {
for _, output := range config.Multiplexer.Loggers {
newcfg := GetItemConfig("loggers", config, output)
if _, ok := mapLoggers[output.Name]; ok {
diff --git a/multiplexer_test.go b/pkglinker/multiplexer_test.go
similarity index 98%
rename from multiplexer_test.go
rename to pkglinker/multiplexer_test.go
index 5297c8e4..bf62490d 100644
--- a/multiplexer_test.go
+++ b/pkglinker/multiplexer_test.go
@@ -1,4 +1,4 @@
-package main
+package pkglinker
import (
"testing"
diff --git a/pkglinker/pipelines.go b/pkglinker/pipelines.go
new file mode 100644
index 00000000..c3e6a421
--- /dev/null
+++ b/pkglinker/pipelines.go
@@ -0,0 +1,243 @@
+package pkglinker
+
+import (
+ "fmt"
+
+ "github.com/dmachard/go-dnscollector/collectors"
+ "github.com/dmachard/go-dnscollector/loggers"
+ "github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
+ "github.com/dmachard/go-logger"
+ "github.com/pkg/errors"
+ "gopkg.in/yaml.v2"
+)
+
+func GetStanzaConfig(config *pkgconfig.Config, item pkgconfig.ConfigPipelines) *pkgconfig.Config {
+
+ cfg := make(map[string]interface{})
+ section := "collectors"
+
+ // Enable the provided collector or loggers
+ for k, p := range item.Params {
+ // is a logger or collector ?
+ if !config.Loggers.IsValid(k) && !config.Collectors.IsValid(k) {
+ panic(fmt.Sprintln("main - get stanza config error"))
+ }
+ if config.Loggers.IsValid(k) {
+ section = "loggers"
+ }
+ if p == nil {
+ item.Params[k] = make(map[string]interface{})
+ }
+ item.Params[k].(map[string]interface{})["enable"] = true
+
+ // ignore other keys
+ break
+ }
+
+ // prepare a new config
+ subcfg := &pkgconfig.Config{}
+ subcfg.SetDefault()
+
+ cfg[section] = item.Params
+ cfg[section+"-transformers"] = make(map[string]interface{})
+
+ // add transformers
+ for k, v := range item.Transforms {
+ v.(map[string]interface{})["enable"] = true
+ cfg[section+"-transformers"].(map[string]interface{})[k] = v
+ }
+
+ // copy global config
+ subcfg.Global = config.Global
+
+ yamlcfg, _ := yaml.Marshal(cfg)
+ if err := yaml.Unmarshal(yamlcfg, subcfg); err != nil {
+ panic(fmt.Sprintf("main - yaml logger config error: %v", err))
+ }
+
+ return subcfg
+}
+
+func StanzaNameIsUniq(name string, config *pkgconfig.Config) (ret error) {
+ stanzaCounter := 0
+ for _, stanza := range config.Pipelines {
+ if name == stanza.Name {
+ stanzaCounter += 1
+ }
+ }
+
+ if stanzaCounter > 1 {
+ return fmt.Errorf("stanza=%s allready exists", name)
+ }
+ return nil
+}
+
+func IsRouteExist(target string, config *pkgconfig.Config) (ret error) {
+ for _, stanza := range config.Pipelines {
+ if target == stanza.Name {
+ return nil
+ }
+ }
+ return fmt.Errorf("route=%s doest not exist", target)
+}
+
+func CreateRouting(stanza pkgconfig.ConfigPipelines, mapCollectors map[string]pkgutils.Worker, mapLoggers map[string]pkgutils.Worker, logger *logger.Logger) {
+ var currentStanza pkgutils.Worker
+ if collector, ok := mapCollectors[stanza.Name]; ok {
+ currentStanza = collector
+ }
+ if logger, ok := mapLoggers[stanza.Name]; ok {
+ currentStanza = logger
+ }
+
+ // default routing
+ for _, route := range stanza.RoutingPolicy.Default {
+ if _, ok := mapCollectors[route]; ok {
+ currentStanza.AddDefaultRoute(mapCollectors[route])
+ logger.Info("main - routing (policy=default) stanza=[%s] to stanza=[%s]", stanza.Name, route)
+ } else if _, ok := mapLoggers[route]; ok {
+ currentStanza.AddDefaultRoute(mapLoggers[route])
+ logger.Info("main - routing (policy=default) stanza=[%s] to stanza=[%s]", stanza.Name, route)
+ } else {
+ logger.Error("main - default routing error from stanza=%s to stanza=%s doest not exist", stanza.Name, route)
+ break
+ }
+ }
+
+ // dropped routing
+ for _, route := range stanza.RoutingPolicy.Dropped {
+ if _, ok := mapCollectors[route]; ok {
+ currentStanza.AddDroppedRoute(mapCollectors[route])
+ logger.Info("main - routing (policy=dropped) stanza=[%s] to stanza=[%s]", stanza.Name, route)
+ } else if _, ok := mapLoggers[route]; ok {
+ currentStanza.AddDroppedRoute(mapLoggers[route])
+ logger.Info("main - routing (policy=dropped) stanza=[%s] to stanza=[%s]", stanza.Name, route)
+ } else {
+ logger.Error("main - routing error with dropped messages from stanza=%s to stanza=%s doest not exist", stanza.Name, route)
+ break
+ }
+ }
+}
+
+func CreateStanza(stanzaName string, config *pkgconfig.Config, mapCollectors map[string]pkgutils.Worker, mapLoggers map[string]pkgutils.Worker, logger *logger.Logger) {
+ // register the logger if enabled
+ if config.Loggers.RestAPI.Enable {
+ mapLoggers[stanzaName] = loggers.NewRestAPI(config, logger, stanzaName)
+ }
+ if config.Loggers.Prometheus.Enable {
+ mapLoggers[stanzaName] = loggers.NewPrometheus(config, logger, stanzaName)
+ }
+ if config.Loggers.Stdout.Enable {
+ mapLoggers[stanzaName] = loggers.NewStdOut(config, logger, stanzaName)
+ }
+ if config.Loggers.LogFile.Enable {
+ mapLoggers[stanzaName] = loggers.NewLogFile(config, logger, stanzaName)
+ }
+ if config.Loggers.DNSTap.Enable {
+ mapLoggers[stanzaName] = loggers.NewDnstapSender(config, logger, stanzaName)
+ }
+ if config.Loggers.TCPClient.Enable {
+ mapLoggers[stanzaName] = loggers.NewTCPClient(config, logger, stanzaName)
+ }
+ if config.Loggers.Syslog.Enable {
+ mapLoggers[stanzaName] = loggers.NewSyslog(config, logger, stanzaName)
+ }
+ if config.Loggers.Fluentd.Enable {
+ mapLoggers[stanzaName] = loggers.NewFluentdClient(config, logger, stanzaName)
+ }
+ if config.Loggers.InfluxDB.Enable {
+ mapLoggers[stanzaName] = loggers.NewInfluxDBClient(config, logger, stanzaName)
+ }
+ if config.Loggers.LokiClient.Enable {
+ mapLoggers[stanzaName] = loggers.NewLokiClient(config, logger, stanzaName)
+ }
+ if config.Loggers.Statsd.Enable {
+ mapLoggers[stanzaName] = loggers.NewStatsdClient(config, logger, stanzaName)
+ }
+ if config.Loggers.ElasticSearchClient.Enable {
+ mapLoggers[stanzaName] = loggers.NewElasticSearchClient(config, logger, stanzaName)
+ }
+ if config.Loggers.ScalyrClient.Enable {
+ mapLoggers[stanzaName] = loggers.NewScalyrClient(config, logger, stanzaName)
+ }
+ if config.Loggers.RedisPub.Enable {
+ mapLoggers[stanzaName] = loggers.NewRedisPub(config, logger, stanzaName)
+ }
+ if config.Loggers.KafkaProducer.Enable {
+ mapLoggers[stanzaName] = loggers.NewKafkaProducer(config, logger, stanzaName)
+ }
+ if config.Loggers.FalcoClient.Enable {
+ mapLoggers[stanzaName] = loggers.NewFalcoClient(config, logger, stanzaName)
+ }
+
+ // register the collector if enabled
+ if config.Collectors.DNSMessage.Enable {
+ mapCollectors[stanzaName] = collectors.NewDNSMessage(nil, config, logger, stanzaName)
+ }
+ if config.Collectors.Dnstap.Enable {
+ mapCollectors[stanzaName] = collectors.NewDnstap(nil, config, logger, stanzaName)
+ }
+ if config.Collectors.DnstapProxifier.Enable {
+ mapCollectors[stanzaName] = collectors.NewDnstapProxifier(nil, config, logger, stanzaName)
+ }
+ if config.Collectors.AfpacketLiveCapture.Enable {
+ mapCollectors[stanzaName] = collectors.NewAfpacketSniffer(nil, config, logger, stanzaName)
+ }
+ if config.Collectors.XdpLiveCapture.Enable {
+ mapCollectors[stanzaName] = collectors.NewXDPSniffer(nil, config, logger, stanzaName)
+ }
+ if config.Collectors.Tail.Enable {
+ mapCollectors[stanzaName] = collectors.NewTail(nil, config, logger, stanzaName)
+ }
+ if config.Collectors.PowerDNS.Enable {
+ mapCollectors[stanzaName] = collectors.NewProtobufPowerDNS(nil, config, logger, stanzaName)
+ }
+ if config.Collectors.FileIngestor.Enable {
+ mapCollectors[stanzaName] = collectors.NewFileIngestor(nil, config, logger, stanzaName)
+ }
+ if config.Collectors.Tzsp.Enable {
+ mapCollectors[stanzaName] = collectors.NewTZSP(nil, config, logger, stanzaName)
+ }
+}
+
+func InitPipelines(mapLoggers map[string]pkgutils.Worker, mapCollectors map[string]pkgutils.Worker, config *pkgconfig.Config, logger *logger.Logger) error {
+ // check if the name of each stanza is uniq
+ for _, stanza := range config.Pipelines {
+ if err := StanzaNameIsUniq(stanza.Name, config); err != nil {
+ return errors.Errorf("stanza with name=[%s] is duplicated", stanza.Name)
+ }
+ }
+
+ // check if all routes exists before continue
+ for _, stanza := range config.Pipelines {
+ for _, route := range stanza.RoutingPolicy.Default {
+ if err := IsRouteExist(route, config); err != nil {
+ return errors.Errorf("stanza=[%s] default route=[%s] doest not exist", stanza.Name, route)
+ }
+ }
+ for _, route := range stanza.RoutingPolicy.Dropped {
+ if err := IsRouteExist(route, config); err != nil {
+ return errors.Errorf("stanza=[%s] dropped route=[%s] doest not exist", stanza.Name, route)
+ }
+ }
+ }
+
+ // read each stanza and init
+ for _, stanza := range config.Pipelines {
+ stanzaConfig := GetStanzaConfig(config, stanza)
+ CreateStanza(stanza.Name, stanzaConfig, mapCollectors, mapLoggers, logger)
+
+ }
+
+ // create routing
+ for _, stanza := range config.Pipelines {
+ if mapCollectors[stanza.Name] != nil || mapLoggers[stanza.Name] != nil {
+ CreateRouting(stanza, mapCollectors, mapLoggers, logger)
+ } else {
+ return errors.Errorf("stanza=[%v] doest not exist", stanza.Name)
+ }
+ }
+
+ return nil
+}
diff --git a/pkglinker/pipelines_test.go b/pkglinker/pipelines_test.go
new file mode 100644
index 00000000..dd62b589
--- /dev/null
+++ b/pkglinker/pipelines_test.go
@@ -0,0 +1,53 @@
+package pkglinker
+
+import (
+ "testing"
+
+ "github.com/dmachard/go-dnscollector/pkgconfig"
+)
+
+func TestPipeline_IsRouteExist(t *testing.T) {
+ // Create a mock configuration for testing
+ config := &pkgconfig.Config{}
+ config.Pipelines = []pkgconfig.ConfigPipelines{
+ {Name: "validroute"},
+ }
+
+ // Case where the route exists
+ existingRoute := "validroute"
+ err := IsRouteExist(existingRoute, config)
+ if err != nil {
+ t.Errorf("For the existing route %s, an unexpected error was returned: %v", existingRoute, err)
+ }
+
+ // Case where the route does not exist
+ nonExistingRoute := "non-existent-route"
+ err = IsRouteExist(nonExistingRoute, config)
+ if err == nil {
+ t.Errorf("For the non-existing route %s, an expected error was not returned. Received error: %v", nonExistingRoute, err)
+ }
+}
+
+func TestPipeline_StanzaNameIsUniq(t *testing.T) {
+ // Create a mock configuration for testing
+ config := &pkgconfig.Config{}
+ config.Pipelines = []pkgconfig.ConfigPipelines{
+ {Name: "unique-stanza"},
+ {Name: "duplicate-stanza"},
+ {Name: "duplicate-stanza"},
+ }
+
+ // Case where the stanza name is unique
+ uniqueStanzaName := "unique-stanza"
+ err := StanzaNameIsUniq(uniqueStanzaName, config)
+ if err != nil {
+ t.Errorf("For the unique stanza name %s, an unexpected error was returned: %v", uniqueStanzaName, err)
+ }
+
+ // Case where the stanza name is not unique
+ duplicateStanzaName := "duplicate-stanza"
+ err = StanzaNameIsUniq(duplicateStanzaName, config)
+ if err == nil {
+ t.Errorf("For the duplicate stanza name %s, an expected error was not returned. Received error: %v", duplicateStanzaName, err)
+ }
+}
diff --git a/pkgutils/routing.go b/pkgutils/routing.go
new file mode 100644
index 00000000..85d813d0
--- /dev/null
+++ b/pkgutils/routing.go
@@ -0,0 +1,125 @@
+package pkgutils
+
+import (
+ "time"
+
+ "github.com/dmachard/go-dnscollector/dnsutils"
+ "github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-logger"
+)
+
+func GetRoutes(routes []Worker) ([]chan dnsutils.DNSMessage, []string) {
+ channels := []chan dnsutils.DNSMessage{}
+ names := []string{}
+ for _, p := range routes {
+ if c := p.GetInputChannel(); c != nil {
+ channels = append(channels, c)
+ names = append(names, p.GetName())
+ } else {
+ panic("default routing to stanza=[" + p.GetName() + "] not supported")
+ }
+ }
+ return channels, names
+}
+
+type RoutingHandler struct {
+ name string
+ logger *logger.Logger
+ config *pkgconfig.Config
+ stopRun chan bool
+ doneRun chan bool
+ droppedCount map[string]int
+ dropped chan string
+ droppedRoutes []Worker
+ defaultRoutes []Worker
+}
+
+func NewRoutingHandler(config *pkgconfig.Config, console *logger.Logger, name string) RoutingHandler {
+ rh := RoutingHandler{
+ name: name,
+ logger: console,
+ config: config,
+ stopRun: make(chan bool),
+ doneRun: make(chan bool),
+ }
+ go rh.Run()
+ return rh
+}
+func (rh *RoutingHandler) LogInfo(msg string, v ...interface{}) {
+ rh.logger.Info("["+rh.name+"] "+msg, v...)
+}
+
+func (rh *RoutingHandler) LogError(msg string, v ...interface{}) {
+ rh.logger.Error("["+rh.name+"] "+msg, v...)
+}
+
+func (rh *RoutingHandler) LogFatal(msg string) {
+ rh.logger.Error("[" + rh.name + "] " + msg)
+}
+
+func (rh *RoutingHandler) AddDroppedRoute(wrk Worker) {
+ rh.droppedRoutes = append(rh.droppedRoutes, wrk)
+}
+
+func (rh *RoutingHandler) AddDefaultRoute(wrk Worker) {
+ rh.defaultRoutes = append(rh.defaultRoutes, wrk)
+}
+
+func (rh *RoutingHandler) SetDefaultRoutes(workers []Worker) {
+ rh.defaultRoutes = workers
+}
+
+func (rh *RoutingHandler) GetDefaultRoutes() ([]chan dnsutils.DNSMessage, []string) {
+ return GetRoutes(rh.defaultRoutes)
+}
+
+func (rh *RoutingHandler) GetDroppedRoutes() ([]chan dnsutils.DNSMessage, []string) {
+ return GetRoutes(rh.droppedRoutes)
+}
+
+func (rh *RoutingHandler) Stop() {
+ rh.LogInfo("stopping routing handler...")
+ rh.stopRun <- true
+ <-rh.doneRun
+
+ rh.LogInfo("routing handler stopped")
+}
+
+func (rh *RoutingHandler) Run() {
+ rh.LogInfo("starting routing handler...")
+ nextBufferInterval := 10 * time.Second
+ nextBufferFull := time.NewTimer(nextBufferInterval)
+
+RUN_LOOP:
+ for {
+ select {
+ case <-rh.stopRun:
+ rh.doneRun <- true
+ break RUN_LOOP
+ case stanzaName := <-rh.dropped:
+ if _, ok := rh.droppedCount[stanzaName]; !ok {
+ rh.droppedCount[stanzaName] = 1
+ } else {
+ rh.droppedCount[stanzaName]++
+ }
+ case <-nextBufferFull.C:
+ for v, k := range rh.droppedCount {
+ if k > 0 {
+ rh.LogError("stanza[%s] buffer is full, %d packet(s) dropped", v, k)
+ rh.droppedCount[v] = 0
+ }
+ }
+ nextBufferFull.Reset(nextBufferInterval)
+ }
+ }
+}
+
+func (rh *RoutingHandler) SendTo(routes []chan dnsutils.DNSMessage, routesName []string, dm dnsutils.DNSMessage) {
+ for i := range routes {
+ select {
+ case routes[i] <- dm:
+ default:
+ rh.dropped <- routesName[i]
+ }
+ }
+}
diff --git a/pkgutils/utils.go b/pkgutils/utils.go
new file mode 100644
index 00000000..77086425
--- /dev/null
+++ b/pkgutils/utils.go
@@ -0,0 +1,18 @@
+package pkgutils
+
+import (
+ "github.com/dmachard/go-dnscollector/dnsutils"
+ "github.com/dmachard/go-dnscollector/pkgconfig"
+)
+
+type Worker interface {
+ AddDefaultRoute(wrk Worker)
+ AddDroppedRoute(wrk Worker)
+ SetLoggers(loggers []Worker)
+ GetName() string
+ Stop()
+ Run()
+ GetInputChannel() chan dnsutils.DNSMessage
+ ReadConfig()
+ ReloadConfig(config *pkgconfig.Config)
+}
diff --git a/processors/dns.go b/processors/dns.go
index 9b3d6a6f..f1691192 100644
--- a/processors/dns.go
+++ b/processors/dns.go
@@ -6,6 +6,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
"github.com/miekg/dns"
@@ -18,33 +19,31 @@ func GetFakeDNS() ([]byte, error) {
}
type DNSProcessor struct {
- doneRun chan bool
- stopRun chan bool
- doneMonitor chan bool
- stopMonitor chan bool
- recvFrom chan dnsutils.DNSMessage
- logger *logger.Logger
- config *pkgconfig.Config
- ConfigChan chan *pkgconfig.Config
- name string
- dropped chan string
- droppedCount map[string]int
+ doneRun chan bool
+ stopRun chan bool
+ doneMonitor chan bool
+ stopMonitor chan bool
+ recvFrom chan dnsutils.DNSMessage
+ logger *logger.Logger
+ config *pkgconfig.Config
+ ConfigChan chan *pkgconfig.Config
+ name string
+ RoutingHandler pkgutils.RoutingHandler
}
func NewDNSProcessor(config *pkgconfig.Config, logger *logger.Logger, name string, size int) DNSProcessor {
logger.Info("[%s] processor=dns - initialization...", name)
d := DNSProcessor{
- doneMonitor: make(chan bool),
- doneRun: make(chan bool),
- stopMonitor: make(chan bool),
- stopRun: make(chan bool),
- recvFrom: make(chan dnsutils.DNSMessage, size),
- logger: logger,
- config: config,
- ConfigChan: make(chan *pkgconfig.Config),
- name: name,
- dropped: make(chan string),
- droppedCount: map[string]int{},
+ doneMonitor: make(chan bool),
+ doneRun: make(chan bool),
+ stopMonitor: make(chan bool),
+ stopRun: make(chan bool),
+ recvFrom: make(chan dnsutils.DNSMessage, size),
+ logger: logger,
+ config: config,
+ ConfigChan: make(chan *pkgconfig.Config),
+ name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
}
return d
}
@@ -68,55 +67,21 @@ func (d *DNSProcessor) GetChannelList() []chan dnsutils.DNSMessage {
}
func (d *DNSProcessor) Stop() {
+ d.LogInfo("stopping routing handler...")
+ d.RoutingHandler.Stop()
+
d.LogInfo("stopping to process...")
d.stopRun <- true
<-d.doneRun
-
- d.LogInfo("stopping to monitor loggers...")
- d.stopMonitor <- true
- <-d.doneMonitor
}
-func (d *DNSProcessor) MonitorLoggers() {
- watchInterval := 10 * time.Second
- bufferFull := time.NewTimer(watchInterval)
-FOLLOW_LOOP:
- for {
- select {
- case <-d.stopMonitor:
- close(d.dropped)
- bufferFull.Stop()
- d.doneMonitor <- true
- break FOLLOW_LOOP
-
- case loggerName := <-d.dropped:
- if _, ok := d.droppedCount[loggerName]; !ok {
- d.droppedCount[loggerName] = 1
- } else {
- d.droppedCount[loggerName]++
- }
-
- case <-bufferFull.C:
-
- for v, k := range d.droppedCount {
- if k > 0 {
- d.LogError("logger[%s] buffer is full, %d packet(s) dropped", v, k)
- d.droppedCount[v] = 0
- }
- }
- bufferFull.Reset(watchInterval)
-
- }
- }
- d.LogInfo("monitor terminated")
-}
+func (d *DNSProcessor) Run(defaultWorkers []pkgutils.Worker, droppedworkers []pkgutils.Worker) {
+ // prepare next channels
+ defaultRoutes, defaultNames := pkgutils.GetRoutes(defaultWorkers)
+ droppedRoutes, droppedNames := pkgutils.GetRoutes(droppedworkers)
-func (d *DNSProcessor) Run(loggersChannel []chan dnsutils.DNSMessage, loggersName []string) {
// prepare enabled transformers
- transforms := transformers.NewTransforms(&d.config.IngoingTransformers, d.logger, d.name, loggersChannel, 0)
-
- // start goroutine to count dropped messsages
- go d.MonitorLoggers()
+ transforms := transformers.NewTransforms(&d.config.IngoingTransformers, d.logger, d.name, defaultRoutes, 0)
// read incoming dns message
d.LogInfo("waiting dns message to process...")
@@ -180,6 +145,7 @@ RUN_LOOP:
// apply all enabled transformers
if transforms.ProcessMessage(&dm) == transformers.ReturnDrop {
+ d.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
@@ -187,13 +153,8 @@ RUN_LOOP:
dm.DNSTap.LatencySec = fmt.Sprintf("%.6f", dm.DNSTap.Latency)
// dispatch dns message to all generators
- for i := range loggersChannel {
- select {
- case loggersChannel[i] <- dm: // Successful send to logger channel
- default:
- d.dropped <- loggersName[i]
- }
- }
+ d.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
+
}
}
d.LogInfo("processing terminated")
diff --git a/processors/dns_test.go b/processors/dns_test.go
index 8cc872b9..fc5927fc 100644
--- a/processors/dns_test.go
+++ b/processors/dns_test.go
@@ -5,7 +5,9 @@ import (
"testing"
"github.com/dmachard/go-dnscollector/dnsutils"
+ "github.com/dmachard/go-dnscollector/loggers"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-logger"
)
@@ -16,15 +18,15 @@ func Test_DnsProcessor(t *testing.T) {
// init and run the dns processor
consumer := NewDNSProcessor(pkgconfig.GetFakeConfig(), logger, "test", 512)
- chanTo := make(chan dnsutils.DNSMessage, 512)
- go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"})
+
+ fl := loggers.NewFakeLogger()
+ go consumer.Run([]pkgutils.Worker{fl}, []pkgutils.Worker{fl})
dm := dnsutils.GetFakeDNSMessageWithPayload()
consumer.GetChannel() <- dm
// read dns message from dnstap consumer
- dmOut := <-chanTo
-
+ dmOut := <-fl.GetInputChannel()
if dmOut.DNS.Qname != "dnscollector.dev" {
t.Errorf("invalid qname in dns message: %s", dm.DNS.Qname)
}
diff --git a/processors/dnstap.go b/processors/dnstap.go
index ffaa73b2..67839c73 100644
--- a/processors/dnstap.go
+++ b/processors/dnstap.go
@@ -9,6 +9,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-dnstap-protobuf"
"github.com/dmachard/go-logger"
@@ -51,38 +52,38 @@ func GetFakeDNSTap(dnsquery []byte) *dnstap.Dnstap {
}
type DNSTapProcessor struct {
- ConnID int
- doneRun chan bool
- stopRun chan bool
- doneMonitor chan bool
- stopMonitor chan bool
- recvFrom chan []byte
- logger *logger.Logger
- config *pkgconfig.Config
- ConfigChan chan *pkgconfig.Config
- name string
- chanSize int
- dropped chan string
- droppedCount map[string]int
+ ConnID int
+ doneRun chan bool
+ stopRun chan bool
+ doneMonitor chan bool
+ stopMonitor chan bool
+ recvFrom chan []byte
+ logger *logger.Logger
+ config *pkgconfig.Config
+ ConfigChan chan *pkgconfig.Config
+ name string
+ chanSize int
+ RoutingHandler pkgutils.RoutingHandler
}
func NewDNSTapProcessor(connID int, config *pkgconfig.Config, logger *logger.Logger, name string, size int) DNSTapProcessor {
logger.Info("[%s] processor=dnstap#%d - initialization...", name, connID)
d := DNSTapProcessor{
- ConnID: connID,
- doneMonitor: make(chan bool),
- doneRun: make(chan bool),
- stopMonitor: make(chan bool),
- stopRun: make(chan bool),
- recvFrom: make(chan []byte, size),
- chanSize: size,
- logger: logger,
- config: config,
- ConfigChan: make(chan *pkgconfig.Config),
- name: name,
- dropped: make(chan string),
- droppedCount: map[string]int{},
+ ConnID: connID,
+ doneMonitor: make(chan bool),
+ doneRun: make(chan bool),
+ stopMonitor: make(chan bool),
+ stopRun: make(chan bool),
+ recvFrom: make(chan []byte, size),
+ chanSize: size,
+ logger: logger,
+ config: config,
+ ConfigChan: make(chan *pkgconfig.Config),
+ name: name,
+ // dropped: make(chan string),
+ // droppedCount: map[string]int{},
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
}
return d
@@ -113,56 +114,64 @@ func (d *DNSTapProcessor) GetChannel() chan []byte {
}
func (d *DNSTapProcessor) Stop() {
+ d.LogInfo("stopping routing handler...")
+ d.RoutingHandler.Stop()
+
d.LogInfo("stopping to process...")
d.stopRun <- true
<-d.doneRun
- d.LogInfo("stopping to monitor loggers...")
- d.stopMonitor <- true
- <-d.doneMonitor
-}
-
-func (d *DNSTapProcessor) MonitorLoggers() {
- watchInterval := 10 * time.Second
- bufferFull := time.NewTimer(watchInterval)
-MONITOR_LOOP:
- for {
- select {
- case <-d.stopMonitor:
- close(d.dropped)
- bufferFull.Stop()
- d.doneMonitor <- true
- break MONITOR_LOOP
-
- case loggerName := <-d.dropped:
- if _, ok := d.droppedCount[loggerName]; !ok {
- d.droppedCount[loggerName] = 1
- } else {
- d.droppedCount[loggerName]++
- }
-
- case <-bufferFull.C:
- for v, k := range d.droppedCount {
- if k > 0 {
- d.LogError("logger[%s] buffer is full, %d packet(s) dropped", v, k)
- d.droppedCount[v] = 0
- }
- }
- bufferFull.Reset(watchInterval)
-
- }
- }
- d.LogInfo("monitor terminated")
+ // d.LogInfo("stopping to monitor loggers...")
+ // d.stopMonitor <- true
+ // <-d.doneMonitor
}
-func (d *DNSTapProcessor) Run(loggersChannel []chan dnsutils.DNSMessage, loggersName []string) {
+// func (d *DNSTapProcessor) MonitorLoggers() {
+// // watchInterval := 10 * time.Second
+// // bufferFull := time.NewTimer(watchInterval)
+// MONITOR_LOOP:
+// for {
+// select {
+// case <-d.stopMonitor:
+// // close(d.dropped)
+// // bufferFull.Stop()
+// d.doneMonitor <- true
+// break MONITOR_LOOP
+
+// case loggerName := <-d.dropped:
+// if _, ok := d.droppedCount[loggerName]; !ok {
+// d.droppedCount[loggerName] = 1
+// } else {
+// d.droppedCount[loggerName]++
+// }
+
+// case <-bufferFull.C:
+// for v, k := range d.droppedCount {
+// if k > 0 {
+// d.LogError("logger[%s] buffer is full, %d packet(s) dropped", v, k)
+// d.droppedCount[v] = 0
+// }
+// }
+// bufferFull.Reset(watchInterval)
+
+// }
+// }
+// d.LogInfo("monitor terminated")
+// }
+
+// func (d *DNSTapProcessor) Run(loggersChannel []chan dnsutils.DNSMessage, loggersName []string) {
+func (d *DNSTapProcessor) Run(defaultWorkers []pkgutils.Worker, droppedworkers []pkgutils.Worker) {
dt := &dnstap.Dnstap{}
+ // prepare next channels
+ defaultRoutes, defaultNames := pkgutils.GetRoutes(defaultWorkers)
+ droppedRoutes, droppedNames := pkgutils.GetRoutes(droppedworkers)
+
// prepare enabled transformers
- transforms := transformers.NewTransforms(&d.config.IngoingTransformers, d.logger, d.name, loggersChannel, d.ConnID)
+ transforms := transformers.NewTransforms(&d.config.IngoingTransformers, d.logger, d.name, defaultRoutes, d.ConnID)
// start goroutine to count dropped messsages
- go d.MonitorLoggers()
+ // go d.MonitorLoggers()
// read incoming dns message
d.LogInfo("waiting dns message to process...")
@@ -283,20 +292,15 @@ RUN_LOOP:
// apply all enabled transformers
if transforms.ProcessMessage(&dm) == transformers.ReturnDrop {
+ d.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
// convert latency to human
dm.DNSTap.LatencySec = fmt.Sprintf("%.6f", dm.DNSTap.Latency)
- // dispatch dns message to connected loggers
- for i := range loggersChannel {
- select {
- case loggersChannel[i] <- dm: // Successful send to logger channel
- default:
- d.dropped <- loggersName[i]
- }
- }
+ // dispatch dns message to connected routes
+ d.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
}
}
diff --git a/processors/dnstap_test.go b/processors/dnstap_test.go
index 99405173..1f981079 100644
--- a/processors/dnstap_test.go
+++ b/processors/dnstap_test.go
@@ -4,8 +4,9 @@ import (
"bytes"
"testing"
- "github.com/dmachard/go-dnscollector/dnsutils"
+ "github.com/dmachard/go-dnscollector/loggers"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnstap-protobuf"
"github.com/dmachard/go-logger"
"github.com/miekg/dns"
@@ -19,7 +20,7 @@ func Test_DnstapProcessor(t *testing.T) {
// init the dnstap consumer
consumer := NewDNSTapProcessor(0, pkgconfig.GetFakeConfig(), logger, "test", 512)
- chanTo := make(chan dnsutils.DNSMessage, 512)
+ // chanTo := make(chan dnsutils.DNSMessage, 512)
// prepare dns query
dnsmsg := new(dns.Msg)
@@ -36,12 +37,15 @@ func Test_DnstapProcessor(t *testing.T) {
data, _ := proto.Marshal(dt)
- go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"})
+ // run the consumer with a fake logger
+ fl := loggers.NewFakeLogger()
+ go consumer.Run([]pkgutils.Worker{fl}, []pkgutils.Worker{fl})
+
// add packet to consumer
consumer.GetChannel() <- data
// read dns message from dnstap consumer
- dm := <-chanTo
+ dm := <-fl.GetInputChannel()
if dm.DNS.Qname != "www.google.fr" {
t.Errorf("invalid qname in dns message: %s", dm.DNS.Qname)
}
@@ -54,7 +58,7 @@ func Test_DnstapProcessor_MalformedDnsHeader(t *testing.T) {
// init the dnstap consumer
consumer := NewDNSTapProcessor(0, pkgconfig.GetFakeConfig(), logger, "test", 512)
- chanTo := make(chan dnsutils.DNSMessage, 512)
+ // chanTo := make(chan dnsutils.DNSMessage, 512)
// prepare dns query
dnsmsg := new(dns.Msg)
@@ -71,12 +75,16 @@ func Test_DnstapProcessor_MalformedDnsHeader(t *testing.T) {
data, _ := proto.Marshal(dt)
- go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"})
+ // run the consumer with a fake logger
+ fl := loggers.NewFakeLogger()
+ go consumer.Run([]pkgutils.Worker{fl}, []pkgutils.Worker{fl})
+
+ // go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"})
// add packet to consumer
consumer.GetChannel() <- data
// read dns message from dnstap consumer
- dm := <-chanTo
+ dm := <-fl.GetInputChannel()
if dm.DNS.MalformedPacket == false {
t.Errorf("malformed packet not detected")
}
@@ -89,7 +97,7 @@ func Test_DnstapProcessor_MalformedDnsQuestion(t *testing.T) {
// init the dnstap consumer
consumer := NewDNSTapProcessor(0, pkgconfig.GetFakeConfig(), logger, "test", 512)
- chanTo := make(chan dnsutils.DNSMessage, 512)
+ // chanTo := make(chan dnsutils.DNSMessage, 512)
// prepare dns query
dnsquestion := []byte{88, 27, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 15, 100, 110, 115, 116, 97, 112,
@@ -105,12 +113,16 @@ func Test_DnstapProcessor_MalformedDnsQuestion(t *testing.T) {
data, _ := proto.Marshal(dt)
- go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"})
+ // run the consumer with a fake logger
+ fl := loggers.NewFakeLogger()
+ go consumer.Run([]pkgutils.Worker{fl}, []pkgutils.Worker{fl})
+
+ // go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"})
// add packet to consumer
consumer.GetChannel() <- data
// read dns message from dnstap consumer
- dm := <-chanTo
+ dm := <-fl.GetInputChannel()
if dm.DNS.MalformedPacket == false {
t.Errorf("malformed packet not detected")
}
@@ -123,7 +135,7 @@ func Test_DnstapProcessor_MalformedDnsAnswer(t *testing.T) {
// init the dnstap consumer
consumer := NewDNSTapProcessor(0, pkgconfig.GetFakeConfig(), logger, "test", 512)
- chanTo := make(chan dnsutils.DNSMessage, 512)
+ // chanTo := make(chan dnsutils.DNSMessage, 512)
// prepare dns query
dnsanswer := []byte{46, 172, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 15, 100, 110, 115, 116, 97, 112, 99, 111, 108, 108, 101, 99, 116,
@@ -140,12 +152,16 @@ func Test_DnstapProcessor_MalformedDnsAnswer(t *testing.T) {
data, _ := proto.Marshal(dt)
- go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"})
+ // run the consumer with a fake logger
+ fl := loggers.NewFakeLogger()
+ go consumer.Run([]pkgutils.Worker{fl}, []pkgutils.Worker{fl})
+
+ // go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"})
// add packet to consumer
consumer.GetChannel() <- data
// read dns message from dnstap consumer
- dm := <-chanTo
+ dm := <-fl.GetInputChannel()
if dm.DNS.MalformedPacket == false {
t.Errorf("malformed packet not detected")
}
@@ -161,7 +177,7 @@ func Test_DnstapProcessor_DisableDNSParser(t *testing.T) {
cfg.Collectors.Dnstap.DisableDNSParser = true
consumer := NewDNSTapProcessor(0, cfg, logger, "test", 512)
- chanTo := make(chan dnsutils.DNSMessage, 512)
+ // chanTo := make(chan dnsutils.DNSMessage, 512)
// prepare dns query
dnsmsg := new(dns.Msg)
@@ -178,12 +194,16 @@ func Test_DnstapProcessor_DisableDNSParser(t *testing.T) {
data, _ := proto.Marshal(dt)
- go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"})
+ // run the consumer with a fake logger
+ fl := loggers.NewFakeLogger()
+ go consumer.Run([]pkgutils.Worker{fl}, []pkgutils.Worker{fl})
+
+ // go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"})
// add packet to consumer
consumer.GetChannel() <- data
// read dns message from dnstap consumer
- dm := <-chanTo
+ dm := <-fl.GetInputChannel()
if dm.DNS.ID != 0 {
t.Errorf("DNS ID should be equal to zero: %d", dm.DNS.ID)
}
diff --git a/processors/powerdns.go b/processors/powerdns.go
index 7fd0f629..9d2b317c 100644
--- a/processors/powerdns.go
+++ b/processors/powerdns.go
@@ -10,6 +10,7 @@ import (
"github.com/dmachard/go-dnscollector/dnsutils"
"github.com/dmachard/go-dnscollector/netlib"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-dnscollector/transformers"
"github.com/dmachard/go-logger"
powerdns_protobuf "github.com/dmachard/go-powerdns-protobuf"
@@ -27,37 +28,35 @@ var (
)
type PdnsProcessor struct {
- ConnID int
- doneRun chan bool
- stopRun chan bool
- doneMonitor chan bool
- stopMonitor chan bool
- recvFrom chan []byte
- logger *logger.Logger
- config *pkgconfig.Config
- ConfigChan chan *pkgconfig.Config
- name string
- chanSize int
- dropped chan string
- droppedCount map[string]int
+ ConnID int
+ doneRun chan bool
+ stopRun chan bool
+ doneMonitor chan bool
+ stopMonitor chan bool
+ recvFrom chan []byte
+ logger *logger.Logger
+ config *pkgconfig.Config
+ ConfigChan chan *pkgconfig.Config
+ name string
+ chanSize int
+ RoutingHandler pkgutils.RoutingHandler
}
func NewPdnsProcessor(connID int, config *pkgconfig.Config, logger *logger.Logger, name string, size int) PdnsProcessor {
logger.Info("[%s] processor=pdns#%d - initialization...", name, connID)
d := PdnsProcessor{
- ConnID: connID,
- doneMonitor: make(chan bool),
- doneRun: make(chan bool),
- stopMonitor: make(chan bool),
- stopRun: make(chan bool),
- recvFrom: make(chan []byte, size),
- chanSize: size,
- logger: logger,
- config: config,
- ConfigChan: make(chan *pkgconfig.Config),
- name: name,
- dropped: make(chan string),
- droppedCount: map[string]int{},
+ ConnID: connID,
+ doneMonitor: make(chan bool),
+ doneRun: make(chan bool),
+ stopMonitor: make(chan bool),
+ stopRun: make(chan bool),
+ recvFrom: make(chan []byte, size),
+ chanSize: size,
+ logger: logger,
+ config: config,
+ ConfigChan: make(chan *pkgconfig.Config),
+ name: name,
+ RoutingHandler: pkgutils.NewRoutingHandler(config, logger, name),
}
return d
}
@@ -87,57 +86,23 @@ func (p *PdnsProcessor) GetChannel() chan []byte {
}
func (p *PdnsProcessor) Stop() {
+ p.LogInfo("stopping routing handler...")
+ p.RoutingHandler.Stop()
+
p.LogInfo("stopping to process...")
p.stopRun <- true
<-p.doneRun
-
- p.LogInfo("stopping to monitor loggers...")
- p.stopMonitor <- true
- <-p.doneMonitor
}
-func (p *PdnsProcessor) MonitorLoggers() {
- watchInterval := 10 * time.Second
- bufferFull := time.NewTimer(watchInterval)
-FOLLOW_LOOP:
- for {
- select {
- case <-p.stopMonitor:
- close(p.dropped)
- bufferFull.Stop()
- p.doneMonitor <- true
- break FOLLOW_LOOP
-
- case loggerName := <-p.dropped:
- if _, ok := p.droppedCount[loggerName]; !ok {
- p.droppedCount[loggerName] = 1
- } else {
- p.droppedCount[loggerName]++
- }
-
- case <-bufferFull.C:
-
- for v, k := range p.droppedCount {
- if k > 0 {
- p.LogError("logger[%s] buffer is full, %d packet(s) dropped", v, k)
- p.droppedCount[v] = 0
- }
- }
- bufferFull.Reset(watchInterval)
-
- }
- }
- p.LogInfo("monitor terminated")
-}
-
-func (p *PdnsProcessor) Run(loggersChannel []chan dnsutils.DNSMessage, loggersName []string) {
+func (p *PdnsProcessor) Run(defaultWorkers []pkgutils.Worker, droppedworkers []pkgutils.Worker) {
pbdm := &powerdns_protobuf.PBDNSMessage{}
- // prepare enabled transformers
- transforms := transformers.NewTransforms(&p.config.IngoingTransformers, p.logger, p.name, loggersChannel, p.ConnID)
+ // prepare next channels
+ defaultRoutes, defaultNames := pkgutils.GetRoutes(defaultWorkers)
+ droppedRoutes, droppedNames := pkgutils.GetRoutes(droppedworkers)
- // start goroutine to count dropped messsages
- go p.MonitorLoggers()
+ // prepare enabled transformers
+ transforms := transformers.NewTransforms(&p.config.IngoingTransformers, p.logger, p.name, defaultRoutes, p.ConnID)
// read incoming dns message
p.LogInfo("waiting dns message to process...")
@@ -354,17 +319,12 @@ RUN_LOOP:
// apply all enabled transformers
if transforms.ProcessMessage(&dm) == transformers.ReturnDrop {
+ p.RoutingHandler.SendTo(droppedRoutes, droppedNames, dm)
continue
}
// dispatch dns messages to connected loggers
- for i := range loggersChannel {
- select {
- case loggersChannel[i] <- dm: // Successful send to logger channel
- default:
- p.dropped <- loggersName[i]
- }
- }
+ p.RoutingHandler.SendTo(defaultRoutes, defaultNames, dm)
}
}
p.LogInfo("processing terminated")
diff --git a/processors/powerdns_test.go b/processors/powerdns_test.go
index 9babc599..2c34bcda 100644
--- a/processors/powerdns_test.go
+++ b/processors/powerdns_test.go
@@ -3,8 +3,9 @@ package processors
import (
"testing"
- "github.com/dmachard/go-dnscollector/dnsutils"
+ "github.com/dmachard/go-dnscollector/loggers"
"github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-dnscollector/pkgutils"
"github.com/dmachard/go-logger"
powerdns_protobuf "github.com/dmachard/go-powerdns-protobuf"
"github.com/miekg/dns"
@@ -14,7 +15,6 @@ import (
func TestPowerDNS_Processor(t *testing.T) {
// init the dnstap consumer
consumer := NewPdnsProcessor(0, pkgconfig.GetFakeConfig(), logger.New(false), "test", 512)
- chanTo := make(chan dnsutils.DNSMessage, 512)
// init the powerdns processor
dnsQname := pkgconfig.ValidDomain
@@ -29,12 +29,15 @@ func TestPowerDNS_Processor(t *testing.T) {
data, _ := proto.Marshal(dm)
- go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"})
+ // run the consumer with a fake logger
+ fl := loggers.NewFakeLogger()
+ go consumer.Run([]pkgutils.Worker{fl}, []pkgutils.Worker{fl})
+
// add packet to consumer
consumer.GetChannel() <- data
// read dns message from dnstap consumer
- msg := <-chanTo
+ msg := <-fl.GetInputChannel()
if msg.DNSTap.Identity != "powerdnspb" {
t.Errorf("invalid identity in dns message: %s", msg.DNSTap.Identity)
}
@@ -46,7 +49,6 @@ func TestPowerDNS_Processor_AddDNSPayload_Valid(t *testing.T) {
// init the powerdns processor
consumer := NewPdnsProcessor(0, cfg, logger.New(false), "test", 512)
- chanTo := make(chan dnsutils.DNSMessage, 1)
// prepare powerdns message
dnsQname := pkgconfig.ValidDomain
@@ -63,11 +65,14 @@ func TestPowerDNS_Processor_AddDNSPayload_Valid(t *testing.T) {
data, _ := proto.Marshal(dm)
// start the consumer and add packet
- go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"})
+ // run the consumer with a fake logger
+ fl := loggers.NewFakeLogger()
+ go consumer.Run([]pkgutils.Worker{fl}, []pkgutils.Worker{fl})
+
consumer.GetChannel() <- data
// read dns message
- msg := <-chanTo
+ msg := <-fl.GetInputChannel()
// checks
if msg.DNS.Length == 0 {
@@ -94,7 +99,6 @@ func TestPowerDNS_Processor_AddDNSPayload_InvalidLabelLength(t *testing.T) {
// init the dnstap consumer
consumer := NewPdnsProcessor(0, cfg, logger.New(false), "test", 512)
- chanTo := make(chan dnsutils.DNSMessage, 512)
// prepare dnstap
dnsQname := pkgconfig.BadDomainLabel
@@ -110,12 +114,15 @@ func TestPowerDNS_Processor_AddDNSPayload_InvalidLabelLength(t *testing.T) {
data, _ := proto.Marshal(dm)
- go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"})
+ // run the consumer with a fake logger
+ fl := loggers.NewFakeLogger()
+ go consumer.Run([]pkgutils.Worker{fl}, []pkgutils.Worker{fl})
+
// add packet to consumer
consumer.GetChannel() <- data
// read dns message from dnstap consumer
- msg := <-chanTo
+ msg := <-fl.GetInputChannel()
if !msg.DNS.MalformedPacket {
t.Errorf("DNS message should malformed")
}
@@ -127,7 +134,6 @@ func TestPowerDNS_Processor_AddDNSPayload_QnameTooLongDomain(t *testing.T) {
// init the dnstap consumer
consumer := NewPdnsProcessor(0, cfg, logger.New(false), "test", 512)
- chanTo := make(chan dnsutils.DNSMessage, 512)
// prepare dnstap
dnsQname := pkgconfig.BadVeryLongDomain
@@ -142,12 +148,15 @@ func TestPowerDNS_Processor_AddDNSPayload_QnameTooLongDomain(t *testing.T) {
data, _ := proto.Marshal(dm)
- go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"})
+ // run the consumer with a fake logger
+ fl := loggers.NewFakeLogger()
+ go consumer.Run([]pkgutils.Worker{fl}, []pkgutils.Worker{fl})
+
// add packet to consumer
consumer.GetChannel() <- data
// read dns message from dnstap consumer
- msg := <-chanTo
+ msg := <-fl.GetInputChannel()
if !msg.DNS.MalformedPacket {
t.Errorf("DNS message should malformed because of qname too long")
}
@@ -159,7 +168,6 @@ func TestPowerDNS_Processor_AddDNSPayload_AnswersTooLongDomain(t *testing.T) {
// init the dnstap consumer
consumer := NewPdnsProcessor(0, cfg, logger.New(false), "test", 512)
- chanTo := make(chan dnsutils.DNSMessage, 512)
// prepare dnstap
dnsQname := pkgconfig.ValidDomain
@@ -185,12 +193,15 @@ func TestPowerDNS_Processor_AddDNSPayload_AnswersTooLongDomain(t *testing.T) {
data, _ := proto.Marshal(dm)
- go consumer.Run([]chan dnsutils.DNSMessage{chanTo}, []string{"test"})
+ // run the consumer with a fake logger
+ fl := loggers.NewFakeLogger()
+ go consumer.Run([]pkgutils.Worker{fl}, []pkgutils.Worker{fl})
+
// add packet to consumer
consumer.GetChannel() <- data
// read dns message from dnstap consumer
- msg := <-chanTo
+ msg := <-fl.GetInputChannel()
// tests verifications
if !msg.DNS.MalformedPacket {
diff --git a/transformers/atags.go b/transformers/atags.go
new file mode 100644
index 00000000..aff6e0b2
--- /dev/null
+++ b/transformers/atags.go
@@ -0,0 +1,57 @@
+package transformers
+
+import (
+ "github.com/dmachard/go-dnscollector/dnsutils"
+ "github.com/dmachard/go-dnscollector/pkgconfig"
+ "github.com/dmachard/go-logger"
+)
+
+type ATagsProcessor struct {
+ config *pkgconfig.ConfigTransformers
+ logger *logger.Logger
+ name string
+ instance int
+ outChannels []chan dnsutils.DNSMessage
+ logInfo func(msg string, v ...interface{})
+ logError func(msg string, v ...interface{})
+}
+
+func NewATagsSubprocessor(config *pkgconfig.ConfigTransformers, logger *logger.Logger, name string,
+ instance int, outChannels []chan dnsutils.DNSMessage,
+ logInfo func(msg string, v ...interface{}), logError func(msg string, v ...interface{})) ATagsProcessor {
+ s := ATagsProcessor{
+ config: config,
+ logger: logger,
+ name: name,
+ instance: instance,
+ outChannels: outChannels,
+ logInfo: logInfo,
+ logError: logError,
+ }
+
+ return s
+}
+
+func (p *ATagsProcessor) ReloadConfig(config *pkgconfig.ConfigTransformers) {
+ p.config = config
+}
+
+func (p *ATagsProcessor) InitDNSMessage(dm *dnsutils.DNSMessage) {
+ if dm.ATags == nil {
+ dm.ATags = &dnsutils.TransformATags{
+ Tags: []string{},
+ }
+
+ }
+}
+
+func (p *ATagsProcessor) IsEnabled() bool {
+ return p.config.ATags.Enable
+}
+
+func (p *ATagsProcessor) AddTags(dm *dnsutils.DNSMessage) int {
+ if p.config.ATags.Enable {
+ dm.ATags.Tags = append(dm.ATags.Tags, p.config.ATags.Tags...)
+ }
+ return ReturnSuccess
+}
diff --git a/transformers/subprocessors.go b/transformers/subprocessors.go
index 075b0604..71b5cd6b 100644
--- a/transformers/subprocessors.go
+++ b/transformers/subprocessors.go
@@ -33,6 +33,7 @@ type Transforms struct {
ReducerTransform *ReducerProcessor
ExtractProcessor ExtractProcessor
MachineLearningTransform MlProcessor
+ ATagsTransform ATagsProcessor
activeTransforms []func(dm *dnsutils.DNSMessage) int
}
@@ -55,6 +56,7 @@ func NewTransforms(config *pkgconfig.ConfigTransformers, logger *logger.Logger,
d.FilteringTransform = NewFilteringProcessor(config, logger, name, instance, outChannels, d.LogInfo, d.LogError)
d.GeoipTransform = NewDNSGeoIPProcessor(config, logger, name, instance, outChannels, d.LogInfo, d.LogError)
d.MachineLearningTransform = NewMachineLearningSubprocessor(config, logger, name, instance, outChannels, d.LogInfo, d.LogError)
+ d.ATagsTransform = NewATagsSubprocessor(config, logger, name, instance, outChannels, d.LogInfo, d.LogError)
d.Prepare()
return d
@@ -71,6 +73,7 @@ func (p *Transforms) ReloadConfig(config *pkgconfig.ConfigTransformers) {
p.ReducerTransform.ReloadConfig(config)
p.ExtractProcessor.ReloadConfig(config)
p.MachineLearningTransform.ReloadConfig(config)
+ p.ATagsTransform.ReloadConfig(config)
p.Prepare()
}
@@ -169,6 +172,12 @@ func (p *Transforms) Prepare() error {
p.LogInfo(prefixlog + enabled)
}
+ if p.config.ATags.Enable {
+ p.activeTransforms = append(p.activeTransforms, p.ATagsTransform.AddTags)
+ prefixlog := fmt.Sprintf("transformer=atags#%d - ", p.instance)
+ p.LogInfo(prefixlog + "subprocessor atags is enabled")
+ }
+
return nil
}
@@ -204,6 +213,10 @@ func (p *Transforms) InitDNSMessageFormat(dm *dnsutils.DNSMessage) {
if p.config.MachineLearning.Enable {
p.MachineLearningTransform.InitDNSMessage(dm)
}
+
+ if p.config.ATags.Enable {
+ p.ATagsTransform.InitDNSMessage(dm)
+ }
}
func (p *Transforms) Reset() {