More flexibility in the multiplexer configuration (#100)

* More flexibility in the multiplexer configuration * add missing method in dnssniffer * fix test unit
dmachard · Jul 29, 2022 · befada6 · befada6
1 parent 4c42a7a
commit befada6
Show file tree

Hide file tree

Showing 38 changed files with 1,009 additions and 920 deletions.
diff --git a/README.md b/README.md
@@ -17,38 +17,35 @@ NOTE: The code before version 1.x is considered beta quality and is subject to b
 ## Features
 
 - [Logs routing](doc/multiplexer.md)
-
+- [Queries/Replies JSON encoding](doc/dnsjson.md)
 - Collectors:
-    - [DNStap streams](doc/configuration.md#dns-tap) 
-    - [DNS packets sniffer](doc/configuration.md#dns-sniffer)
-    - [Tail on log file](doc/configuration.md#tail)
-    - [Protobuf PowerDNS](doc/configuration.md#protobuf-powerdns)
-
-- Transformers:
-    - [Queries/Replies JSON encoding](doc/dnsjson.md)
-    - [DNS filtering](doc/configuration.md#dns-filtering)
-    - [User Privacy](doc/configuration.md#user-privacy)
-    - [Normalize Qname](doc/configuration.md#qname-lowercase)
-
+    - [DNStap streams](doc/collectors.md#dns-tap) 
+    - [DNS packets sniffer](doc/collectors.md#dns-sniffer)
+    - [Tail on log file](doc/collectors.md#tail)
+    - [Protobuf PowerDNS](doc/collectors.md#protobuf-powerdns)
 - Loggers:
-    - [Console](doc/configuration.md#stdout)
-    - [Prometheus](doc/configuration.md#prometheus)
-    - [File](doc/configuration.md#log-file)
-    - [DNStap](doc/configuration.md#dnstap-client)
-    - [TCP](doc/configuration.md#tcp-client)
-    - [REST API](doc/configuration.md#rest-api)
-    - [Syslog](doc/configuration.md#syslog)
-    - [Fluentd](doc/configuration.md#fluentd-client)
-    - [Pcap](doc/configuration.md#pcap-file)
-    - [InfluxDB](doc/configuration.md#influxdb-client)
-    - [Loki](doc/configuration.md#loki-client)
-    - [Statsd](doc/configuration.md#statsd-client)
+    - [Console](doc/loggers.md#stdout)
+    - [Prometheus](doc/loggers.md#prometheus)
+    - [File](doc/loggers.md#log-file)
+    - [DNStap](doc/loggers.md#dnstap-client)
+    - [TCP](doc/loggers.md#tcp-client)
+    - [REST API](doc/loggers.md#rest-api)
+    - [Syslog](doc/loggers.md#syslog)
+    - [Fluentd](doc/loggers.md#fluentd-client)
+    - [Pcap](doc/loggers.md#pcap-file)
+    - [InfluxDB](doc/loggers.md#influxdb-client)
+    - [Loki](doc/loggers.md#loki-client)
+    - [Statsd](doc/loggers.md#statsd-client)
 
 - Other features
     - [DNS decoder with extended options support](doc/dnsparser.md)
     - [Built-in Grafana dashboards](doc/dashboards.md)
     - [GeoIP support](doc/configuration.md#geoip-support)
     - [Text format](doc/configuration.md#custom-text-format)
+    - [DNS filtering](doc/configuration.md#dns-filtering)
+    - [User Privacy](doc/configuration.md#user-privacy)
+    - [Normalize Qname](doc/configuration.md#qname-lowercase)
+
 ## Installation
 
 **Run-it from binary**

diff --git a/collectors/dnssniffer.go b/collectors/dnssniffer.go
@@ -134,6 +134,12 @@ func (c *DnsSniffer) LogError(msg string, v ...interface{}) {
 	c.logger.Error("["+c.name+"] sniffer collector - "+msg, v...)
 }
 
+func (c *DnsSniffer) GetName() string { return c.name }
+
+func (c *DnsSniffer) SetLoggers(loggers []dnsutils.Worker) {
+	c.loggers = loggers
+}
+
 func (c *DnsSniffer) Loggers() []chan dnsutils.DnsMessage {
 	channels := []chan dnsutils.DnsMessage{}
 	for _, p := range c.loggers {

diff --git a/collectors/dnssniffer_darwin.go b/collectors/dnssniffer_darwin.go
@@ -32,6 +32,12 @@ func NewDnsSniffer(loggers []dnsutils.Worker, config *dnsutils.Config, logger *l
 	return s
 }
 
+func (c *DnsSniffer) GetName() string { return c.name }
+
+func (c *DnsSniffer) SetLoggers(loggers []dnsutils.Worker) {
+	c.loggers = loggers
+}
+
 func (c *DnsSniffer) LogInfo(msg string, v ...interface{}) {
 	c.logger.Info("["+c.name+"] collector dns sniffer - "+msg, v...)
 }

diff --git a/collectors/dnssniffer_windows.go b/collectors/dnssniffer_windows.go
@@ -32,6 +32,11 @@ func NewDnsSniffer(loggers []dnsutils.Worker, config *dnsutils.Config, logger *l
 	return s
 }
 
+func (c *DnsSniffer) GetName() string { return c.name }
+
+func (c *DnsSniffer) SetLoggers(loggers []dnsutils.Worker) {
+	c.loggers = loggers
+}
 func (c *DnsSniffer) LogInfo(msg string, v ...interface{}) {
 	c.logger.Info("["+c.name+"] collector dns sniffer - "+msg, v...)
 }

diff --git a/collectors/dnstap.go b/collectors/dnstap.go
@@ -37,6 +37,12 @@ func NewDnstap(loggers []dnsutils.Worker, config *dnsutils.Config, logger *logge
 	return s
 }
 
+func (c *Dnstap) GetName() string { return c.name }
+
+func (c *Dnstap) SetLoggers(loggers []dnsutils.Worker) {
+	c.loggers = loggers
+}
+
 func (c *Dnstap) Loggers() []chan dnsutils.DnsMessage {
 	channels := []chan dnsutils.DnsMessage{}
 	for _, p := range c.loggers {

diff --git a/collectors/powerdns.go b/collectors/powerdns.go
@@ -35,6 +35,12 @@ func NewProtobufPowerDNS(loggers []dnsutils.Worker, config *dnsutils.Config, log
 	return s
 }
 
+func (c *ProtobufPowerDNS) GetName() string { return c.name }
+
+func (c *ProtobufPowerDNS) SetLoggers(loggers []dnsutils.Worker) {
+	c.loggers = loggers
+}
+
 func (c *ProtobufPowerDNS) Loggers() []chan dnsutils.DnsMessage {
 	channels := []chan dnsutils.DnsMessage{}
 	for _, p := range c.loggers {

diff --git a/collectors/tail.go b/collectors/tail.go
@@ -38,6 +38,12 @@ func NewTail(loggers []dnsutils.Worker, config *dnsutils.Config, logger *logger.
 	return s
 }
 
+func (c *Tail) GetName() string { return c.name }
+
+func (c *Tail) SetLoggers(loggers []dnsutils.Worker) {
+	c.loggers = loggers
+}
+
 func (c *Tail) Loggers() []chan dnsutils.DnsMessage {
 	channels := []chan dnsutils.DnsMessage{}
 	for _, p := range c.loggers {

diff --git a/config.yml b/config.yml
@@ -1,3 +1,62 @@
+
+
+################################################
+# global configuration
+################################################
+global:
+  # If turned on, log some applications messages
+  trace:
+    # debug informations
+    verbose: true
+    # log malformed packet
+    # log-malformed: false
+    # # filename is the file to write logs to.
+    # filename: ""
+    # # maximum size in megabytes of the log file it gets rotated
+    # max-size: 10
+    # # maximum number of old log files to retain
+    # max-backups: 10
+
+  # all directives for text format output
+  # - timestamp-rfc3339ns: timestamp rfc3339 format, with nano support
+  # - timestamp-unixms: unix timestamp with ms support
+  # - timestamp-unixus: unix timestamp with us support
+  # - timestamp-unixns: unix timestamp with nano support
+  # - localtime: local time
+  # - identity: dnstap identity
+  # - operation: dnstap operation
+  # - opcode: dns opcode (integer)
+  # - rcode: dns return code
+  # - queryip: dns query ip
+  # - queryport: dns query port
+  # - responseip: dns response ip
+  # - responseport: dns response port
+  # - id: dns id
+  # - family: ip protocol version INET or INET6
+  # - protocol: protocol UDP, TCP
+  # - length: the length of the query or reply
+  # - qtype: dns qtype
+  # - qname: dns qname
+  # - latency: computed latency between queries and replies
+  # - answercount: the number of answer
+  # - continent: continent code
+  # - country: country iso code
+  # - city: city name
+  # - as-number: autonomous system number
+  # - as-owner: autonomous system organization
+  # - ttl: answer ttl, only the first one value
+  # - answer: rdata answer, only the first one, prefer to use the JSON format if you wamt all answers
+  # - malformed: malformed dns packet, integer value 1/0
+  # - qr: query or reply flag, string value Q/R
+  # - tc: truncated flag
+  # - aa: authoritative answer
+  # - ra: recursion available
+  # - ad: authenticated data
+  # - edns-csubnet: client subnet
+  # - pdns-tags: powerdns metadata
+  # - pdns-original-request-client: powerdns metadata
+  # text-format: "timestamp-rfc3339ns identity operation rcode queryip queryport family protocol length qname qtype latency"
+
 # create your dns collector, please refer bellow to see the list 
 # of supported collectors, loggers and transformers
 multiplexer:
@@ -7,20 +66,14 @@ multiplexer:
         listen-ip: 0.0.0.0
         listen-port: 6000
 
-  transformers:
-    - name: qnamelower 
-      normalize:  
-        lowercase-qname: true
-
   loggers:
     - name: console
       stdout:
         mode: text
 
   routes:
     - from: [ tap ]
-      transforms: [ qnamelower ]
-      to: [ prom ]
+      to: [ console ]
 
 ################################################
 # list of supported collectors
@@ -386,61 +439,3 @@ multiplexer:
 #   mmdb-city-file: ""
 #   # path file to your mmdb ASN database
 #   mmdb-asn-file: ""
-
-
-################################################
-# global configuration
-################################################
-global:
-  # If turned on, log some applications messages
-  trace:
-    # debug informations
-    verbose: false
-    # log malformed packet
-    log-malformed: false
-    # filename is the file to write logs to.
-    filename: ""
-    # maximum size in megabytes of the log file it gets rotated
-    max-size: 10
-    # maximum number of old log files to retain
-    max-backups: 10
-
-  # all directives for text format output
-  # - timestamp-rfc3339ns: timestamp rfc3339 format, with nano support
-  # - timestamp-unixms: unix timestamp with ms support
-  # - timestamp-unixus: unix timestamp with us support
-  # - timestamp-unixns: unix timestamp with nano support
-  # - localtime: local time
-  # - identity: dnstap identity
-  # - operation: dnstap operation
-  # - opcode: dns opcode (integer)
-  # - rcode: dns return code
-  # - queryip: dns query ip
-  # - queryport: dns query port
-  # - responseip: dns response ip
-  # - responseport: dns response port
-  # - id: dns id
-  # - family: ip protocol version INET or INET6
-  # - protocol: protocol UDP, TCP
-  # - length: the length of the query or reply
-  # - qtype: dns qtype
-  # - qname: dns qname
-  # - latency: computed latency between queries and replies
-  # - answercount: the number of answer
-  # - continent: continent code
-  # - country: country iso code
-  # - city: city name
-  # - as-number: autonomous system number
-  # - as-owner: autonomous system organization
-  # - ttl: answer ttl, only the first one value
-  # - answer: rdata answer, only the first one, prefer to use the JSON format if you wamt all answers
-  # - malformed: malformed dns packet, integer value 1/0
-  # - qr: query or reply flag, string value Q/R
-  # - tc: truncated flag
-  # - aa: authoritative answer
-  # - ra: recursion available
-  # - ad: authenticated data
-  # - edns-csubnet: client subnet
-  # - pdns-tags: powerdns metadata
-  # - pdns-original-request-client: powerdns metadata
-  text-format: "timestamp-rfc3339ns identity operation rcode queryip queryport family protocol length qname qtype latency"