add global telemetry (#724)

* add telemetry
* add test
* add dashboard,
* update config
* update workers
* update docs
* fix usecase24
* Update README.md

.github/workflows/testing-go.yml

@@ -141,7 +141,7 @@ jobs:
 
     - id: count_tests
       run: |
-        data=$(sudo go test -timeout 360s  -v ./workers ./dnsutils ./transformers ./pkgconfig ./pkginit ././ 2>&1 | grep -c RUN)
+        data=$(sudo go test -timeout 360s  -v ./workers ./dnsutils ./transformers ./pkgconfig ./pkginit ./telemetry ././ 2>&1 | grep -c RUN)
         echo "Count of Tests: $data"
         echo "data=$data" >> $GITHUB_OUTPUT
 

Dockerfile

@@ -20,7 +20,7 @@ USER dnscollector
 COPY --from=builder /build/go-dnscollector /bin/go-dnscollector
 COPY --from=builder /build/docker-config.yml ./etc/dnscollector/config.yml
 
-EXPOSE 6000/tcp 8080/tcp
+EXPOSE 6000/tcp 8080/tcp 9165/tcp
 
 ENTRYPOINT ["/bin/go-dnscollector"]
 

Makefile

@@ -74,6 +74,7 @@ tests: check-go
 	@go test ./pkgconfig/ -race -cover -v
 	@go test ./pkginit/ -race -cover -v
 	@go test ./netutils/ -race -cover -v
+	@go test ./telemetry/ -race -cover -v
 	@go test -timeout 90s ./transformers/ -race -cover -v
 	@go test -timeout 180s ./workers/ -race -cover -v
 

README.md

@@ -1,9 +1,9 @@
 <p align="center">
 <img src="https://goreportcard.com/badge/github.com/dmachard/go-dns-collector" alt="Go Report"/>
 <img src="https://img.shields.io/badge/go%20version-min%201.21-green" alt="Go version"/>
-<img src="https://img.shields.io/badge/go%20tests-437-green" alt="Go tests"/>
+<img src="https://img.shields.io/badge/go%20tests-439-green" alt="Go tests"/>
 <img src="https://img.shields.io/badge/go%20bench-20-green" alt="Go bench"/>
-<img src="https://img.shields.io/badge/go%20lines-29859-green" alt="Go lines"/>
+<img src="https://img.shields.io/badge/go%20lines-30720-green" alt="Go lines"/>
 </p>
 
 <p align="center">
@@ -115,6 +115,17 @@ The [`_integration`](./docs/_integration) folder contains DNS-collector `configu
 - [Elasticsearch](./docs/_integration/elasticsearch/README.md)
 - [Kafka](./docs/_integration/kafka/README.md)
 
+## Telemetry
+
+Performance metrics are available to evaluate the efficiency of your pipelines. These metrics allow you to track:
+- The number of incoming and outgoing packets processed by each worker
+- The number of packets matching the policies applied (forwarded, dropped)
+- The number of "discarded" packets
+- Memory consumption
+- CPU consumption
+
+A [build-in](./docs/dashboards/grafana_exporter.json) dashboard is available for monitoring these metrics.
+
 ## Performance
 
 Tuning may be necessary to deal with a large traffic loads.

config.yml

@@ -10,6 +10,20 @@ global:
   text-format-delimiter: " "
   text-format-boundary: "\""
   pid-file: ""
+  worker:
+    interval-monitor: 10
+  telemetry:
+    enabled: true
+    web-path: "/metrics"
+    web-listen: ":9165"
+    prometheus-prefix: "dnscollector_exporter"
+    tls-support: false
+    tls-cert-file: ""
+    tls-key-file: ""
+    client-ca-file: ""
+    basic-auth-enable: false
+    basic-auth-login: admin
+    basic-auth-pwd: changeme
 
 ################################################
 # Pipelining configuration
@@ -27,7 +41,7 @@ pipelines:
         qname-lowercase: true
     routing-policy:
       forward: [ console ]
-      dropped: []
+      dropped: [ ]
 
   - name: console
     stdout:

dnscollector.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"os/signal"
@@ -12,6 +13,7 @@ import (
 
 	"github.com/dmachard/go-dnscollector/pkgconfig"
 	"github.com/dmachard/go-dnscollector/pkginit"
+	"github.com/dmachard/go-dnscollector/telemetry"
 	"github.com/dmachard/go-dnscollector/workers"
 	"github.com/dmachard/go-logger"
 	"github.com/natefinch/lumberjack"
@@ -150,6 +152,12 @@ func main() {
 	InitLogger(logger, config)
 	logger.Info("main - version=%s revision=%s", version.Version, version.Revision)
 
+	// // telemetry
+	if config.Global.Telemetry.Enabled {
+		logger.Info("main - telemetry enabled on local address: %s", config.Global.Telemetry.WebListen)
+	}
+	promServer, metrics, errTelemetry := telemetry.InitTelemetryServer(config, logger)
+
 	// init active collectors and loggers
 	mapLoggers := make(map[string]workers.Worker)
 	mapCollectors := make(map[string]workers.Worker)
@@ -164,8 +172,8 @@ func main() {
 
 	// or pipeline ?
 	if pkginit.IsPipelinesEnabled(config) {
-		logger.Info("main - running in pipelines mode")
-		err := pkginit.InitPipelines(mapLoggers, mapCollectors, config, logger)
+		logger.Info("main - running in pipeline mode")
+		err := pkginit.InitPipelines(mapLoggers, mapCollectors, config, logger, metrics)
 		if err != nil {
 			logger.Error("main - %s", err.Error())
 			removePIDFile(config)
@@ -183,6 +191,11 @@ func main() {
 	go func() {
 		for {
 			select {
+			case err := <-errTelemetry:
+				logger.Error("main - unable to start telemetry: %v", err)
+				removePIDFile(config)
+				os.Exit(1)
+
 			case <-sigHUP:
 				logger.Warning("main - SIGHUP received")
 
@@ -205,6 +218,19 @@ func main() {
 
 			case <-sigTerm:
 				logger.Warning("main - exiting...")
+
+				// gracefully shutdown the HTTP server
+				if config.Global.Telemetry.Enabled {
+					logger.Info("main - telemetry is stopping")
+					metrics.Stop()
+
+					if err := promServer.Shutdown(context.Background()); err != nil {
+						logger.Error("main - telemetry error shutting down http server - %s", err.Error())
+					}
+
+				}
+
+				// and stop all workers
 				for _, c := range mapCollectors {
 					c.Stop()
 				}

docker-compose.yml

@@ -12,4 +12,5 @@ services:
     ports:
       - "6000:6000/tcp"
       - "8080:8080/tcp"
+      - "9165:9165/tcp"
     restart: always

docs/_examples/use-case-24.yml

@@ -31,7 +31,7 @@ pipelines:
           dns.qtype: "TXT"
     transforms:
       atags:
-        tags: [ "TAG:TXT-QUERIES" ]
+        add-tags: [ "TAG:TXT-QUERIES" ]
     routing-policy:
       forward: [ apple-txt, all-txt ]
 
@@ -51,7 +51,7 @@ pipelines:
           dns.qname: "^*.apple.com$"
     transforms:
       atags:
-        tags: [ "TXT:apple" ]
+        add-tags: [ "TXT:apple" ]
     routing-policy:
       forward: [ outputfile-apple ]
 

docs/configuration.md

@@ -13,6 +13,7 @@ You can find the global settings below
   - [Custom text format](#custom-text-format)
   - [Server identity](#server-identity)
   - [Pid file](#pid-file)
+  - [Telemetry](#telemetry)
 
 ### Trace
 
@@ -127,9 +128,30 @@ Output example:
 
 ### Pid file
 
-Set path to create pid file.
+Set path to create DNS-collector PID.
+By default, this settings is empty.
 
 ```yaml
 global:
   pid-file: "/path/to/your/pidfile.pid"
-```
+```
+
+### Telemetry
+
+Enable and configure telemetry
+
+```yaml
+global:
+  telemetry:
+    enabled: true
+    web-path: "/metrics"
+    web-listen: ":9165"
+    prometheus-prefix: "dnscollector_exporter"
+    tls-support: false
+    tls-cert-file: ""
+    tls-key-file: ""
+    client-ca-file: ""
+    basic-auth-enable: false
+    basic-auth-login: admin
+    basic-auth-pwd: changeme
+```