extend statistics by domain level #6

config.yml

@@ -1,7 +1,7 @@
 # If turned on, log some applications messages
 trace:
   # debug informations
-  verbose: false
+  verbose: true
   # filename is the file to write logs to.
   filename: ""
   # maximum size in megabytes of the log file it gets rotated

loggers/webserver.go

@@ -118,20 +118,14 @@ func (s *Webserver) metricsHandler(w http.ResponseWriter, r *http.Request) {
 		// add build version info
 		fmt.Fprintf(w, "# HELP %s_build_info Build version\n", suffix)
 		fmt.Fprintf(w, "# TYPE %s_build_info gauge\n", suffix)
-		fmt.Fprintf(w, "%s_build_info{version=\"%s\"} 1\n", suffix, s.ver)
 
-		// bytes
-		fmt.Fprintf(w, "# HELP %s_received_bytes_total Total bytes received\n", suffix)
-		fmt.Fprintf(w, "# TYPE %s_received_bytes_total counter\n", suffix)
-		fmt.Fprintf(w, "# HELP %s_sent_bytes_total Total bytes sent\n", suffix)
-		fmt.Fprintf(w, "# TYPE %s_sent_bytes_total counter\n", suffix)
-
-		// docs
+		// client
 		fmt.Fprintf(w, "# HELP %s_requesters_total Number of clients\n", suffix)
 		fmt.Fprintf(w, "# TYPE %s_requesters_total counter\n", suffix)
 		fmt.Fprintf(w, "# HELP %s_requesters_top_total Number of hit per client, partitioned by client ip\n", suffix)
 		fmt.Fprintf(w, "# TYPE %s_requesters_top_total counter\n", suffix)
 
+		// domains
 		fmt.Fprintf(w, "# HELP %s_domains_total Number of domains\n", suffix)
 		fmt.Fprintf(w, "# TYPE %s_domains_total counter\n", suffix)
 		fmt.Fprintf(w, "# HELP %s_domains_top_total Number of hit per domain, partitioned by qname\n", suffix)
@@ -152,6 +146,7 @@ func (s *Webserver) metricsHandler(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprintf(w, "# HELP %s_domains_suspicious_top_total Number of hit per suspicious domains, partitioned by qname\n", suffix)
 		fmt.Fprintf(w, "# TYPE %s_domains_suspicious_top_total counter\n", suffix)
 
+		// packets
 		fmt.Fprintf(w, "# HELP %s_pps Number of packets per second received\n", suffix)
 		fmt.Fprintf(w, "# TYPE %s_pps gauge\n", suffix)
 		fmt.Fprintf(w, "# HELP %s_pps_max_total Maximum number of packets per second received\n", suffix)
@@ -198,18 +193,35 @@ func (s *Webserver) metricsHandler(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprintf(w, "# HELP %s_reply_len_min_total Minimum reply length observed\n", suffix)
 		fmt.Fprintf(w, "# TYPE %s_reply_len_min_total counter\n", suffix)
 
+		// malformed
 		fmt.Fprintf(w, "# HELP %s_packets_malformed_total Number of packets\n", suffix)
 		fmt.Fprintf(w, "# TYPE %s_packets_malformed_total counter\n", suffix)
 		fmt.Fprintf(w, "# HELP %s_clients_suspicious_total Number of suspicious clients\n", suffix)
 		fmt.Fprintf(w, "# TYPE %s_clients_suspicious_total counter\n", suffix)
 		fmt.Fprintf(w, "# HELP %s_clients_suspicious_top_total Number of hit per suspicious clients, partitioned by ip\n", suffix)
 		fmt.Fprintf(w, "# TYPE %s_clients_suspicious_top_total counter\n", suffix)
 
+		// bytes
+		fmt.Fprintf(w, "# HELP %s_received_bytes_total Total bytes received\n", suffix)
+		fmt.Fprintf(w, "# TYPE %s_received_bytes_total counter\n", suffix)
+		fmt.Fprintf(w, "# HELP %s_sent_bytes_total Total bytes sent\n", suffix)
+		fmt.Fprintf(w, "# TYPE %s_sent_bytes_total counter\n", suffix)
+
+		// first level domains
+		fmt.Fprintf(w, "# HELP %s_firstleveldomains_total Number of first level domains\n", suffix)
+		fmt.Fprintf(w, "# TYPE %s_firstleveldomains_total counter\n", suffix)
+		fmt.Fprintf(w, "# HELP %s_firstleveldomains_top_total Number of hit per first level domains\n", suffix)
+		fmt.Fprintf(w, "# TYPE %s_firstleveldomains_top_total counter\n", suffix)
+
+		fmt.Fprintf(w, "%s_build_info{version=\"%s\"} 1\n", suffix, s.ver)
 		for _, stream := range s.stats.Streams() {
 
 			counters := s.stats.GetCounters(stream)
 			totalClients := s.stats.GetTotalClients(stream)
 
+			totalFlds := s.stats.GetTotalFirstLevelDomains(stream)
+			topFlds := s.stats.GetTopFirstLevelDomains(stream)
+
 			totalDomains := s.stats.GetTotalDomains(stream)
 			topDomains := s.stats.GetTopQnames(stream)
 
@@ -335,6 +347,12 @@ func (s *Webserver) metricsHandler(w http.ResponseWriter, r *http.Request) {
 			// bytes
 			fmt.Fprintf(w, "%s_received_bytes_total{stream=\"%s\"} %d\n", suffix, stream, counters.ReceivedBytesTotal)
 			fmt.Fprintf(w, "%s_sent_bytes_total{stream=\"%s\"} %d\n", suffix, stream, counters.SentBytesTotal)
+
+			// first level domains
+			fmt.Fprintf(w, "%s_firstleveldomains_total{stream=\"%s\"} %d\n", suffix, stream, totalFlds)
+			for _, v := range topFlds {
+				fmt.Fprintf(w, "%s_firstleveldomains_top_total{stream=\"%s\",domain=\"%s\"} %d\n", suffix, stream, v.Name, v.Hit)
+			}
 		}
 	default:
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)

subprocessors/statistics.go

@@ -74,6 +74,18 @@ func (c *StatsStreams) GetTotalDomains(identity string) (ret int) {
 	return v.GetTotalDomains()
 }
 
+func (c *StatsStreams) GetTotalFirstLevelDomains(identity string) (ret int) {
+	c.RLock()
+	defer c.RUnlock()
+
+	v, found := c.streams[identity]
+	if !found {
+		return 0
+	}
+
+	return v.GetTotalFirstLevelDomains()
+}
+
 func (c *StatsStreams) GetTotalNxdomains(identity string) (ret int) {
 	c.RLock()
 	defer c.RUnlock()
@@ -146,6 +158,18 @@ func (c *StatsStreams) GetTopQnames(identity string) (ret []topmap.TopMapItem) {
 	return v.GetTopQnames()
 }
 
+func (c *StatsStreams) GetTopFirstLevelDomains(identity string) (ret []topmap.TopMapItem) {
+	c.RLock()
+	defer c.RUnlock()
+
+	v, found := c.streams[identity]
+	if !found {
+		return []topmap.TopMapItem{}
+	}
+
+	return v.GetTopFirstLevelDomains()
+}
+
 func (c *StatsStreams) GetTopNxdomains(identity string) (ret []topmap.TopMapItem) {
 	c.RLock()
 	defer c.RUnlock()

subprocessors/statsperstream.go

@@ -1,6 +1,7 @@
 package subprocessors
 
 import (
+	"strings"
 	"sync"
 
 	"github.com/dmachard/go-dnscollector/dnsutils"
@@ -58,6 +59,8 @@ type StatsPerStream struct {
 	config *dnsutils.Config
 
 	total                Counters
+	firstleveldomains    map[string]int
+	firstleveldomainstop *topmap.TopMap
 	qnames               map[string]int
 	qnamestop            *topmap.TopMap
 	qnamesNxd            map[string]int
@@ -88,6 +91,8 @@ func NewStatsPerStream(config *dnsutils.Config) *StatsPerStream {
 	c := &StatsPerStream{
 		config:               config,
 		total:                Counters{},
+		firstleveldomains:    make(map[string]int),
+		firstleveldomainstop: topmap.NewTopMap(config.Subprocessors.Statistics.TopMaxItems),
 		qnames:               make(map[string]int),
 		qnamestop:            topmap.NewTopMap(config.Subprocessors.Statistics.TopMaxItems),
 		qnamesNxd:            make(map[string]int),
@@ -312,6 +317,18 @@ func (c *StatsPerStream) Record(dm dnsutils.DnsMessage) {
 	}
 	c.transportstop.Record(dm.Protocol, c.transports[dm.Protocol])
 
+	// record first level domain
+	i := strings.LastIndex(dm.Qname, ".")
+	if i > -1 {
+		fld := dm.Qname[i+1:]
+		if _, ok := c.firstleveldomains[fld]; !ok {
+			c.firstleveldomains[fld] = 1
+		} else {
+			c.firstleveldomains[fld]++
+		}
+		c.firstleveldomainstop.Record(fld, c.firstleveldomains[fld])
+	}
+
 	// record all qnames
 	if _, ok := c.qnames[dm.Qname]; !ok {
 		c.qnames[dm.Qname] = 1
@@ -399,6 +416,13 @@ func (c *StatsPerStream) GetTotalDomains() (ret int) {
 	return len(c.qnames)
 }
 
+func (c *StatsPerStream) GetTotalFirstLevelDomains() (ret int) {
+	c.RLock()
+	defer c.RUnlock()
+
+	return len(c.firstleveldomains)
+}
+
 func (c *StatsPerStream) GetTotalNxdomains() (ret int) {
 	c.RLock()
 	defer c.RUnlock()
@@ -441,6 +465,13 @@ func (c *StatsPerStream) GetTopQnames() (ret []topmap.TopMapItem) {
 	return c.qnamestop.Get()
 }
 
+func (c *StatsPerStream) GetTopFirstLevelDomains() (ret []topmap.TopMapItem) {
+	c.RLock()
+	defer c.RUnlock()
+
+	return c.firstleveldomainstop.Get()
+}
+
 func (c *StatsPerStream) GetTopNxdomains() (ret []topmap.TopMapItem) {
 	c.RLock()
 	defer c.RUnlock()