From e6737bc1156445093097c199cba66d85c909b748 Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Sat, 11 Dec 2021 07:17:25 +0100 Subject: [PATCH] extend statistics by domain level #6 --- config.yml | 2 +- loggers/webserver.go | 34 +++++++++++++++++++++++++-------- subprocessors/statistics.go | 24 +++++++++++++++++++++++ subprocessors/statsperstream.go | 31 ++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 9 deletions(-) diff --git a/config.yml b/config.yml index 3456b8a0..90a5a0e1 100644 --- a/config.yml +++ b/config.yml @@ -1,7 +1,7 @@ # If turned on, log some applications messages trace: # debug informations - verbose: false + verbose: true # filename is the file to write logs to. filename: "" # maximum size in megabytes of the log file it gets rotated diff --git a/loggers/webserver.go b/loggers/webserver.go index d414827f..45f83dc3 100644 --- a/loggers/webserver.go +++ b/loggers/webserver.go @@ -118,20 +118,14 @@ func (s *Webserver) metricsHandler(w http.ResponseWriter, r *http.Request) { // add build version info fmt.Fprintf(w, "# HELP %s_build_info Build version\n", suffix) fmt.Fprintf(w, "# TYPE %s_build_info gauge\n", suffix) - fmt.Fprintf(w, "%s_build_info{version=\"%s\"} 1\n", suffix, s.ver) - // bytes - fmt.Fprintf(w, "# HELP %s_received_bytes_total Total bytes received\n", suffix) - fmt.Fprintf(w, "# TYPE %s_received_bytes_total counter\n", suffix) - fmt.Fprintf(w, "# HELP %s_sent_bytes_total Total bytes sent\n", suffix) - fmt.Fprintf(w, "# TYPE %s_sent_bytes_total counter\n", suffix) - - // docs + // client fmt.Fprintf(w, "# HELP %s_requesters_total Number of clients\n", suffix) fmt.Fprintf(w, "# TYPE %s_requesters_total counter\n", suffix) fmt.Fprintf(w, "# HELP %s_requesters_top_total Number of hit per client, partitioned by client ip\n", suffix) fmt.Fprintf(w, "# TYPE %s_requesters_top_total counter\n", suffix) + // domains fmt.Fprintf(w, "# HELP %s_domains_total Number of domains\n", suffix) fmt.Fprintf(w, "# TYPE %s_domains_total counter\n", suffix) fmt.Fprintf(w, "# HELP %s_domains_top_total Number of hit per domain, partitioned by qname\n", suffix) @@ -152,6 +146,7 @@ func (s *Webserver) metricsHandler(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "# HELP %s_domains_suspicious_top_total Number of hit per suspicious domains, partitioned by qname\n", suffix) fmt.Fprintf(w, "# TYPE %s_domains_suspicious_top_total counter\n", suffix) + // packets fmt.Fprintf(w, "# HELP %s_pps Number of packets per second received\n", suffix) fmt.Fprintf(w, "# TYPE %s_pps gauge\n", suffix) fmt.Fprintf(w, "# HELP %s_pps_max_total Maximum number of packets per second received\n", suffix) @@ -198,6 +193,7 @@ func (s *Webserver) metricsHandler(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "# HELP %s_reply_len_min_total Minimum reply length observed\n", suffix) fmt.Fprintf(w, "# TYPE %s_reply_len_min_total counter\n", suffix) + // malformed fmt.Fprintf(w, "# HELP %s_packets_malformed_total Number of packets\n", suffix) fmt.Fprintf(w, "# TYPE %s_packets_malformed_total counter\n", suffix) fmt.Fprintf(w, "# HELP %s_clients_suspicious_total Number of suspicious clients\n", suffix) @@ -205,11 +201,27 @@ func (s *Webserver) metricsHandler(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "# HELP %s_clients_suspicious_top_total Number of hit per suspicious clients, partitioned by ip\n", suffix) fmt.Fprintf(w, "# TYPE %s_clients_suspicious_top_total counter\n", suffix) + // bytes + fmt.Fprintf(w, "# HELP %s_received_bytes_total Total bytes received\n", suffix) + fmt.Fprintf(w, "# TYPE %s_received_bytes_total counter\n", suffix) + fmt.Fprintf(w, "# HELP %s_sent_bytes_total Total bytes sent\n", suffix) + fmt.Fprintf(w, "# TYPE %s_sent_bytes_total counter\n", suffix) + + // first level domains + fmt.Fprintf(w, "# HELP %s_firstleveldomains_total Number of first level domains\n", suffix) + fmt.Fprintf(w, "# TYPE %s_firstleveldomains_total counter\n", suffix) + fmt.Fprintf(w, "# HELP %s_firstleveldomains_top_total Number of hit per first level domains\n", suffix) + fmt.Fprintf(w, "# TYPE %s_firstleveldomains_top_total counter\n", suffix) + + fmt.Fprintf(w, "%s_build_info{version=\"%s\"} 1\n", suffix, s.ver) for _, stream := range s.stats.Streams() { counters := s.stats.GetCounters(stream) totalClients := s.stats.GetTotalClients(stream) + totalFlds := s.stats.GetTotalFirstLevelDomains(stream) + topFlds := s.stats.GetTopFirstLevelDomains(stream) + totalDomains := s.stats.GetTotalDomains(stream) topDomains := s.stats.GetTopQnames(stream) @@ -335,6 +347,12 @@ func (s *Webserver) metricsHandler(w http.ResponseWriter, r *http.Request) { // bytes fmt.Fprintf(w, "%s_received_bytes_total{stream=\"%s\"} %d\n", suffix, stream, counters.ReceivedBytesTotal) fmt.Fprintf(w, "%s_sent_bytes_total{stream=\"%s\"} %d\n", suffix, stream, counters.SentBytesTotal) + + // first level domains + fmt.Fprintf(w, "%s_firstleveldomains_total{stream=\"%s\"} %d\n", suffix, stream, totalFlds) + for _, v := range topFlds { + fmt.Fprintf(w, "%s_firstleveldomains_top_total{stream=\"%s\",domain=\"%s\"} %d\n", suffix, stream, v.Name, v.Hit) + } } default: http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) diff --git a/subprocessors/statistics.go b/subprocessors/statistics.go index c4ae0f2f..bf20309b 100644 --- a/subprocessors/statistics.go +++ b/subprocessors/statistics.go @@ -74,6 +74,18 @@ func (c *StatsStreams) GetTotalDomains(identity string) (ret int) { return v.GetTotalDomains() } +func (c *StatsStreams) GetTotalFirstLevelDomains(identity string) (ret int) { + c.RLock() + defer c.RUnlock() + + v, found := c.streams[identity] + if !found { + return 0 + } + + return v.GetTotalFirstLevelDomains() +} + func (c *StatsStreams) GetTotalNxdomains(identity string) (ret int) { c.RLock() defer c.RUnlock() @@ -146,6 +158,18 @@ func (c *StatsStreams) GetTopQnames(identity string) (ret []topmap.TopMapItem) { return v.GetTopQnames() } +func (c *StatsStreams) GetTopFirstLevelDomains(identity string) (ret []topmap.TopMapItem) { + c.RLock() + defer c.RUnlock() + + v, found := c.streams[identity] + if !found { + return []topmap.TopMapItem{} + } + + return v.GetTopFirstLevelDomains() +} + func (c *StatsStreams) GetTopNxdomains(identity string) (ret []topmap.TopMapItem) { c.RLock() defer c.RUnlock() diff --git a/subprocessors/statsperstream.go b/subprocessors/statsperstream.go index 662ba0ce..f15c4ccd 100644 --- a/subprocessors/statsperstream.go +++ b/subprocessors/statsperstream.go @@ -1,6 +1,7 @@ package subprocessors import ( + "strings" "sync" "github.com/dmachard/go-dnscollector/dnsutils" @@ -58,6 +59,8 @@ type StatsPerStream struct { config *dnsutils.Config total Counters + firstleveldomains map[string]int + firstleveldomainstop *topmap.TopMap qnames map[string]int qnamestop *topmap.TopMap qnamesNxd map[string]int @@ -88,6 +91,8 @@ func NewStatsPerStream(config *dnsutils.Config) *StatsPerStream { c := &StatsPerStream{ config: config, total: Counters{}, + firstleveldomains: make(map[string]int), + firstleveldomainstop: topmap.NewTopMap(config.Subprocessors.Statistics.TopMaxItems), qnames: make(map[string]int), qnamestop: topmap.NewTopMap(config.Subprocessors.Statistics.TopMaxItems), qnamesNxd: make(map[string]int), @@ -312,6 +317,18 @@ func (c *StatsPerStream) Record(dm dnsutils.DnsMessage) { } c.transportstop.Record(dm.Protocol, c.transports[dm.Protocol]) + // record first level domain + i := strings.LastIndex(dm.Qname, ".") + if i > -1 { + fld := dm.Qname[i+1:] + if _, ok := c.firstleveldomains[fld]; !ok { + c.firstleveldomains[fld] = 1 + } else { + c.firstleveldomains[fld]++ + } + c.firstleveldomainstop.Record(fld, c.firstleveldomains[fld]) + } + // record all qnames if _, ok := c.qnames[dm.Qname]; !ok { c.qnames[dm.Qname] = 1 @@ -399,6 +416,13 @@ func (c *StatsPerStream) GetTotalDomains() (ret int) { return len(c.qnames) } +func (c *StatsPerStream) GetTotalFirstLevelDomains() (ret int) { + c.RLock() + defer c.RUnlock() + + return len(c.firstleveldomains) +} + func (c *StatsPerStream) GetTotalNxdomains() (ret int) { c.RLock() defer c.RUnlock() @@ -441,6 +465,13 @@ func (c *StatsPerStream) GetTopQnames() (ret []topmap.TopMapItem) { return c.qnamestop.Get() } +func (c *StatsPerStream) GetTopFirstLevelDomains() (ret []topmap.TopMapItem) { + c.RLock() + defer c.RUnlock() + + return c.firstleveldomainstop.Get() +} + func (c *StatsPerStream) GetTopNxdomains() (ret []topmap.TopMapItem) { c.RLock() defer c.RUnlock()