From 0087683450ca7afb539fd41cc5e21399b70a6d4c Mon Sep 17 00:00:00 2001 From: dmachard <5562930+dmachard@users.noreply.github.com> Date: Sun, 12 Dec 2021 09:20:47 +0100 Subject: [PATCH] rest api: dump full list of clients ip and domains #21 --- README.md | 9 ++++--- doc/swagger.yml | 33 ++++++++++++++++++++++- loggers/webserver.go | 46 +++++++++++++++++++++++++++++++++ subprocessors/statistics.go | 24 +++++++++++++++++ subprocessors/statsperstream.go | 30 +++++++++++++++++++-- 5 files changed, 136 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ae8ea3e5..9f1ffaad 100644 --- a/README.md +++ b/README.md @@ -95,12 +95,15 @@ incoming traffics. See [Configuration guide](doc/configuration.md). - [x] [Use case 2: collect dnstap stream and get statistics usage with Prometheus/Grafana](https://dmachard.github.io/posts/0035-dnscollector-grafana-prometheus/) - [x] [Use case 3: collect dnstap stream and log dns answers in JSON format](https://dmachard.github.io/posts/0042-dnscollector-dnstap-json-answers/) -## Openmetrics - +## Metrics | Metric | Description | | ---- | ---- | -| xx | xx | +| dnscollector_qps | Number of queries per second received | +| dnscollector_requesters_total | Number of clients | +| dnscollector_domains_total | Number of domains observed | +| dnscollector_received_bytes_total | Total bytes received | +| dnscollector_sent_bytes_total | Total bytes sent | ## Benchmark diff --git a/doc/swagger.yml b/doc/swagger.yml index 746f453e..46dbb8f6 100644 --- a/doc/swagger.yml +++ b/doc/swagger.yml @@ -151,7 +151,38 @@ paths: schema: type: string summary: Top suspicious domains list - + /dump/requesters: + get: + parameters: + - in: query + name: stream + schema: + type: string + description: stream name + responses: + '200': + description: Return full list of requesters + content: + text/plain: + schema: + type: string + summary: Return full list of requesters + /dump/domains: + get: + parameters: + - in: query + name: stream + schema: + type: string + description: stream name + responses: + '200': + description: Return full list of domains + content: + text/plain: + schema: + type: string + summary: Return full list of domains security: [] externalDocs: url: 'https://github.com/dmachard/go-dnscollector' \ No newline at end of file diff --git a/loggers/webserver.go b/loggers/webserver.go index 47997cce..9877f707 100644 --- a/loggers/webserver.go +++ b/loggers/webserver.go @@ -374,6 +374,48 @@ func (s *Webserver) metricsHandler(w http.ResponseWriter, r *http.Request) { } } +func (s *Webserver) dumpRequestersHandler(w http.ResponseWriter, r *http.Request) { + if !s.BasicAuth(w, r) { + http.Error(w, "Not authorized", http.StatusUnauthorized) + return + } + + w.Header().Set("Content-Type", "application/json") + + switch r.Method { + case http.MethodGet: + stream, ok := r.URL.Query()["stream"] + if !ok || len(stream) < 1 { + stream = []string{"global"} + } + t := s.stats.GetClients(stream[0]) + json.NewEncoder(w).Encode(t) + default: + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + } +} + +func (s *Webserver) dumpDomainsHandler(w http.ResponseWriter, r *http.Request) { + if !s.BasicAuth(w, r) { + http.Error(w, "Not authorized", http.StatusUnauthorized) + return + } + + w.Header().Set("Content-Type", "application/json") + + switch r.Method { + case http.MethodGet: + stream, ok := r.URL.Query()["stream"] + if !ok || len(stream) < 1 { + stream = []string{"global"} + } + t := s.stats.GetDomains(stream[0]) + json.NewEncoder(w).Encode(t) + default: + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + } +} + func (s *Webserver) topRequestersHandler(w http.ResponseWriter, r *http.Request) { if !s.BasicAuth(w, r) { http.Error(w, "Not authorized", http.StatusUnauthorized) @@ -527,6 +569,7 @@ func (s *Webserver) ListenAndServe() { mux := http.NewServeMux() mux.HandleFunc("/metrics", s.metricsHandler) mux.HandleFunc("/reset", s.resetHandler) + mux.HandleFunc("/top/requesters", s.topRequestersHandler) mux.HandleFunc("/top/requesters/suspicious", s.topSuspiciousClientsHandler) mux.HandleFunc("/top/firstleveldomains", s.topAllFirstLevelDomainsHandler) @@ -535,6 +578,9 @@ func (s *Webserver) ListenAndServe() { mux.HandleFunc("/top/domains/slow", s.topSlowDomainsHandler) mux.HandleFunc("/top/domains/suspicious", s.topSuspiciousDomainsHandler) + mux.HandleFunc("/dump/requesters", s.dumpRequestersHandler) + mux.HandleFunc("/dump/domains", s.dumpDomainsHandler) + var err error var listener net.Listener addrlisten := s.config.Loggers.WebServer.ListenIP + ":" + strconv.Itoa(s.config.Loggers.WebServer.ListenPort) diff --git a/subprocessors/statistics.go b/subprocessors/statistics.go index 808dfbbf..a22cd0f6 100644 --- a/subprocessors/statistics.go +++ b/subprocessors/statistics.go @@ -300,3 +300,27 @@ func (c *StatsStreams) GetTopIpProto(identity string) (ret []topmap.TopMapItem) return v.GetTopIpProto() } + +func (c *StatsStreams) GetClients(identity string) (ret map[string]int) { + c.RLock() + defer c.RUnlock() + + v, found := c.streams[identity] + if !found { + return map[string]int{} + } + + return v.GetClients() +} + +func (c *StatsStreams) GetDomains(identity string) (ret map[string]int) { + c.RLock() + defer c.RUnlock() + + v, found := c.streams[identity] + if !found { + return map[string]int{} + } + + return v.GetDomains() +} diff --git a/subprocessors/statsperstream.go b/subprocessors/statsperstream.go index 9be76298..318a1bf4 100644 --- a/subprocessors/statsperstream.go +++ b/subprocessors/statsperstream.go @@ -62,7 +62,8 @@ type Counters struct { type StatsPerStream struct { config *dnsutils.Config - total Counters + total Counters + firstleveldomains map[string]int firstleveldomainstop *topmap.TopMap qnames map[string]int @@ -87,7 +88,8 @@ type StatsPerStream struct { transportstop *topmap.TopMap ipproto map[string]int ipprototop *topmap.TopMap - commonQtypes map[string]bool + + commonQtypes map[string]bool sync.RWMutex } @@ -634,3 +636,27 @@ func (c *StatsPerStream) GetTopIpProto() (ret []topmap.TopMapItem) { return c.ipprototop.Get() } + +func (c *StatsPerStream) GetClients() (ret map[string]int) { + c.RLock() + defer c.RUnlock() + + retMap := map[string]int{} + for k, v := range c.clients { + retMap[k] = v + } + + return retMap +} + +func (c *StatsPerStream) GetDomains() (ret map[string]int) { + c.RLock() + defer c.RUnlock() + + retMap := map[string]int{} + for k, v := range c.qnames { + retMap[k] = v + } + + return retMap +}