rest api: dump full list of clients ip and domains #21

README.md

@@ -95,12 +95,15 @@ incoming traffics. See [Configuration guide](doc/configuration.md).
 - [x] [Use case 2: collect dnstap stream and get statistics usage with Prometheus/Grafana](https://dmachard.github.io/posts/0035-dnscollector-grafana-prometheus/)
 - [x] [Use case 3: collect dnstap stream and log dns answers in JSON format](https://dmachard.github.io/posts/0042-dnscollector-dnstap-json-answers/)
 
-## Openmetrics
-
+## Metrics
 
 | Metric | Description |
 | ---- | ---- | 
-| xx   | xx | 
+| dnscollector_qps   | Number of queries per second received | 
+| dnscollector_requesters_total | Number of clients |
+| dnscollector_domains_total | Number of domains observed |
+| dnscollector_received_bytes_total | Total bytes received |
+| dnscollector_sent_bytes_total | Total bytes sent |
 
 ## Benchmark
 

doc/swagger.yml

@@ -151,7 +151,38 @@ paths:
               schema:
                 type: string
       summary: Top suspicious domains list         
-
+  /dump/requesters:
+    get:
+      parameters:
+        - in: query
+          name: stream
+          schema:
+            type: string
+          description: stream name
+      responses:
+        '200':
+          description: Return full list of requesters
+          content:
+            text/plain:
+              schema:
+                type: string
+      summary: Return full list of requesters 
+  /dump/domains:
+    get:
+      parameters:
+        - in: query
+          name: stream
+          schema:
+            type: string
+          description: stream name
+      responses:
+        '200':
+          description: Return full list of domains
+          content:
+            text/plain:
+              schema:
+                type: string
+      summary: Return full list of domains
 security: []
 externalDocs:
   url: 'https://github.com/dmachard/go-dnscollector'

loggers/webserver.go

@@ -374,6 +374,48 @@ func (s *Webserver) metricsHandler(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func (s *Webserver) dumpRequestersHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.BasicAuth(w, r) {
+		http.Error(w, "Not authorized", http.StatusUnauthorized)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+
+	switch r.Method {
+	case http.MethodGet:
+		stream, ok := r.URL.Query()["stream"]
+		if !ok || len(stream) < 1 {
+			stream = []string{"global"}
+		}
+		t := s.stats.GetClients(stream[0])
+		json.NewEncoder(w).Encode(t)
+	default:
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *Webserver) dumpDomainsHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.BasicAuth(w, r) {
+		http.Error(w, "Not authorized", http.StatusUnauthorized)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+
+	switch r.Method {
+	case http.MethodGet:
+		stream, ok := r.URL.Query()["stream"]
+		if !ok || len(stream) < 1 {
+			stream = []string{"global"}
+		}
+		t := s.stats.GetDomains(stream[0])
+		json.NewEncoder(w).Encode(t)
+	default:
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
 func (s *Webserver) topRequestersHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.BasicAuth(w, r) {
 		http.Error(w, "Not authorized", http.StatusUnauthorized)
@@ -527,6 +569,7 @@ func (s *Webserver) ListenAndServe() {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/metrics", s.metricsHandler)
 	mux.HandleFunc("/reset", s.resetHandler)
+
 	mux.HandleFunc("/top/requesters", s.topRequestersHandler)
 	mux.HandleFunc("/top/requesters/suspicious", s.topSuspiciousClientsHandler)
 	mux.HandleFunc("/top/firstleveldomains", s.topAllFirstLevelDomainsHandler)
@@ -535,6 +578,9 @@ func (s *Webserver) ListenAndServe() {
 	mux.HandleFunc("/top/domains/slow", s.topSlowDomainsHandler)
 	mux.HandleFunc("/top/domains/suspicious", s.topSuspiciousDomainsHandler)
 
+	mux.HandleFunc("/dump/requesters", s.dumpRequestersHandler)
+	mux.HandleFunc("/dump/domains", s.dumpDomainsHandler)
+
 	var err error
 	var listener net.Listener
 	addrlisten := s.config.Loggers.WebServer.ListenIP + ":" + strconv.Itoa(s.config.Loggers.WebServer.ListenPort)

subprocessors/statistics.go

@@ -300,3 +300,27 @@ func (c *StatsStreams) GetTopIpProto(identity string) (ret []topmap.TopMapItem)
 
 	return v.GetTopIpProto()
 }
+
+func (c *StatsStreams) GetClients(identity string) (ret map[string]int) {
+	c.RLock()
+	defer c.RUnlock()
+
+	v, found := c.streams[identity]
+	if !found {
+		return map[string]int{}
+	}
+
+	return v.GetClients()
+}
+
+func (c *StatsStreams) GetDomains(identity string) (ret map[string]int) {
+	c.RLock()
+	defer c.RUnlock()
+
+	v, found := c.streams[identity]
+	if !found {
+		return map[string]int{}
+	}
+
+	return v.GetDomains()
+}

subprocessors/statsperstream.go

@@ -62,7 +62,8 @@ type Counters struct {
 type StatsPerStream struct {
 	config *dnsutils.Config
 
-	total                Counters
+	total Counters
+
 	firstleveldomains    map[string]int
 	firstleveldomainstop *topmap.TopMap
 	qnames               map[string]int
@@ -87,7 +88,8 @@ type StatsPerStream struct {
 	transportstop        *topmap.TopMap
 	ipproto              map[string]int
 	ipprototop           *topmap.TopMap
-	commonQtypes         map[string]bool
+
+	commonQtypes map[string]bool
 	sync.RWMutex
 }
 
@@ -634,3 +636,27 @@ func (c *StatsPerStream) GetTopIpProto() (ret []topmap.TopMapItem) {
 
 	return c.ipprototop.Get()
 }
+
+func (c *StatsPerStream) GetClients() (ret map[string]int) {
+	c.RLock()
+	defer c.RUnlock()
+
+	retMap := map[string]int{}
+	for k, v := range c.clients {
+		retMap[k] = v
+	}
+
+	return retMap
+}
+
+func (c *StatsPerStream) GetDomains() (ret map[string]int) {
+	c.RLock()
+	defer c.RUnlock()
+
+	retMap := map[string]int{}
+	for k, v := range c.qnames {
+		retMap[k] = v
+	}
+
+	return retMap
+}