Merge pull request #548 from zhuoyuan-liu/ingested-metrics

Add metrics for size of ingested data
jmpsec · Oct 30, 2024 · 8e7065d · 8e7065d
2 parents a2cb1a7 + d92852d
commit 8e7065d
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 0 deletions.
diff --git a/tls/handlers/metrics.go b/tls/handlers/metrics.go
@@ -6,6 +6,8 @@ const (
 	RequestPath   = "path"
 	RequestMethod = "method"
 	StatusCode    = "status_code"
+	Environment   = "osctrl_env"
+	RequestType   = "type"
 )
 
 var (
@@ -14,8 +16,14 @@ var (
 		Help:    "The duration of requests",
 		Buckets: []float64{0.0005, 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.5, 1, 5},
 	}, []string{RequestMethod, RequestPath, StatusCode})
+	requestSize = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "osctrl_tls_request_size_bytes",
+		Help:    "The size of requests",
+		Buckets: []float64{100, 1000, 10000, 100000, 1000000},
+	}, []string{Environment, RequestType})
 )
 
 func RegisterMetrics(reg prometheus.Registerer) {
 	reg.MustRegister(requestDuration)
+	reg.MustRegister(requestSize)
 }
diff --git a/tls/handlers/post.go b/tls/handlers/post.go
@@ -162,6 +162,8 @@ func (h *HandlersTLS) ConfigHandler(w http.ResponseWriter, r *http.Request) {
 			h.Inc(metricConfigErr)
 			log.Err(err).Msg("error with ingested config")
 		}
+		requestSize.WithLabelValues(string(env.UUID), "ConfigHandler").Observe(float64(len(body)))
+		log.Info().Msgf("node UUID: %s in %s environment ingested %d bytes for ConfigHandler endpoint", node.UUID, env.Name, len(body))
 		response = []byte(env.Configuration)
 	} else {
 		response = types.ConfigResponse{NodeInvalid: true}
@@ -250,6 +252,8 @@ func (h *HandlersTLS) LogHandler(w http.ResponseWriter, r *http.Request) {
 			h.Inc(metricLogErr)
 			log.Err(err).Msg("error with ingested log")
 		}
+		requestSize.WithLabelValues(string(env.UUID), "LogHandler").Observe(float64(len(body)))
+		log.Info().Msgf("node UUID: %s in %s environment ingested %d bytes for LogHandler endpoint", node.UUID, env.Name, len(body))
 		// Process logs and update metadata
 		go h.Logs.ProcessLogs(t.Data, t.LogType, env.Name, utils.GetIP(r), len(body), (*h.EnvsMap)[env.Name].DebugHTTP)
 	} else {
@@ -310,6 +314,8 @@ func (h *HandlersTLS) QueryReadHandler(w http.ResponseWriter, r *http.Request) {
 			h.Inc(metricReadErr)
 			log.Err(err).Msg("error with ingested query-read")
 		}
+		requestSize.WithLabelValues(string(env.UUID), "QueryRead").Observe(float64(len(body)))
+		log.Info().Msgf("node UUID: %s in %s environment ingested %d bytes for QueryReadHandler endpoint", node.UUID, env.Name, len(body))
 		ip := utils.GetIP(r)
 		if err := h.Nodes.RecordIPAddress(ip, node); err != nil {
 			h.Inc(metricReadErr)
@@ -391,6 +397,8 @@ func (h *HandlersTLS) QueryWriteHandler(w http.ResponseWriter, r *http.Request)
 			h.Inc(metricWriteErr)
 			log.Err(err).Msg("error with ingested query-write")
 		}
+		requestSize.WithLabelValues(string(env.UUID), "QueryWrite").Observe(float64(len(body)))
+		log.Info().Msgf("node UUID: %s in %s environment ingested %d bytes for QueryWriteHandler endpoint", node.UUID, env.Name, len(body))
 		ip := utils.GetIP(r)
 		if err := h.Nodes.RecordIPAddress(ip, node); err != nil {
 			h.Inc(metricWriteErr)
@@ -630,6 +638,8 @@ func (h *HandlersTLS) CarveInitHandler(w http.ResponseWriter, r *http.Request) {
 			h.Inc(metricInitErr)
 			log.Err(err).Msg("error with ingested carve-init")
 		}
+		requestSize.WithLabelValues(string(env.UUID), "CarveInit").Observe(float64(len(body)))
+		log.Info().Msgf("node UUID: %s in %s environment ingested %d bytes for CarveInitHandler endpoint", node.UUID, env.Name, len(body))
 		ip := utils.GetIP(r)
 		if err := h.Nodes.RecordIPAddress(ip, node); err != nil {
 			h.Inc(metricInitErr)
@@ -703,6 +713,8 @@ func (h *HandlersTLS) CarveBlockHandler(w http.ResponseWriter, r *http.Request)
 			h.Inc(metricInitErr)
 			log.Err(err).Msg("error with ingested carve-block")
 		}
+		requestSize.WithLabelValues(string(env.UUID), "CarveBlock").Observe(float64(len(body)))
+		log.Info().Msgf("node %d in %s environment ingested %d bytes for CarveBlockHandler endpoint", carve.NodeID, env.Name, len(body))
 		blockCarve = true
 		// Process received block
 		go h.ProcessCarveBlock(t, env.Name, carve.UUID, env.ID)