Add upstream_connect metrics

martin-helmich · Apr 24, 2022 · 34735db · 34735db
1 parent f308c93
commit 34735db
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 8 deletions.
diff --git a/features/response_times.feature b/features/response_times.feature
@@ -19,6 +19,17 @@ Feature: Upstream response times are summarized
       """
     Then the exporter should report value 20 for metric nginx_http_upstream_time_seconds{method="GET",status="200",quantile="0.5"}
 
+  Scenario: .5 quantile of upstream connect time is computed
+    Given a running exporter listening on "access.log" with upstream-connect-time format
+    When the following HTTP request is logged to "access.log"
+      """
+      172.17.0.1 - - [23/Jun/2016:16:04:20 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-" 10 5
+      172.17.0.1 - - [23/Jun/2016:16:04:20 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-" 20 5
+      172.17.0.1 - - [23/Jun/2016:16:04:20 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-" 30 10
+      172.17.0.1 - - [23/Jun/2016:16:04:20 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-" 40 10
+      """
+    Then the exporter should report value 5 for metric nginx_http_upstream_connect_time_seconds{method="GET",status="200",quantile="0.5"}
+
   Scenario: .5 quantile of response time is computed
     Given a running exporter listening on "access.log" with request-time format
     When the following HTTP request is logged to "access.log"

diff --git a/features/steps/steps.py b/features/steps/steps.py
@@ -8,6 +8,7 @@
 formats = {
     "default": '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for"',
     "upstream-time": '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" $upstream_response_time',
+    "upstream-connect-time": '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" $upstream_response_time $upstream_connect_time',
     "request-time": '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" $request_time'
 }
 

diff --git a/main.go b/main.go
@@ -57,6 +57,8 @@ func NewNSMetrics(cfg *config.NamespaceConfig) *NSMetrics {
 	m.registry.MustRegister(m.responseBytesTotal)
 	m.registry.MustRegister(m.upstreamSeconds)
 	m.registry.MustRegister(m.upstreamSecondsHist)
+	m.registry.MustRegister(m.upstreamConnectSeconds)
+	m.registry.MustRegister(m.upstreamConnectSecondsHist)
 	m.registry.MustRegister(m.responseSeconds)
 	m.registry.MustRegister(m.responseSecondsHist)
 	m.registry.MustRegister(m.parseErrorsTotal)
@@ -66,14 +68,16 @@ func NewNSMetrics(cfg *config.NamespaceConfig) *NSMetrics {
 // Metrics is a struct containing pointers to all metrics that should be
 // exposed to Prometheus
 type Metrics struct {
-	countTotal          *prometheus.CounterVec
-	responseBytesTotal  *prometheus.CounterVec
-	requestBytesTotal   *prometheus.CounterVec
-	upstreamSeconds     *prometheus.SummaryVec
-	upstreamSecondsHist *prometheus.HistogramVec
-	responseSeconds     *prometheus.SummaryVec
-	responseSecondsHist *prometheus.HistogramVec
-	parseErrorsTotal    prometheus.Counter
+	countTotal                 *prometheus.CounterVec
+	responseBytesTotal         *prometheus.CounterVec
+	requestBytesTotal          *prometheus.CounterVec
+	upstreamSeconds            *prometheus.SummaryVec
+	upstreamSecondsHist        *prometheus.HistogramVec
+	upstreamConnectSeconds     *prometheus.SummaryVec
+	upstreamConnectSecondsHist *prometheus.HistogramVec
+	responseSeconds            *prometheus.SummaryVec
+	responseSecondsHist        *prometheus.HistogramVec
+	parseErrorsTotal           prometheus.Counter
 }
 
 func inLabels(label string, labels []string) bool {
@@ -145,6 +149,22 @@ func (m *Metrics) Init(cfg *config.NamespaceConfig) {
 		Buckets:     cfg.HistogramBuckets,
 	}, labels)
 
+	m.upstreamConnectSeconds = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+		Namespace:   cfg.NamespacePrefix,
+		ConstLabels: cfg.NamespaceLabels,
+		Name:        "http_upstream_connect_time_seconds",
+		Help:        "Time needed to connect to upstream servers",
+		Objectives:  map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+	}, labels)
+
+	m.upstreamConnectSecondsHist = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   cfg.NamespacePrefix,
+		ConstLabels: cfg.NamespaceLabels,
+		Name:        "http_upstream_connect_time_seconds_hist",
+		Help:        "Time needed to connect to upstream servers",
+		Buckets:     cfg.HistogramBuckets,
+	}, labels)
+
 	m.responseSeconds = prometheus.NewSummaryVec(prometheus.SummaryOpts{
 		Namespace:   cfg.NamespacePrefix,
 		ConstLabels: cfg.NamespaceLabels,
@@ -412,6 +432,11 @@ func processSource(nsCfg config.NamespaceConfig, t tail.Follower, parser parser.
 			metrics.upstreamSecondsHist.WithLabelValues(notCounterValues...).Observe(v)
 		}
 
+		if v, ok := observeMetrics(fields, "upstream_connect_time", floatFromFieldsMulti, metrics.parseErrorsTotal); ok {
+			metrics.upstreamConnectSeconds.WithLabelValues(notCounterValues...).Observe(v)
+			metrics.upstreamConnectSecondsHist.WithLabelValues(notCounterValues...).Observe(v)
+		}
+
 		if v, ok := observeMetrics(fields, "request_time", floatFromFields, metrics.parseErrorsTotal); ok {
 			metrics.responseSeconds.WithLabelValues(notCounterValues...).Observe(v)
 			metrics.responseSecondsHist.WithLabelValues(notCounterValues...).Observe(v)