From 684fa6c2cc957915c3c609f8c97f351196b34cd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20PIEL?= Date: Thu, 6 Feb 2020 09:59:42 +0100 Subject: [PATCH] Add allowAllHostsMetrics option --- cmd/nginx/flags.go | 3 + cmd/nginx/main.go | 2 +- internal/ingress/controller/controller.go | 5 +- internal/ingress/metric/collectors/socket.go | 8 +- .../ingress/metric/collectors/socket_test.go | 150 +++++++++++++++++- internal/ingress/metric/main.go | 4 +- 6 files changed, 164 insertions(+), 8 deletions(-) diff --git a/cmd/nginx/flags.go b/cmd/nginx/flags.go index 54a52abe07..db6fef26d7 100644 --- a/cmd/nginx/flags.go +++ b/cmd/nginx/flags.go @@ -152,6 +152,8 @@ Requires the update-status parameter.`) `Enables the collection of NGINX metrics`) metricsPerHost = flags.Bool("metrics-per-host", true, `Export metrics per-host`) + allowAllHostsMetrics = flags.Bool("allow-all-hosts-metrics", false, + `Allow collecting metrics for ingresses with no specified host`) httpPort = flags.Int("http-port", 80, `Port to use for servicing HTTP traffic.`) httpsPort = flags.Int("https-port", 443, `Port to use for servicing HTTPS traffic.`) @@ -270,6 +272,7 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g EnableProfiling: *profiling, EnableMetrics: *enableMetrics, MetricsPerHost: *metricsPerHost, + AllowAllHostsMetrics: *allowAllHostsMetrics, EnableSSLPassthrough: *enableSSLPassthrough, ResyncPeriod: *resyncPeriod, DefaultService: *defaultSvc, diff --git a/cmd/nginx/main.go b/cmd/nginx/main.go index eed0485ade..4497c60fa2 100644 --- a/cmd/nginx/main.go +++ b/cmd/nginx/main.go @@ -118,7 +118,7 @@ func main() { mc := metric.NewDummyCollector() if conf.EnableMetrics { - mc, err = metric.NewCollector(conf.MetricsPerHost, reg) + mc, err = metric.NewCollector(conf.MetricsPerHost, conf.AllowAllHostsMetrics, reg) if err != nil { klog.Fatalf("Error creating prometheus collector: %v", err) } diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index 8634e5e072..48eec2287b 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -85,8 +85,9 @@ type Configuration struct { EnableProfiling bool - EnableMetrics bool - MetricsPerHost bool + EnableMetrics bool + MetricsPerHost bool + AllowAllHostsMetrics bool FakeCertificate *ingress.SSLCert diff --git a/internal/ingress/metric/collectors/socket.go b/internal/ingress/metric/collectors/socket.go index dcf6d35e53..4f1cd7034f 100644 --- a/internal/ingress/metric/collectors/socket.go +++ b/internal/ingress/metric/collectors/socket.go @@ -79,6 +79,8 @@ type SocketCollector struct { hosts sets.String metricsPerHost bool + + allowAllHostsMetrics bool } var ( @@ -100,7 +102,7 @@ var defObjectives = map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001} // NewSocketCollector creates a new SocketCollector instance using // the ingress watch namespace and class used by the controller -func NewSocketCollector(pod, namespace, class string, metricsPerHost bool) (*SocketCollector, error) { +func NewSocketCollector(pod, namespace, class string, metricsPerHost bool, allowAllHostsMetrics bool) (*SocketCollector, error) { socket := "/tmp/prometheus-nginx.socket" // unix sockets must be unlink()ed before being used _ = syscall.Unlink(socket) @@ -131,6 +133,8 @@ func NewSocketCollector(pod, namespace, class string, metricsPerHost bool) (*Soc metricsPerHost: metricsPerHost, + allowAllHostsMetrics: allowAllHostsMetrics, + responseTime: prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "response_duration_seconds", @@ -230,7 +234,7 @@ func (sc *SocketCollector) handleMessage(msg []byte) { } for _, stats := range statsBatch { - if !sc.hosts.Has(stats.Host) { + if !sc.hosts.Has(stats.Host) && !sc.allowAllHostsMetrics { klog.V(3).Infof("skiping metric for host %v that is not being served", stats.Host) continue } diff --git a/internal/ingress/metric/collectors/socket_test.go b/internal/ingress/metric/collectors/socket_test.go index b9b3c68482..7a9a8335e4 100644 --- a/internal/ingress/metric/collectors/socket_test.go +++ b/internal/ingress/metric/collectors/socket_test.go @@ -288,7 +288,7 @@ func TestCollector(t *testing.T) { t.Run(c.name, func(t *testing.T) { registry := prometheus.NewPedanticRegistry() - sc, err := NewSocketCollector("pod", "default", "ingress", true) + sc, err := NewSocketCollector("pod", "default", "ingress", true, false) if err != nil { t.Errorf("%v: unexpected error creating new SocketCollector: %v", c.name, err) } @@ -322,3 +322,151 @@ func TestCollector(t *testing.T) { }) } } + +func TestCollectorAllowAllHostMetrics(t *testing.T) { + cases := []struct { + name string + data []string + metrics []string + wantBefore string + wantAfter string + allowAllHostMetrics bool + }{ + { + name: "ingress without host should not update prometheus metrics when allow all hosts metrics disabled", + data: []string{`[{ + "host":"testshop.com", + "status":"200", + "bytesSent":150.0, + "method":"GET", + "path":"/admin", + "requestLength":300.0, + "requestTime":60.0, + "upstreamName":"test-upstream", + "upstreamIP":"1.1.1.1:8080", + "upstreamResponseTime":200, + "upstreamStatus":"220", + "namespace":"test-app-production", + "ingress":"web-yml", + "service":"test-app" + }]`}, + metrics: []string{"nginx_ingress_controller_response_duration_seconds"}, + wantBefore: ` + `, + wantAfter: ` + `, + allowAllHostMetrics: false, + }, + + { + name: "ingress rules with host should increase prometheus metric when allow all hosts metrics enabled", + data: []string{`[{ + "host":"testshop.com", + "status":"200", + "bytesSent":150.0, + "method":"GET", + "path":"/admin", + "requestLength":300.0, + "requestTime":60.0, + "upstreamName":"test-upstream", + "upstreamIP":"1.1.1.1:8080", + "upstreamResponseTime":200, + "upstreamStatus":"220", + "namespace":"test-app-production", + "ingress":"web-yml", + "service":"test-app" + }]`, `[{ + "host":"testshop.com", + "status":"200", + "bytesSent":150.0, + "method":"GET", + "path":"/admin", + "requestLength":300.0, + "requestTime":60.0, + "upstreamName":"test-upstream", + "upstreamIP":"1.1.1.1:8080", + "upstreamResponseTime":200, + "upstreamStatus":"220", + "namespace":"test-app-qa", + "ingress":"web-yml-qa", + "service":"test-app-qa" + }]`, `[{ + "host":"testshop.com", + "status":"200", + "bytesSent":150.0, + "method":"GET", + "path":"/admin", + "requestLength":300.0, + "requestTime":60.0, + "upstreamName":"test-upstream", + "upstreamIP":"1.1.1.1:8080", + "upstreamResponseTime":200, + "upstreamStatus":"220", + "namespace":"test-app-qa", + "ingress":"web-yml-qa", + "service":"test-app-qa" + }]`}, + metrics: []string{"nginx_ingress_controller_response_duration_seconds"}, + wantBefore: ` + # HELP nginx_ingress_controller_response_duration_seconds The time spent on receiving the response from the upstream server + # TYPE nginx_ingress_controller_response_duration_seconds histogram + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.005"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.01"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.025"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.05"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.1"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.25"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="1"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="2.5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="10"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="+Inf"} 1 + nginx_ingress_controller_response_duration_seconds_sum{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 200 + nginx_ingress_controller_response_duration_seconds_count{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 1 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.005"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.01"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.025"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.05"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.1"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.25"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="1"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="2.5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="10"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="+Inf"} 2 + nginx_ingress_controller_response_duration_seconds_sum{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200"} 400 + nginx_ingress_controller_response_duration_seconds_count{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200"} 2 + `, + allowAllHostMetrics: true, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + registry := prometheus.NewPedanticRegistry() + + sc, err := NewSocketCollector("pod", "default", "ingress", true, c.allowAllHostMetrics) + if err != nil { + t.Errorf("%v: unexpected error creating new SocketCollector: %v", c.name, err) + } + + if err := registry.Register(sc); err != nil { + t.Errorf("registering collector failed: %s", err) + } + + for _, d := range c.data { + sc.handleMessage([]byte(d)) + } + + if err := GatherAndCompare(sc, c.wantBefore, c.metrics, registry); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + sc.Stop() + + registry.Unregister(sc) + }) + } +} diff --git a/internal/ingress/metric/main.go b/internal/ingress/metric/main.go index ecc59c5b81..d2d0bba50b 100644 --- a/internal/ingress/metric/main.go +++ b/internal/ingress/metric/main.go @@ -66,7 +66,7 @@ type collector struct { } // NewCollector creates a new metric collector the for ingress controller -func NewCollector(metricsPerHost bool, registry *prometheus.Registry) (Collector, error) { +func NewCollector(metricsPerHost bool, allowAllHostsMetrics bool, registry *prometheus.Registry) (Collector, error) { podNamespace := os.Getenv("POD_NAMESPACE") if podNamespace == "" { podNamespace = "default" @@ -84,7 +84,7 @@ func NewCollector(metricsPerHost bool, registry *prometheus.Registry) (Collector return nil, err } - s, err := collectors.NewSocketCollector(podName, podNamespace, class.IngressClass, metricsPerHost) + s, err := collectors.NewSocketCollector(podName, podNamespace, class.IngressClass, metricsPerHost, allowAllHostsMetrics) if err != nil { return nil, err }