From b8bbbdb0f9e4334cf739934d1277d138a438d32a Mon Sep 17 00:00:00 2001 From: Raul Marrero Date: Mon, 8 Apr 2019 15:41:22 +0100 Subject: [PATCH] Add Ingress Controller Prometheus metrics --- cmd/nginx-ingress/main.go | 55 +++++--- docs/installation.md | 4 +- docs/prometheus.md | 20 +++ internal/configs/configurator.go | 24 ++++ internal/k8s/controller.go | 17 ++- internal/k8s/controller_test.go | 44 +++--- internal/k8s/utils.go | 2 +- internal/metrics/collectors/collectors.go | 3 + internal/metrics/collectors/controller.go | 66 +++++++++ internal/metrics/collectors/manager.go | 126 ++++++++++++++++++ .../metrics/{prometheus.go => listener.go} | 12 +- internal/nginx/manager.go | 15 ++- 12 files changed, 344 insertions(+), 44 deletions(-) create mode 100644 docs/prometheus.md create mode 100644 internal/metrics/collectors/collectors.go create mode 100644 internal/metrics/collectors/controller.go create mode 100644 internal/metrics/collectors/manager.go rename internal/metrics/{prometheus.go => listener.go} (83%) diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index 5a9ea2b645..70bb1f684b 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -12,6 +12,8 @@ import ( "syscall" "time" + "github.com/nginxinc/kubernetes-ingress/internal/metrics/collectors" + "github.com/golang/glog" "github.com/nginxinc/kubernetes-ingress/internal/configs" "github.com/nginxinc/kubernetes-ingress/internal/configs/version1" @@ -19,6 +21,7 @@ import ( "github.com/nginxinc/kubernetes-ingress/internal/metrics" "github.com/nginxinc/kubernetes-ingress/internal/nginx" "github.com/nginxinc/nginx-plus-go-sdk/client" + "github.com/prometheus/client_golang/prometheus" api_v1 "k8s.io/api/core/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -187,13 +190,34 @@ func main() { glog.Fatalf("Error creating TemplateExecutor: %v", err) } - useFakeNginxManager := *proxyURL != "" + var registry *prometheus.Registry + var managerCollector collectors.ManagerCollector + var controllerCollector collectors.ControllerCollector + managerCollector = collectors.NewManagerFakeCollector() + controllerCollector = collectors.NewControllerFakeCollector() + if *enablePrometheusMetrics { + registry = prometheus.NewRegistry() + managerCollector = collectors.NewLocalManagerMetricsCollector() + controllerCollector = collectors.NewControllerMetricsCollector() + + err = managerCollector.Register(registry) + if err != nil { + glog.Errorf("Error registering Manager Prometheus metrics: %v", err) + } + + err = controllerCollector.Register(registry) + if err != nil { + glog.Errorf("Error registering Controller Prometheus metrics: %v", err) + } + } + + useFakeNginxManager := *proxyURL != "" var nginxManager nginx.Manager if useFakeNginxManager { nginxManager = nginx.NewFakeManager("/etc/nginx") } else { - nginxManager = nginx.NewLocalManager("/etc/nginx/", nginxBinaryPath) + nginxManager = nginx.NewLocalManager("/etc/nginx/", nginxBinaryPath, managerCollector) } if *defaultServerSecret != "" { @@ -283,6 +307,19 @@ func main() { nginxManager.SetPlusClients(plusClient, httpClient) } + if *enablePrometheusMetrics { + if *nginxPlus { + go metrics.RunPrometheusListenerForNginxPlus(*prometheusMetricsListenPort, plusClient, registry) + } else { + httpClient := getSocketClient("/var/run/nginx-status.sock") + client, err := metrics.NewNginxMetricsClient(httpClient) + if err != nil { + glog.Fatalf("Error creating the Nginx client for Prometheus metrics: %v", err) + } + go metrics.RunPrometheusListenerForNginx(*prometheusMetricsListenPort, client, registry) + } + } + isWildcardEnabled := *wildcardTLSSecret != "" cnf := configs.NewConfigurator(nginxManager, staticCfgParams, cfgParams, templateExecutor, *nginxPlus, isWildcardEnabled) controllerNamespace := os.Getenv("POD_NAMESPACE") @@ -303,23 +340,11 @@ func main() { LeaderElectionLockName: *leaderElectionLockName, WildcardTLSSecret: *wildcardTLSSecret, ConfigMaps: *nginxConfigMaps, + MetricsCollector: controllerCollector, } lbc := k8s.NewLoadBalancerController(lbcInput) - if *enablePrometheusMetrics { - if *nginxPlus { - go metrics.RunPrometheusListenerForNginxPlus(*prometheusMetricsListenPort, plusClient) - } else { - httpClient := getSocketClient("/var/run/nginx-status.sock") - client, err := metrics.NewNginxMetricsClient(httpClient) - if err != nil { - glog.Fatalf("Error creating the Nginx client for Prometheus metrics: %v", err) - } - go metrics.RunPrometheusListenerForNginx(*prometheusMetricsListenPort, client) - } - } - go handleTermination(lbc, nginxManager, nginxDone) lbc.Run() diff --git a/docs/installation.md b/docs/installation.md index a16fdf792c..bcdaefe8ff 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -165,7 +165,7 @@ For NGINX Plus, you can access the live activity monitoring dashboard: ## Support For Prometheus Monitoring -You can expose NGINX or NGINX Plus metrics for collection by [Prometheus](https://prometheus.io/): +You can expose NGINX/NGINX Plus and Ingress Controller [metrics](./prometheus.md) for collection by [Prometheus](https://prometheus.io/): 1. Run the Ingress controller with the `-enable-prometheus-metrics` [command-line argument](cli-arguments.md). As a result, the Ingress Controller will expose NGINX or NGINX Plus metrics in the Prometheus format via the path `/metrics` on port `9113` (customizable via the `-prometheus-metrics-listen-port` command-line argument). 1. Add the Prometheus port to the list of the ports of the Ingress Controller container: @@ -173,7 +173,7 @@ You can expose NGINX or NGINX Plus metrics for collection by [Prometheus](https: - name: prometheus containerPort: 9113 ``` -1. Make the Prometheus aware of the Ingress Controller targets by adding the following annotations to the template of the Ingress Controller pod (note: this assumes your Prometheus is configured to discover targets by analyzing the annotations of pods): +1. Make Prometheus aware of the Ingress Controller targets by adding the following annotations to the template of the Ingress Controller pod (note: this assumes your Prometheus is configured to discover targets by analyzing the annotations of pods): ```yaml annotations: prometheus.io/scrape: "true" diff --git a/docs/prometheus.md b/docs/prometheus.md new file mode 100644 index 0000000000..88ab0ff629 --- /dev/null +++ b/docs/prometheus.md @@ -0,0 +1,20 @@ +# Monitoring the Ingress Controller Using Prometheus + +The Ingress Controller exposes a number of metrics in the [Prometheus](https://prometheus.io/) format. Those include NGINX/NGINX Plus and the Ingress Controller metrics. + +## Enabling Metrics +To enable Prometheus metrics, follow the [Support For Prometheus Monitoring](./installation.md#support-for-prometheus-monitoring) section of the installation doc. Once enabled, the metrics will be available via the configured endpoint. + +## Available Metrics +The Ingress Controller exports the following metrics: + +* NGINX/NGINX Plus metrics. Please see this [doc](https://github.com/nginxinc/nginx-prometheus-exporter#exported-metrics) to find more information about the exported metrics. + +* Ingress Controller metrics + * `controller_nginx_reloads_total`. Number of successful NGINX reloads. + * `controller_nginx_reload_errors_total`. Number of unsuccessful NGINX reloads. + * `controller_nginx_last_reload_status`. Status of the last NGINX reload, 0 meaning down and 1 up. + * `controller_nginx_last_reload_milliseconds`. Duration in milliseconds of the last NGINX reload. + * `controller_ingress_resources_total`. Number of handled Ingress resources. This metric includes the label type, that groups the Ingress resources by their type (regular, [minion or master](./../examples/mergeable-ingress-types)) + +**Note**: all metrics have the namespace nginx_ingress. For example, nginx_ingress_controller_nginx_reloads_total. diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 6bb79ad52e..6b767d0717 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -469,3 +469,27 @@ func (cnf *Configurator) HasMinion(master *extensions.Ingress, minion *extension func (cnf *Configurator) IsResolverConfigured() bool { return len(cnf.cfgParams.ResolverAddresses) != 0 } + +// GetIngressCounts returns the total count of Ingress resources that are handled by the Ingress Controller grouped by their type +func (cnf *Configurator) GetIngressCounts() map[string]int { + counters := map[string]int{ + "master": 0, + "regular": 0, + "minion": 0, + } + + // cnf.ingresses contains only master and regular Ingress Resources + for _, ing := range cnf.ingresses { + if ing.Ingress.Annotations["nginx.org/mergeable-ingress-type"] == "master" { + counters["master"]++ + } else { + counters["regular"]++ + } + } + + for _, min := range cnf.minions { + counters["minion"] += len(min) + } + + return counters +} diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index cac9ce8198..779fa3e609 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -24,6 +24,7 @@ import ( "github.com/golang/glog" "github.com/nginxinc/kubernetes-ingress/internal/configs" + "github.com/nginxinc/kubernetes-ingress/internal/metrics/collectors" "k8s.io/api/extensions/v1beta1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" @@ -79,6 +80,7 @@ type LoadBalancerController struct { namespace string controllerNamespace string wildcardTLSSecret string + metricsCollector collectors.ControllerCollector } var keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc @@ -100,6 +102,7 @@ type NewLoadBalancerControllerInput struct { LeaderElectionLockName string WildcardTLSSecret string ConfigMaps string + MetricsCollector collectors.ControllerCollector } // NewLoadBalancerController creates a controller @@ -118,6 +121,7 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc namespace: input.Namespace, controllerNamespace: input.ControllerNamespace, wildcardTLSSecret: input.WildcardTLSSecret, + metricsCollector: input.MetricsCollector, } eventBroadcaster := record.NewBroadcaster() @@ -456,20 +460,20 @@ func (lbc *LoadBalancerController) sync(task task) { switch task.Kind { case ingress: lbc.syncIng(task) + lbc.updateIngressMetrics() case ingressMinion: lbc.syncIngMinion(task) + lbc.updateIngressMetrics() case configMap: lbc.syncConfig(task) - return case endpoints: lbc.syncEndpoint(task) - return case secret: lbc.syncSecret(task) - return case service: lbc.syncExternalService(task) } + } func (lbc *LoadBalancerController) syncIngMinion(task task) { @@ -591,6 +595,13 @@ func (lbc *LoadBalancerController) syncIng(task task) { } } +func (lbc *LoadBalancerController) updateIngressMetrics() { + counters := lbc.configurator.GetIngressCounts() + for nType, count := range counters { + lbc.metricsCollector.SetIngressResources(nType, count) + } +} + // syncExternalService does not sync all services. // We only watch the Service specified by the external-service flag. func (lbc *LoadBalancerController) syncExternalService(task task) { diff --git a/internal/k8s/controller_test.go b/internal/k8s/controller_test.go index 3dd239fa2c..89b3097134 100644 --- a/internal/k8s/controller_test.go +++ b/internal/k8s/controller_test.go @@ -7,10 +7,12 @@ import ( "time" "unsafe" + "github.com/nginxinc/kubernetes-ingress/internal/metrics/collectors" + "github.com/nginxinc/kubernetes-ingress/internal/configs" "github.com/nginxinc/kubernetes-ingress/internal/configs/version1" "github.com/nginxinc/kubernetes-ingress/internal/nginx" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -31,6 +33,7 @@ func TestIsNginxIngress(t *testing.T) { &LoadBalancerController{ ingressClass: ingressClass, useIngressClassOnly: false, + metricsCollector: collectors.NewControllerFakeCollector(), }, &extensions.Ingress{ ObjectMeta: meta_v1.ObjectMeta{ @@ -43,6 +46,7 @@ func TestIsNginxIngress(t *testing.T) { &LoadBalancerController{ ingressClass: ingressClass, useIngressClassOnly: false, + metricsCollector: collectors.NewControllerFakeCollector(), }, &extensions.Ingress{ ObjectMeta: meta_v1.ObjectMeta{ @@ -55,6 +59,7 @@ func TestIsNginxIngress(t *testing.T) { &LoadBalancerController{ ingressClass: ingressClass, useIngressClassOnly: false, + metricsCollector: collectors.NewControllerFakeCollector(), }, &extensions.Ingress{ ObjectMeta: meta_v1.ObjectMeta{ @@ -67,6 +72,7 @@ func TestIsNginxIngress(t *testing.T) { &LoadBalancerController{ ingressClass: ingressClass, useIngressClassOnly: false, + metricsCollector: collectors.NewControllerFakeCollector(), }, &extensions.Ingress{ ObjectMeta: meta_v1.ObjectMeta{ @@ -86,6 +92,7 @@ func TestIsNginxIngress(t *testing.T) { &LoadBalancerController{ ingressClass: ingressClass, useIngressClassOnly: true, + metricsCollector: collectors.NewControllerFakeCollector(), }, &extensions.Ingress{ ObjectMeta: meta_v1.ObjectMeta{ @@ -98,6 +105,7 @@ func TestIsNginxIngress(t *testing.T) { &LoadBalancerController{ ingressClass: ingressClass, useIngressClassOnly: true, + metricsCollector: collectors.NewControllerFakeCollector(), }, &extensions.Ingress{ ObjectMeta: meta_v1.ObjectMeta{ @@ -110,6 +118,7 @@ func TestIsNginxIngress(t *testing.T) { &LoadBalancerController{ ingressClass: ingressClass, useIngressClassOnly: true, + metricsCollector: collectors.NewControllerFakeCollector(), }, &extensions.Ingress{ ObjectMeta: meta_v1.ObjectMeta{ @@ -122,6 +131,7 @@ func TestIsNginxIngress(t *testing.T) { &LoadBalancerController{ ingressClass: ingressClass, useIngressClassOnly: true, + metricsCollector: collectors.NewControllerFakeCollector(), }, &extensions.Ingress{ ObjectMeta: meta_v1.ObjectMeta{ @@ -634,9 +644,10 @@ func getMergableDefaults() (cafeMaster, coffeeMinion, teaMinion extensions.Ingre fakeClient := fake.NewSimpleClientset() lbc = LoadBalancerController{ - client: fakeClient, - ingressClass: "nginx", - configurator: cnf, + client: fakeClient, + ingressClass: "nginx", + configurator: cnf, + metricsCollector: collectors.NewControllerFakeCollector(), } lbc.svcLister, _ = cache.NewInformer( cache.NewListWatchFromClient(lbc.client.ExtensionsV1beta1().RESTClient(), "services", "default", fields.Everything()), @@ -836,9 +847,10 @@ func TestGetServicePortForIngressPort(t *testing.T) { fakeClient := fake.NewSimpleClientset() cnf := configs.NewConfigurator(&nginx.LocalManager{}, &configs.StaticConfigParams{}, &configs.ConfigParams{}, &version1.TemplateExecutor{}, false, false) lbc := LoadBalancerController{ - client: fakeClient, - ingressClass: "nginx", - configurator: cnf, + client: fakeClient, + ingressClass: "nginx", + configurator: cnf, + metricsCollector: collectors.NewControllerFakeCollector(), } svc := v1.Service{ TypeMeta: meta_v1.TypeMeta{}, @@ -986,10 +998,11 @@ func TestFindIngressesForSecret(t *testing.T) { cnf := configs.NewConfigurator(manager, &configs.StaticConfigParams{}, &configs.ConfigParams{}, templateExecutor, false, false) lbc := LoadBalancerController{ - client: fakeClient, - ingressClass: "nginx", - configurator: cnf, - isNginxPlus: true, + client: fakeClient, + ingressClass: "nginx", + configurator: cnf, + isNginxPlus: true, + metricsCollector: collectors.NewControllerFakeCollector(), } lbc.ingressLister.Store, _ = cache.NewInformer( @@ -1170,10 +1183,11 @@ func TestFindIngressesForSecretWithMinions(t *testing.T) { cnf := configs.NewConfigurator(manager, &configs.StaticConfigParams{}, &configs.ConfigParams{}, templateExecutor, false, false) lbc := LoadBalancerController{ - client: fakeClient, - ingressClass: "nginx", - configurator: cnf, - isNginxPlus: true, + client: fakeClient, + ingressClass: "nginx", + configurator: cnf, + isNginxPlus: true, + metricsCollector: collectors.NewControllerFakeCollector(), } lbc.ingressLister.Store, _ = cache.NewInformer( diff --git a/internal/k8s/utils.go b/internal/k8s/utils.go index 813593db55..ce9bddc6fe 100644 --- a/internal/k8s/utils.go +++ b/internal/k8s/utils.go @@ -21,7 +21,7 @@ import ( "reflect" "strings" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/api/extensions/v1beta1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/cache" diff --git a/internal/metrics/collectors/collectors.go b/internal/metrics/collectors/collectors.go new file mode 100644 index 0000000000..037a1d7037 --- /dev/null +++ b/internal/metrics/collectors/collectors.go @@ -0,0 +1,3 @@ +package collectors + +const metricsNamespace = "nginx_ingress_controller" diff --git a/internal/metrics/collectors/controller.go b/internal/metrics/collectors/controller.go new file mode 100644 index 0000000000..52a68da448 --- /dev/null +++ b/internal/metrics/collectors/controller.go @@ -0,0 +1,66 @@ +package collectors + +import "github.com/prometheus/client_golang/prometheus" + +var labelNamesController = []string{"type"} + +// ControllerCollector is an interface for the metrics of the Controller +type ControllerCollector interface { + SetIngressResources(ingressType string, count int) + Register(registry *prometheus.Registry) error +} + +// ControllerMetricsCollector implements the ControllerCollector interface and prometheus.Collector interface +type ControllerMetricsCollector struct { + ingressResourcesTotal *prometheus.GaugeVec +} + +// NewControllerMetricsCollector creates a new ControllerMetricsCollector +func NewControllerMetricsCollector() *ControllerMetricsCollector { + cc := &ControllerMetricsCollector{ + ingressResourcesTotal: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "ingress_resources_total", + Namespace: metricsNamespace, + Help: "Number of handled ingress resources", + }, + labelNamesController, + ), + } + + return cc +} + +// SetIngressResources sets the value of the ingress resources gauge for a given type +func (cc *ControllerMetricsCollector) SetIngressResources(ingressType string, count int) { + cc.ingressResourcesTotal.WithLabelValues(ingressType).Set(float64(count)) +} + +// Describe implements prometheus.Collector interface Describe method +func (cc *ControllerMetricsCollector) Describe(ch chan<- *prometheus.Desc) { + cc.ingressResourcesTotal.Describe(ch) +} + +// Collect implements the prometheus.Collector interface Collect method +func (cc *ControllerMetricsCollector) Collect(ch chan<- prometheus.Metric) { + cc.ingressResourcesTotal.Collect(ch) +} + +// Register registers all the metrics of the collector +func (cc *ControllerMetricsCollector) Register(registry *prometheus.Registry) error { + return registry.Register(cc) +} + +// ControllerFakeCollector is a fake collector that implements the ControllerCollector interface +type ControllerFakeCollector struct{} + +// NewControllerFakeCollector creates a fake collector that implements the ControllerCollector interface +func NewControllerFakeCollector() *ControllerFakeCollector { + return &ControllerFakeCollector{} +} + +// Register implements a fake Register +func (cc *ControllerFakeCollector) Register(registry *prometheus.Registry) error { return nil } + +// SetIngressResources implements a fake SetIngressResources +func (cc *ControllerFakeCollector) SetIngressResources(ingressType string, count int) {} diff --git a/internal/metrics/collectors/manager.go b/internal/metrics/collectors/manager.go new file mode 100644 index 0000000000..04e8812f37 --- /dev/null +++ b/internal/metrics/collectors/manager.go @@ -0,0 +1,126 @@ +package collectors + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// ManagerCollector is an interface for the metrics of the Nginx Manager +type ManagerCollector interface { + IncNginxReloadCount() + IncNginxReloadErrors() + UpdateLastReloadTime(ms time.Duration) + Register(registry *prometheus.Registry) error +} + +// LocalManagerMetricsCollector implements NginxManagerCollector interface and prometheus.Collector interface +type LocalManagerMetricsCollector struct { + // Metrics + reloadsTotal prometheus.Counter + reloadsError prometheus.Counter + lastReloadStatus prometheus.Gauge + lastReloadTime prometheus.Gauge +} + +// NewLocalManagerMetricsCollector creates a new LocalManagerMetricsCollector +func NewLocalManagerMetricsCollector() *LocalManagerMetricsCollector { + nc := &LocalManagerMetricsCollector{ + reloadsTotal: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "nginx_reloads_total", + Namespace: metricsNamespace, + Help: "Number of successful NGINX reloads", + }, + ), + reloadsError: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "nginx_reload_errors_total", + Namespace: metricsNamespace, + Help: "Number of unsuccessful NGINX reloads", + }, + ), + lastReloadStatus: prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "nginx_last_reload_status", + Namespace: metricsNamespace, + Help: "Status of the last NGINX reload", + }, + ), + lastReloadTime: prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "nginx_last_reload_milliseconds", + Namespace: metricsNamespace, + Help: "Duration in milliseconds of the last NGINX reload", + }, + ), + } + return nc +} + +// IncNginxReloadCount increments the counter of successful NGINX reloads and sets the last reload status to true +func (nc *LocalManagerMetricsCollector) IncNginxReloadCount() { + nc.reloadsTotal.Inc() + nc.updateLastReloadStatus(true) +} + +// IncNginxReloadErrors increments the counter of NGINX reload errors and sets the last reload status to false +func (nc *LocalManagerMetricsCollector) IncNginxReloadErrors() { + nc.reloadsError.Inc() + nc.updateLastReloadStatus(false) +} + +// updateLastReloadStatus updates the last NGINX reload status metric +func (nc *LocalManagerMetricsCollector) updateLastReloadStatus(up bool) { + var status float64 + if up { + status = 1.0 + } + nc.lastReloadStatus.Set(status) +} + +// UpdateLastReloadTime updates the last NGINX reload time +func (nc *LocalManagerMetricsCollector) UpdateLastReloadTime(duration time.Duration) { + nc.lastReloadTime.Set(float64(duration / time.Millisecond)) +} + +// Describe implements prometheus.Collector interface Describe method +func (nc *LocalManagerMetricsCollector) Describe(ch chan<- *prometheus.Desc) { + nc.reloadsTotal.Describe(ch) + nc.reloadsError.Describe(ch) + nc.lastReloadStatus.Describe(ch) + nc.lastReloadTime.Describe(ch) +} + +// Collect implements the prometheus.Collector interface Collect method +func (nc *LocalManagerMetricsCollector) Collect(ch chan<- prometheus.Metric) { + nc.reloadsTotal.Collect(ch) + nc.reloadsError.Collect(ch) + nc.lastReloadStatus.Collect(ch) + nc.lastReloadTime.Collect(ch) +} + +// Register registers all the metrics of the collector +func (nc *LocalManagerMetricsCollector) Register(registry *prometheus.Registry) error { + return registry.Register(nc) +} + +// ManagerFakeCollector is a fake collector that will implement ManagerCollector interface +type ManagerFakeCollector struct{} + +// NewManagerFakeCollector creates a fake collector that implements ManagerCollector interface +func NewManagerFakeCollector() *ManagerFakeCollector { + return &ManagerFakeCollector{} +} + +// Register implements a fake Register +func (nc *ManagerFakeCollector) Register(registry *prometheus.Registry) error { return nil } + +// IncNginxReloadCount implements a fake IncNginxReloadCount +func (nc *ManagerFakeCollector) IncNginxReloadCount() {} + +// IncNginxReloadErrors implements a fake IncNginxReloadErrors +func (nc *ManagerFakeCollector) IncNginxReloadErrors() {} + +// UpdateLastReloadTime implements a fake UpdateLastReloadTime +func (nc *ManagerFakeCollector) UpdateLastReloadTime(ms time.Duration) {} diff --git a/internal/metrics/prometheus.go b/internal/metrics/listener.go similarity index 83% rename from internal/metrics/prometheus.go rename to internal/metrics/listener.go index 3864293edf..3af558d5a3 100644 --- a/internal/metrics/prometheus.go +++ b/internal/metrics/listener.go @@ -8,7 +8,7 @@ import ( "github.com/golang/glog" sdkClient "github.com/nginxinc/nginx-plus-go-sdk/client" prometheusClient "github.com/nginxinc/nginx-prometheus-exporter/client" - "github.com/nginxinc/nginx-prometheus-exporter/collector" + nginxCollector "github.com/nginxinc/nginx-prometheus-exporter/collector" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -22,16 +22,14 @@ func NewNginxMetricsClient(httpClient *http.Client) (*prometheusClient.NginxClie } // RunPrometheusListenerForNginx runs an http server to expose Prometheus metrics for NGINX -func RunPrometheusListenerForNginx(port int, client *prometheusClient.NginxClient) { - registry := prometheus.NewRegistry() - registry.MustRegister(collector.NewNginxCollector(client, "nginx")) +func RunPrometheusListenerForNginx(port int, client *prometheusClient.NginxClient, registry *prometheus.Registry) { + registry.MustRegister(nginxCollector.NewNginxCollector(client, "nginx_ingress_nginx")) runServer(strconv.Itoa(port), registry) } // RunPrometheusListenerForNginxPlus runs an http server to expose Prometheus metrics for NGINX Plus -func RunPrometheusListenerForNginxPlus(port int, plusClient *sdkClient.NginxClient) { - registry := prometheus.NewRegistry() - registry.MustRegister(collector.NewNginxPlusCollector(plusClient, "nginxplus")) +func RunPrometheusListenerForNginxPlus(port int, plusClient *sdkClient.NginxClient, registry *prometheus.Registry) { + registry.MustRegister(nginxCollector.NewNginxPlusCollector(plusClient, "nginx_ingress_nginxplus")) runServer(strconv.Itoa(port), registry) } diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index 7a86848be0..9eca6efa1d 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -6,6 +6,9 @@ import ( "os" "os/exec" "path" + "time" + + "github.com/nginxinc/kubernetes-ingress/internal/metrics/collectors" "github.com/golang/glog" "github.com/nginxinc/nginx-plus-go-sdk/client" @@ -60,10 +63,11 @@ type LocalManager struct { quitCmd string plusClient *client.NginxClient plusConfigVersionCheckClient *http.Client + metricsCollector collectors.ManagerCollector } // NewLocalManager creates a LocalManager. -func NewLocalManager(confPath string, binaryFilename string) *LocalManager { +func NewLocalManager(confPath string, binaryFilename string, mc collectors.ManagerCollector) *LocalManager { verifyConfigGenerator, err := newVerifyConfigGenerator() if err != nil { glog.Fatalf("error instantiating a verifyConfigGenerator: %v", err) @@ -81,6 +85,7 @@ func NewLocalManager(confPath string, binaryFilename string) *LocalManager { verifyClient: newVerifyClient(), reloadCmd: fmt.Sprintf("%v -s %v", binaryFilename, "reload"), quitCmd: fmt.Sprintf("%v -s %v", binaryFilename, "quit"), + metricsCollector: mc, } return &manager @@ -194,14 +199,22 @@ func (lm *LocalManager) Reload() error { glog.V(3).Infof("Reloading nginx with configVersion: %v", lm.configVersion) + t1 := time.Now() + if err := shellOut(lm.reloadCmd); err != nil { + lm.metricsCollector.IncNginxReloadErrors() return fmt.Errorf("nginx reload failed: %v", err) } err := lm.verifyClient.WaitForCorrectVersion(lm.configVersion) if err != nil { + lm.metricsCollector.IncNginxReloadErrors() return fmt.Errorf("could not get newest config version: %v", err) } + lm.metricsCollector.IncNginxReloadCount() + + t2 := time.Now() + lm.metricsCollector.UpdateLastReloadTime(t2.Sub(t1)) return nil }