Add Ingress Controller Prometheus metrics

cmd/nginx-ingress/main.go

@@ -12,13 +12,16 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/nginxinc/kubernetes-ingress/internal/metrics/collectors"
+
 	"github.com/golang/glog"
 	"github.com/nginxinc/kubernetes-ingress/internal/configs"
 	"github.com/nginxinc/kubernetes-ingress/internal/configs/version1"
 	"github.com/nginxinc/kubernetes-ingress/internal/k8s"
 	"github.com/nginxinc/kubernetes-ingress/internal/metrics"
 	"github.com/nginxinc/kubernetes-ingress/internal/nginx"
 	"github.com/nginxinc/nginx-plus-go-sdk/client"
+	"github.com/prometheus/client_golang/prometheus"
 	api_v1 "k8s.io/api/core/v1"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -187,13 +190,34 @@ func main() {
 		glog.Fatalf("Error creating TemplateExecutor: %v", err)
 	}
 
-	useFakeNginxManager := *proxyURL != ""
+	var registry *prometheus.Registry
+	var managerCollector collectors.ManagerCollector
+	var controllerCollector collectors.ControllerCollector
+	managerCollector = collectors.NewManagerFakeCollector()
+	controllerCollector = collectors.NewControllerFakeCollector()
 
+	if *enablePrometheusMetrics {
+		registry = prometheus.NewRegistry()
+		managerCollector = collectors.NewLocalManagerMetricsCollector()
+		controllerCollector = collectors.NewControllerMetricsCollector()
+
+		err = managerCollector.Register(registry)
+		if err != nil {
+			glog.Errorf("Error registering Manager Prometheus metrics: %v", err)
+		}
+
+		err = controllerCollector.Register(registry)
+		if err != nil {
+			glog.Errorf("Error registering Controller Prometheus metrics: %v", err)
+		}
+	}
+
+	useFakeNginxManager := *proxyURL != ""
 	var nginxManager nginx.Manager
 	if useFakeNginxManager {
 		nginxManager = nginx.NewFakeManager("/etc/nginx")
 	} else {
-		nginxManager = nginx.NewLocalManager("/etc/nginx/", nginxBinaryPath)
+		nginxManager = nginx.NewLocalManager("/etc/nginx/", nginxBinaryPath, managerCollector)
 	}
 
 	if *defaultServerSecret != "" {
@@ -283,6 +307,19 @@ func main() {
 		nginxManager.SetPlusClients(plusClient, httpClient)
 	}
 
+	if *enablePrometheusMetrics {
+		if *nginxPlus {
+			go metrics.RunPrometheusListenerForNginxPlus(*prometheusMetricsListenPort, plusClient, registry)
+		} else {
+			httpClient := getSocketClient("/var/run/nginx-status.sock")
+			client, err := metrics.NewNginxMetricsClient(httpClient)
+			if err != nil {
+				glog.Fatalf("Error creating the Nginx client for Prometheus metrics: %v", err)
+			}
+			go metrics.RunPrometheusListenerForNginx(*prometheusMetricsListenPort, client, registry)
+		}
+	}
+
 	isWildcardEnabled := *wildcardTLSSecret != ""
 	cnf := configs.NewConfigurator(nginxManager, staticCfgParams, cfgParams, templateExecutor, *nginxPlus, isWildcardEnabled)
 	controllerNamespace := os.Getenv("POD_NAMESPACE")
@@ -303,23 +340,11 @@ func main() {
 		LeaderElectionLockName:  *leaderElectionLockName,
 		WildcardTLSSecret:       *wildcardTLSSecret,
 		ConfigMaps:              *nginxConfigMaps,
+		MetricsCollector:        controllerCollector,
 	}
 
 	lbc := k8s.NewLoadBalancerController(lbcInput)
 
-	if *enablePrometheusMetrics {
-		if *nginxPlus {
-			go metrics.RunPrometheusListenerForNginxPlus(*prometheusMetricsListenPort, plusClient)
-		} else {
-			httpClient := getSocketClient("/var/run/nginx-status.sock")
-			client, err := metrics.NewNginxMetricsClient(httpClient)
-			if err != nil {
-				glog.Fatalf("Error creating the Nginx client for Prometheus metrics: %v", err)
-			}
-			go metrics.RunPrometheusListenerForNginx(*prometheusMetricsListenPort, client)
-		}
-	}
-
 	go handleTermination(lbc, nginxManager, nginxDone)
 	lbc.Run()
 

docs/installation.md

@@ -165,15 +165,15 @@ For NGINX Plus, you can access the live activity monitoring dashboard:
 
 ## Support For Prometheus Monitoring
 
-You can expose NGINX or NGINX Plus metrics for collection by [Prometheus](https://prometheus.io/):
+You can expose NGINX/NGINX Plus and Ingress Controller [metrics](./prometheus.md) for collection by [Prometheus](https://prometheus.io/):
 
 1. Run the Ingress controller with the `-enable-prometheus-metrics` [command-line argument](cli-arguments.md). As a result, the Ingress Controller will expose NGINX or NGINX Plus metrics in the Prometheus format via the path `/metrics` on port `9113` (customizable via the `-prometheus-metrics-listen-port` command-line argument).
 1. Add the Prometheus port to the list of the ports of the Ingress Controller container:
     ```yaml
     - name: prometheus
       containerPort: 9113
     ```
-1. Make the Prometheus aware of the Ingress Controller targets by adding the following annotations to the template of the Ingress Controller pod (note: this assumes your Prometheus is configured to discover targets by analyzing the annotations of pods):
+1. Make Prometheus aware of the Ingress Controller targets by adding the following annotations to the template of the Ingress Controller pod (note: this assumes your Prometheus is configured to discover targets by analyzing the annotations of pods):
     ```yaml
     annotations:
         prometheus.io/scrape: "true"

docs/prometheus.md

@@ -0,0 +1,20 @@
+# Monitoring the Ingress Controller Using Prometheus
+
+The Ingress Controller exposes a number of metrics in the [Prometheus](https://prometheus.io/) format. Those include NGINX/NGINX Plus and the Ingress Controller metrics.
+
+## Enabling Metrics
+To enable Prometheus metrics, follow the [Support For Prometheus Monitoring](./installation.md#support-for-prometheus-monitoring) section of the installation doc. Once enabled, the metrics will be available via the configured endpoint.
+
+## Available Metrics
+The Ingress Controller exports the following metrics:
+
+* NGINX/NGINX Plus metrics. Please see this [doc](https://github.com/nginxinc/nginx-prometheus-exporter#exported-metrics) to find more information about the exported metrics.
+
+* Ingress Controller metrics
+  * `controller_nginx_reloads_total`. Number of successful NGINX reloads.
+  * `controller_nginx_reload_errors_total`. Number of unsuccessful NGINX reloads.
+  * `controller_nginx_last_reload_status`. Status of the last NGINX reload, 0 meaning down and 1 up.
+  * `controller_nginx_last_reload_milliseconds`. Duration in milliseconds of the last NGINX reload.
+  * `controller_ingress_resources_total`. Number of handled Ingress resources. This metric includes the label type, that groups the Ingress resources by their type (regular, [minion or master](./../examples/mergeable-ingress-types))
+
+**Note**: all metrics have the namespace nginx_ingress. For example, nginx_ingress_controller_nginx_reloads_total.

internal/configs/configurator.go

@@ -469,3 +469,27 @@ func (cnf *Configurator) HasMinion(master *extensions.Ingress, minion *extension
 func (cnf *Configurator) IsResolverConfigured() bool {
 	return len(cnf.cfgParams.ResolverAddresses) != 0
 }
+
+// GetIngressCounts returns the total count of Ingress resources that are handled by the Ingress Controller grouped by their type
+func (cnf *Configurator) GetIngressCounts() map[string]int {
+	counters := map[string]int{
+		"master":  0,
+		"regular": 0,
+		"minion":  0,
+	}
+
+	// cnf.ingresses contains only master and regular Ingress Resources
+	for _, ing := range cnf.ingresses {
+		if ing.Ingress.Annotations["nginx.org/mergeable-ingress-type"] == "master" {
+			counters["master"]++
+		} else {
+			counters["regular"]++
+		}
+	}
+
+	for _, min := range cnf.minions {
+		counters["minion"] += len(min)
+	}
+
+	return counters
+}

internal/k8s/controller.go

@@ -24,6 +24,7 @@ import (
 	"github.com/golang/glog"
 
 	"github.com/nginxinc/kubernetes-ingress/internal/configs"
+	"github.com/nginxinc/kubernetes-ingress/internal/metrics/collectors"
 	"k8s.io/api/extensions/v1beta1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
@@ -79,6 +80,7 @@ type LoadBalancerController struct {
 	namespace               string
 	controllerNamespace     string
 	wildcardTLSSecret       string
+	metricsCollector        collectors.ControllerCollector
 }
 
 var keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc
@@ -100,6 +102,7 @@ type NewLoadBalancerControllerInput struct {
 	LeaderElectionLockName  string
 	WildcardTLSSecret       string
 	ConfigMaps              string
+	MetricsCollector        collectors.ControllerCollector
 }
 
 // NewLoadBalancerController creates a controller
@@ -118,6 +121,7 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc
 		namespace:               input.Namespace,
 		controllerNamespace:     input.ControllerNamespace,
 		wildcardTLSSecret:       input.WildcardTLSSecret,
+		metricsCollector:        input.MetricsCollector,
 	}
 
 	eventBroadcaster := record.NewBroadcaster()
@@ -456,20 +460,20 @@ func (lbc *LoadBalancerController) sync(task task) {
 	switch task.Kind {
 	case ingress:
 		lbc.syncIng(task)
+		lbc.updateIngressMetrics()
 	case ingressMinion:
 		lbc.syncIngMinion(task)
+		lbc.updateIngressMetrics()
 	case configMap:
 		lbc.syncConfig(task)
-		return
 	case endpoints:
 		lbc.syncEndpoint(task)
-		return
 	case secret:
 		lbc.syncSecret(task)
-		return
 	case service:
 		lbc.syncExternalService(task)
 	}
+
 }
 
 func (lbc *LoadBalancerController) syncIngMinion(task task) {
@@ -591,6 +595,13 @@ func (lbc *LoadBalancerController) syncIng(task task) {
 	}
 }
 
+func (lbc *LoadBalancerController) updateIngressMetrics() {
+	counters := lbc.configurator.GetIngressCounts()
+	for nType, count := range counters {
+		lbc.metricsCollector.SetIngressResources(nType, count)
+	}
+}
+
 // syncExternalService does not sync all services.
 // We only watch the Service specified by the external-service flag.
 func (lbc *LoadBalancerController) syncExternalService(task task) {

internal/k8s/controller_test.go

@@ -7,10 +7,12 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/nginxinc/kubernetes-ingress/internal/metrics/collectors"
+
 	"github.com/nginxinc/kubernetes-ingress/internal/configs"
 	"github.com/nginxinc/kubernetes-ingress/internal/configs/version1"
 	"github.com/nginxinc/kubernetes-ingress/internal/nginx"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	extensions "k8s.io/api/extensions/v1beta1"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -31,6 +33,7 @@ func TestIsNginxIngress(t *testing.T) {
 			&LoadBalancerController{
 				ingressClass:        ingressClass,
 				useIngressClassOnly: false,
+				metricsCollector:    collectors.NewControllerFakeCollector(),
 			},
 			&extensions.Ingress{
 				ObjectMeta: meta_v1.ObjectMeta{
@@ -43,6 +46,7 @@ func TestIsNginxIngress(t *testing.T) {
 			&LoadBalancerController{
 				ingressClass:        ingressClass,
 				useIngressClassOnly: false,
+				metricsCollector:    collectors.NewControllerFakeCollector(),
 			},
 			&extensions.Ingress{
 				ObjectMeta: meta_v1.ObjectMeta{
@@ -55,6 +59,7 @@ func TestIsNginxIngress(t *testing.T) {
 			&LoadBalancerController{
 				ingressClass:        ingressClass,
 				useIngressClassOnly: false,
+				metricsCollector:    collectors.NewControllerFakeCollector(),
 			},
 			&extensions.Ingress{
 				ObjectMeta: meta_v1.ObjectMeta{
@@ -67,6 +72,7 @@ func TestIsNginxIngress(t *testing.T) {
 			&LoadBalancerController{
 				ingressClass:        ingressClass,
 				useIngressClassOnly: false,
+				metricsCollector:    collectors.NewControllerFakeCollector(),
 			},
 			&extensions.Ingress{
 				ObjectMeta: meta_v1.ObjectMeta{
@@ -86,6 +92,7 @@ func TestIsNginxIngress(t *testing.T) {
 			&LoadBalancerController{
 				ingressClass:        ingressClass,
 				useIngressClassOnly: true,
+				metricsCollector:    collectors.NewControllerFakeCollector(),
 			},
 			&extensions.Ingress{
 				ObjectMeta: meta_v1.ObjectMeta{
@@ -98,6 +105,7 @@ func TestIsNginxIngress(t *testing.T) {
 			&LoadBalancerController{
 				ingressClass:        ingressClass,
 				useIngressClassOnly: true,
+				metricsCollector:    collectors.NewControllerFakeCollector(),
 			},
 			&extensions.Ingress{
 				ObjectMeta: meta_v1.ObjectMeta{
@@ -110,6 +118,7 @@ func TestIsNginxIngress(t *testing.T) {
 			&LoadBalancerController{
 				ingressClass:        ingressClass,
 				useIngressClassOnly: true,
+				metricsCollector:    collectors.NewControllerFakeCollector(),
 			},
 			&extensions.Ingress{
 				ObjectMeta: meta_v1.ObjectMeta{
@@ -122,6 +131,7 @@ func TestIsNginxIngress(t *testing.T) {
 			&LoadBalancerController{
 				ingressClass:        ingressClass,
 				useIngressClassOnly: true,
+				metricsCollector:    collectors.NewControllerFakeCollector(),
 			},
 			&extensions.Ingress{
 				ObjectMeta: meta_v1.ObjectMeta{
@@ -634,9 +644,10 @@ func getMergableDefaults() (cafeMaster, coffeeMinion, teaMinion extensions.Ingre
 
 	fakeClient := fake.NewSimpleClientset()
 	lbc = LoadBalancerController{
-		client:       fakeClient,
-		ingressClass: "nginx",
-		configurator: cnf,
+		client:           fakeClient,
+		ingressClass:     "nginx",
+		configurator:     cnf,
+		metricsCollector: collectors.NewControllerFakeCollector(),
 	}
 	lbc.svcLister, _ = cache.NewInformer(
 		cache.NewListWatchFromClient(lbc.client.ExtensionsV1beta1().RESTClient(), "services", "default", fields.Everything()),
@@ -836,9 +847,10 @@ func TestGetServicePortForIngressPort(t *testing.T) {
 	fakeClient := fake.NewSimpleClientset()
 	cnf := configs.NewConfigurator(&nginx.LocalManager{}, &configs.StaticConfigParams{}, &configs.ConfigParams{}, &version1.TemplateExecutor{}, false, false)
 	lbc := LoadBalancerController{
-		client:       fakeClient,
-		ingressClass: "nginx",
-		configurator: cnf,
+		client:           fakeClient,
+		ingressClass:     "nginx",
+		configurator:     cnf,
+		metricsCollector: collectors.NewControllerFakeCollector(),
 	}
 	svc := v1.Service{
 		TypeMeta: meta_v1.TypeMeta{},
@@ -986,10 +998,11 @@ func TestFindIngressesForSecret(t *testing.T) {
 
 			cnf := configs.NewConfigurator(manager, &configs.StaticConfigParams{}, &configs.ConfigParams{}, templateExecutor, false, false)
 			lbc := LoadBalancerController{
-				client:       fakeClient,
-				ingressClass: "nginx",
-				configurator: cnf,
-				isNginxPlus:  true,
+				client:           fakeClient,
+				ingressClass:     "nginx",
+				configurator:     cnf,
+				isNginxPlus:      true,
+				metricsCollector: collectors.NewControllerFakeCollector(),
 			}
 
 			lbc.ingressLister.Store, _ = cache.NewInformer(
@@ -1170,10 +1183,11 @@ func TestFindIngressesForSecretWithMinions(t *testing.T) {
 
 			cnf := configs.NewConfigurator(manager, &configs.StaticConfigParams{}, &configs.ConfigParams{}, templateExecutor, false, false)
 			lbc := LoadBalancerController{
-				client:       fakeClient,
-				ingressClass: "nginx",
-				configurator: cnf,
-				isNginxPlus:  true,
+				client:           fakeClient,
+				ingressClass:     "nginx",
+				configurator:     cnf,
+				isNginxPlus:      true,
+				metricsCollector: collectors.NewControllerFakeCollector(),
 			}
 
 			lbc.ingressLister.Store, _ = cache.NewInformer(

internal/k8s/utils.go

@@ -21,7 +21,7 @@ import (
 	"reflect"
 	"strings"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/api/extensions/v1beta1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/tools/cache"

internal/metrics/collectors/collectors.go

@@ -0,0 +1,3 @@
+package collectors
+
+const metricsNamespace = "nginx_ingress_controller"