Deep service insight endpoint

cmd/nginx-ingress/flags.go

@@ -139,6 +139,15 @@ var (
 	prometheusMetricsListenPort = flag.Int("prometheus-metrics-listen-port", 9113,
 		"Set the port where the Prometheus metrics are exposed. [1024 - 65535]")
 
+	enableServiceInsight = flag.Bool("enable-service-insight", false,
+		`Enable service insight for external load balancers. Requires -nginx-plus`)
+
+	serviceInsightTLSSecretName = flag.String("service-insight-tls-secret", "",
+		`A Secret with a TLS certificate and key for TLS termination of the service insight.`)
+
+	serviceInsightListenPort = flag.Int("service-insight-listen-port", 9114,
+		"Set the port where the Service Insight stats are exposed. Requires -nginx-plus. [1024 - 65535]")
+
 	enableCustomResources = flag.Bool("enable-custom-resources", true,
 		"Enable custom resources")
 
@@ -250,6 +259,11 @@ func parseFlags() {
 		*enableLatencyMetrics = false
 	}
 
+	if *enableServiceInsight && !*nginxPlus {
+		glog.Warning("enable-service-insight flag support is for NGINX Plus, service insight endpoint will not be exposed")
+		*enableServiceInsight = false
+	}
+
 	if *enableCertManager && !*enableCustomResources {
 		glog.Fatal("enable-cert-manager flag requires -enable-custom-resources")
 	}
@@ -352,6 +366,11 @@ func validationChecks() {
 		glog.Fatalf("Invalid value for ready-status-port: %v", readyStatusPortValidationError)
 	}
 
+	healthProbePortValidationError := validatePort(*serviceInsightListenPort)
+	if healthProbePortValidationError != nil {
+		glog.Fatalf("Invalid value for service-insight-listen-port: %v", metricsPortValidationError)
+	}
+
 	var err error
 	allowedCIDRs, err = parseNginxStatusAllowCIDRs(*nginxStatusAllowCIDRs)
 	if err != nil {

cmd/nginx-ingress/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/nginxinc/kubernetes-ingress/internal/configs"
 	"github.com/nginxinc/kubernetes-ingress/internal/configs/version1"
 	"github.com/nginxinc/kubernetes-ingress/internal/configs/version2"
+	"github.com/nginxinc/kubernetes-ingress/internal/healthcheck"
 	"github.com/nginxinc/kubernetes-ingress/internal/k8s"
 	"github.com/nginxinc/kubernetes-ingress/internal/k8s/secrets"
 	"github.com/nginxinc/kubernetes-ingress/internal/metrics"
@@ -120,6 +121,10 @@ func main() {
 	transportServerValidator := cr_validation.NewTransportServerValidator(*enableTLSPassthrough, *enableSnippets, *nginxPlus)
 	virtualServerValidator := cr_validation.NewVirtualServerValidator(cr_validation.IsPlus(*nginxPlus), cr_validation.IsDosEnabled(*appProtectDos), cr_validation.IsCertManagerEnabled(*enableCertManager), cr_validation.IsExternalDNSEnabled(*enableExternalDNS))
 
+	if *enableServiceInsight {
+		createHealthProbeEndpoint(kubeClient, plusClient, cnf)
+	}
+
 	lbcInput := k8s.NewLoadBalancerControllerInput{
 		KubeClient:                   kubeClient,
 		ConfClient:                   confClient,
@@ -446,6 +451,10 @@ func createGlobalConfigurationValidator() *cr_validation.GlobalConfigurationVali
 		forbiddenListenerPorts[*prometheusMetricsListenPort] = true
 	}
 
+	if *enableServiceInsight {
+		forbiddenListenerPorts[*serviceInsightListenPort] = true
+	}
+
 	return cr_validation.NewGlobalConfigurationValidator(forbiddenListenerPorts)
 }
 
@@ -674,6 +683,22 @@ func createPlusAndLatencyCollectors(
 	return plusCollector, syslogListener, lc
 }
 
+func createHealthProbeEndpoint(kubeClient *kubernetes.Clientset, plusClient *client.NginxClient, cnf *configs.Configurator) {
+	if !*enableServiceInsight {
+		return
+	}
+	var serviceInsightSecret *api_v1.Secret
+	var err error
+
+	if *serviceInsightTLSSecretName != "" {
+		serviceInsightSecret, err = getAndValidateSecret(kubeClient, *serviceInsightTLSSecretName)
+		if err != nil {
+			glog.Fatalf("Error trying to get the service insight TLS secret %v: %v", *serviceInsightTLSSecretName, err)
+		}
+	}
+	go healthcheck.RunHealthCheck(*serviceInsightListenPort, plusClient, cnf, serviceInsightSecret)
+}
+
 func processGlobalConfiguration() {
 	if *globalConfiguration != "" {
 		_, _, err := k8s.ParseNamespaceName(*globalConfiguration)

deployments/deployment/nginx-plus-ingress.yaml

@@ -32,6 +32,8 @@ spec:
           containerPort: 8081
         - name: prometheus
           containerPort: 9113
+        - name: service-insight
+          containerPort: 9114
         readinessProbe:
           httpGet:
             path: /nginx-ready
@@ -75,4 +77,5 @@ spec:
          #- -report-ingress-status
          #- -external-service=nginx-ingress
          #- -enable-prometheus-metrics
+         #- -enable-service-insight
          #- -global-configuration=$(POD_NAMESPACE)/nginx-configuration

deployments/helm-chart/README.md

@@ -262,6 +262,10 @@ Parameter | Description | Default
 `prometheus.port` | Configures the port to scrape the metrics. | 9113
 `prometheus.scheme` | Configures the HTTP scheme to use for connections to the Prometheus endpoint. | http
 `prometheus.secret` | The namespace / name of a Kubernetes TLS Secret. If specified, this secret is used to secure the Prometheus endpoint with TLS connections. | ""
+`serviceInsight.create` | Expose NGINX Plus Service Insight endpoint. | false
+`serviceInsight.port` | Configures the port to expose endpoints. | 9114
+`serviceInsight.scheme` | Configures the HTTP scheme to use for connections to the Service Insight endpoint. | http
+`serviceInsight.secret` | The namespace / name of a Kubernetes TLS Secret. If specified, this secret is used to secure the Service Insight endpoint with TLS connections. | ""
 `nginxServiceMesh.enable` | Enable integration with NGINX Service Mesh. See the NGINX Service Mesh [docs](https://docs.nginx.com/nginx-service-mesh/tutorials/kic/deploy-with-kic/) for more details. Requires `controller.nginxplus`. | false
 `nginxServiceMesh.enableEgress` | Enable NGINX Service Mesh workloads to route egress traffic through the Ingress Controller. See the NGINX Service Mesh [docs](https://docs.nginx.com/nginx-service-mesh/tutorials/kic/deploy-with-kic/#enabling-egress) for more details. Requires `nginxServiceMesh.enable`. | false
 

deployments/helm-chart/templates/controller-daemonset.yaml

@@ -94,6 +94,10 @@ spec:
         - name: prometheus
           containerPort: {{ .Values.prometheus.port }}
 {{- end }}
+{{- if .Values.serviceInsight.create }}
+        - name: service-insight
+          containerPort: {{ .Values.serviceInsight.port }}
+{{- end }}
 {{- if .Values.controller.readyStatus.enable }}
         - name: readiness-port
           containerPort: {{ .Values.controller.readyStatus.port }}
@@ -199,6 +203,9 @@ spec:
           - -enable-prometheus-metrics={{ .Values.prometheus.create }}
           - -prometheus-metrics-listen-port={{ .Values.prometheus.port }}
           - -prometheus-tls-secret={{ .Values.prometheus.secret }}
+          - -enable-service-insight={{ .Values.serviceInsight.create }}
+          - -service-insight-listen-port={{ .Values.serviceInsight.port }}
+          - -service-insight-tls-secret={{ .Values.serviceInsight.secret }}
           - -enable-custom-resources={{ .Values.controller.enableCustomResources }}
           - -enable-snippets={{ .Values.controller.enableSnippets }}
           - -include-year={{ .Values.controller.includeYear }}

deployments/helm-chart/templates/controller-deployment.yaml

@@ -97,6 +97,10 @@ spec:
         - name: prometheus
           containerPort: {{ .Values.prometheus.port }}
 {{- end }}
+{{- if .Values.serviceInsight.create }}
+        - name: service-insight
+          containerPort: {{ .Values.serviceInsight.port }}
+{{- end }}
 {{- if .Values.controller.readyStatus.enable }}
         - name: readiness-port
           containerPort: {{ .Values.controller.readyStatus.port }}
@@ -202,6 +206,9 @@ spec:
           - -enable-prometheus-metrics={{ .Values.prometheus.create }}
           - -prometheus-metrics-listen-port={{ .Values.prometheus.port }}
           - -prometheus-tls-secret={{ .Values.prometheus.secret }}
+          - -enable-service-insight={{ .Values.serviceInsight.create }}
+          - -service-insight-listen-port={{ .Values.serviceInsight.port }}
+          - -service-insight-tls-secret={{ .Values.serviceInsight.secret }}
           - -enable-custom-resources={{ .Values.controller.enableCustomResources }}
           - -enable-snippets={{ .Values.controller.enableSnippets }}
           - -include-year={{ .Values.controller.includeYear }}

deployments/helm-chart/values.schema.json

@@ -7,6 +7,7 @@
     "controller",
     "rbac",
     "prometheus",
+    "serviceInsight",
     "nginxServiceMesh"
   ],
   "properties": {
@@ -1436,6 +1437,56 @@
         }
       ]
     },
+    "serviceInsight": {
+      "type": "object",
+      "default": {},
+      "title": "The Service Insight Schema",
+      "required": [
+        "create"
+      ],
+      "properties": {
+        "create": {
+          "type": "boolean",
+          "default": false,
+          "title": "The create",
+          "examples": [
+            true
+          ]
+        },
+        "port": {
+          "type": "integer",
+          "default": 9114,
+          "title": "The port",
+          "examples": [
+            9114
+          ]
+        },
+        "secret": {
+          "type": "string",
+          "default": "",
+          "title": "The secret",
+          "examples": [
+            ""
+          ]
+        },
+        "scheme": {
+          "type": "string",
+          "default": "http",
+          "title": "The scheme",
+          "examples": [
+            "http"
+          ]
+        }
+      },
+      "examples": [
+        {
+          "create": true,
+          "port": 9114,
+          "secret": "",
+          "scheme": "http"
+        }
+      ]
+    },
     "nginxServiceMesh": {
       "type": "object",
       "default": {},
@@ -1622,6 +1673,12 @@
         "secret": "",
         "scheme": "http"
       },
+      "serviceInsight": {
+        "create": true,
+        "port": 9114,
+        "secret": "",
+        "scheme": "http"
+      },
       "nginxServiceMesh": {
         "enable": false,
         "enableEgress": false

deployments/helm-chart/values.yaml

@@ -433,6 +433,19 @@ prometheus:
   ## Configures the HTTP scheme used.
   scheme: http
 
+serviceInsight:
+  ## Expose NGINX Plus Service Insight endpoint.
+  create: false
+
+  ## Configures the port to expose endpoint.
+  port: 9114
+
+  ## Specifies the namespace/name of a Kubernetes TLS Secret which will be used to protect the Service Insight endpoint.
+  secret: ""
+
+  ## Configures the HTTP scheme used.
+  scheme: http
+
 nginxServiceMesh:
   ## Enables integration with NGINX Service Mesh.
   ## Requires controller.nginxplus

docs/content/configuration/global-configuration/command-line-arguments.md

@@ -336,7 +336,30 @@ Format: `[1024 - 65535]` (default `9113`)
 
 A Secret with a TLS certificate and key for TLS termination of the Prometheus metrics endpoint.
 
-* If the argument is not set, the prometheus endpoint will not use a TLS connection.
+* If the argument is not set, the Prometheus endpoint will not use a TLS connection.
+* If the argument is set, but the Ingress Controller is not able to fetch the Secret from Kubernetes API, the Ingress Controller will fail to start.
+ 
+<a name="cmdoption-enable-service-insight"></a>
+
+### -enable-service-insight
+
+Exposes the Service Insight endpoint for Ingress Controller.
+&nbsp;
+<a name="cmdoption-service-insight-listen-port"></a>
+
+### -service-insight-listen-port `<int>`
+
+Sets the port where the Service Insight is exposed.
+
+Format: `[1024 - 65535]` (default `9114`)
+&nbsp;
+<a name="cmdoption-service-insight-tls-secret"></a>
+
+### -service-insight-tls-secret `<string>`
+
+A Secret with a TLS certificate and key for TLS termination of the Service Insight endpoint.
+
+* If the argument is not set, the Service Insight endpoint will not use a TLS connection.
 * If the argument is set, but the Ingress Controller is not able to fetch the Secret from Kubernetes API, the Ingress Controller will fail to start.
 
 Format: `<namespace>/<name>`

docs/content/installation/installation-with-helm.md

@@ -250,6 +250,10 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont
 |``prometheus.port`` | Configures the port to scrape the metrics. | 9113 |
 |``prometheus.scheme`` | Configures the HTTP scheme that requests must use to connect to the Prometheus endpoint. | http |
 |``prometheus.secret`` | Specifies the namespace/name of a Kubernetes TLS secret which can be used to establish a secure HTTPS connection with the Prometheus endpoint. | "" |
+|``serviceInsight.create`` | Expose NGINX Plus Service Insight endpoint. | false |
+|``serviceInsight.port`` | Configures the port to scrape the metrics. | 9114 |
+|``serviceInsight.scheme`` | Configures the HTTP scheme to use for connections to the Service Insight endpoint. | http |
+|``serviceInsight.secret`` | The namespace / name of a Kubernetes TLS Secret. If specified, this secret is used to secure the Service Insight endpoint with TLS connections. | "" |
 {{% /table %}}
 
 ## Notes

docs/content/logging-and-monitoring/service-insight.md

@@ -0,0 +1,58 @@
+---
+title: Service Insight
+
+description: "The Ingress Controller exposes the Service Insight endpoint."
+weight: 2100
+doctypes: [""]
+aliases:
+    - /service-insight/
+toc: true
+docs: "DOCS-000"
+---
+
+
+The Ingress Controller exposes an endpoint and provides host statistics for Virtual Servers (VS).
+It exposes data in JSON format and returns HTTP status codes.
+The response body holds information about the total, down and the unhealthy number of
+upstreams associated with the hostname.
+Returned HTTP codes indicate the health of the upstreams (service).
+
+The service is not healthy (HTTP response code different than 200 OK) if all upstreams are unhealthy.
+The service is healthy if at least one upstream is healthy. In this case, the endpoint returns HTTP code 200 OK.
+
+
+
+## Enabling Service Insight Endpoint
+
+If you're using *Kubernetes manifests* (Deployment or DaemonSet) to install the Ingress Controller, to enable the Service Insight endpoint:
+1. Run the Ingress Controller with the `-enable-service-insight` [command-line argument](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments). This will expose the Ingress Controller endpoint via the path `/probe/{hostname}` on port `9114` (customizable with the `-service-insight-listen-port` command-line argument).
+1. To enable TLS for the Service Insight endpoint, configure the `-service-insight-tls-secret` cli argument with the namespace and name of a TLS Secret.
+1. Add the Service Insight port to the list of the ports of the Ingress Controller container in the template of the Ingress Controller pod:
+    ```yaml
+    - name: service-insight
+      containerPort: 9114
+    ```
+
+If you're using *Helm* to install the Ingress Controller, to enable Service Insight endpoint, configure the `serviceinsight.*` parameters of the Helm chart. See the [Installation with Helm](/nginx-ingress-controller/installation/installation-with-helm) doc.
+
+## Available Statistics and HTTP Response Codes
+
+The Service Insight provides the following statistics:
+
+* Total number of VS
+* Number of VS in 'Down' state
+* Number of VS in 'Healthy' state
+
+These statistics are returned as JSON:
+
+```json
+{ "Total": <int>, "Up": <int>, "Unhealthy": <int>  }
+```
+
+Response codes:
+
+* HTTP 200 OK - Service is healthy
+* HTTP 404 - No upstreams/VS found for the requested hostname
+* HTTP 503 Service Unavailable - The service is down (All upstreams/VS are "Unhealthy")
+
+**Note**: wildcards in hostnames are not supported at the moment.

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.18.3
 	github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.24
 	github.com/cert-manager/cert-manager v1.10.1
+	github.com/go-chi/chi v1.5.4
 	github.com/golang-jwt/jwt/v4 v4.4.2
 	github.com/golang/glog v1.0.0
 	github.com/google/go-cmp v0.5.9

go.sum

@@ -112,6 +112,8 @@ github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwV
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
 github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs=
+github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=