diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index aade059fad..0731ba8e89 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -154,6 +154,10 @@ var ( spireAgentAddress = flag.String("spire-agent-address", "", `Specifies the address of the running Spire agent. For use with NGINX Service Mesh only. If the flag is set, but the Ingress Controller is not able to connect with the Spire Agent, the Ingress Controller will fail to start.`) + + readyStatus = flag.Bool("ready-status", true, "Enables the readiness endpoint '/nginx-ready'. The endpoint returns a success code when NGINX has loaded all the config after the startup") + + readyStatusPort = flag.Int("ready-status-port", 8081, "Set the port where the readiness endpoint is exposed. [1024 - 65535]") ) func main() { @@ -189,6 +193,11 @@ func main() { glog.Fatalf("Invalid value for prometheus-metrics-listen-port: %v", metricsPortValidationError) } + readyStatusPortValidationError := validatePort(*readyStatusPort) + if readyStatusPortValidationError != nil { + glog.Fatalf("Invalid value for ready-status-port: %v", readyStatusPortValidationError) + } + allowedCIDRs, err := parseNginxStatusAllowCIDRs(*nginxStatusAllowCIDRs) if err != nil { glog.Fatalf(`Invalid value for nginx-status-allow-cidrs: %v`, err) @@ -514,11 +523,21 @@ func main() { lbc := k8s.NewLoadBalancerController(lbcInput) + if *readyStatus { + go func() { + port := fmt.Sprintf(":%v", *readyStatusPort) + s := http.NewServeMux() + s.HandleFunc("/nginx-ready", ready(lbc)) + glog.Fatal(http.ListenAndServe(port, s)) + }() + } + if *appProtect { go handleTerminationWithAppProtect(lbc, nginxManager, nginxDone, aPAgentDone, aPPluginDone) } else { go handleTermination(lbc, nginxManager, nginxDone) } + lbc.Run() for { @@ -708,3 +727,14 @@ func parseReloadTimeout(appProtectEnabled bool, timeout int) int { return defaultTimeout } + +func ready(lbc *k8s.LoadBalancerController) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + if !lbc.IsNginxReady() { + http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, "Ready") + } +} diff --git a/deployments/daemon-set/nginx-ingress.yaml b/deployments/daemon-set/nginx-ingress.yaml index 5fa03633f7..c450f7c523 100644 --- a/deployments/daemon-set/nginx-ingress.yaml +++ b/deployments/daemon-set/nginx-ingress.yaml @@ -27,8 +27,15 @@ spec: - name: https containerPort: 443 hostPort: 443 + - name: readiness-port + containerPort: 8081 #- name: prometheus #containerPort: 9113 + readinessProbe: + httpGet: + path: /nginx-ready + port: readiness-port + periodSeconds: 1 securityContext: allowPrivilegeEscalation: true runAsUser: 101 #nginx diff --git a/deployments/daemon-set/nginx-plus-ingress.yaml b/deployments/daemon-set/nginx-plus-ingress.yaml index 487bfc7412..191926d19d 100644 --- a/deployments/daemon-set/nginx-plus-ingress.yaml +++ b/deployments/daemon-set/nginx-plus-ingress.yaml @@ -27,8 +27,15 @@ spec: - name: https containerPort: 443 hostPort: 443 + - name: readiness-port + containerPort: 8081 #- name: prometheus #containerPort: 9113 + readinessProbe: + httpGet: + path: /nginx-ready + port: readiness-port + periodSeconds: 1 securityContext: allowPrivilegeEscalation: true runAsUser: 101 #nginx diff --git a/deployments/deployment/nginx-ingress.yaml b/deployments/deployment/nginx-ingress.yaml index d59d641735..51c39fd765 100644 --- a/deployments/deployment/nginx-ingress.yaml +++ b/deployments/deployment/nginx-ingress.yaml @@ -26,8 +26,15 @@ spec: containerPort: 80 - name: https containerPort: 443 + - name: readiness-port + containerPort: 8081 #- name: prometheus #containerPort: 9113 + readinessProbe: + httpGet: + path: /nginx-ready + port: readiness-port + periodSeconds: 1 securityContext: allowPrivilegeEscalation: true runAsUser: 101 #nginx diff --git a/deployments/deployment/nginx-plus-ingress.yaml b/deployments/deployment/nginx-plus-ingress.yaml index be41f8b6fc..be871f75b5 100644 --- a/deployments/deployment/nginx-plus-ingress.yaml +++ b/deployments/deployment/nginx-plus-ingress.yaml @@ -26,8 +26,15 @@ spec: containerPort: 80 - name: https containerPort: 443 + - name: readiness-port + containerPort: 8081 #- name: prometheus #containerPort: 9113 + readinessProbe: + httpGet: + path: /nginx-ready + port: readiness-port + periodSeconds: 1 securityContext: allowPrivilegeEscalation: true runAsUser: 101 #nginx diff --git a/deployments/helm-chart/README.md b/deployments/helm-chart/README.md index 4d7e896c05..1a7e3cb6b2 100644 --- a/deployments/helm-chart/README.md +++ b/deployments/helm-chart/README.md @@ -255,6 +255,8 @@ Parameter | Description | Default `controller.reportIngressStatus.annotations` | The annotations of the leader election configmap. | {} `controller.pod.annotations` | The annotations of the Ingress Controller pod. | {} `controller.appprotect.enable` | Enables the App Protect module in the Ingress Controller. | false +`controller.readyStatus.enable` | Enables the readiness endpoint `"/nginx-ready"`. The endpoint returns a success code when NGINX has loaded all the config after the startup. This also configures a readiness probe for the Ingress Controller pods that uses the readiness endpoint. | true +`controller.readyStaus.port` | The HTTP port for the readiness endpoint. | 8081 `rbac.create` | Configures RBAC. | true `prometheus.create` | Expose NGINX or NGINX Plus metrics in the Prometheus format. | false `prometheus.port` | Configures the port to scrape the metrics. | 9113 diff --git a/deployments/helm-chart/templates/controller-daemonset.yaml b/deployments/helm-chart/templates/controller-daemonset.yaml index 9b1c558e56..0ea4a78845 100644 --- a/deployments/helm-chart/templates/controller-daemonset.yaml +++ b/deployments/helm-chart/templates/controller-daemonset.yaml @@ -64,6 +64,15 @@ spec: {{- if .Values.prometheus.create }} - name: prometheus containerPort: {{ .Values.prometheus.port }} +{{- end }} +{{- if .Values.controller.readyStatus.enable }} + - name: readiness-port + containerPort: {{ .Values.controller.readyStatus.port}} + readinessProbe: + httpGet: + path: /nginx-ready + port: readiness-port + periodSeconds: 1 {{- end }} securityContext: allowPrivilegeEscalation: true @@ -137,4 +146,6 @@ spec: - -global-configuration=$(POD_NAMESPACE)/{{ .Release.Name }} {{- end }} {{- end }} + - -ready-status={{ .Values.controller.readyStatus.enable }} + - -ready-status-port={{ .Values.controller.readyStatus.port }} {{- end }} diff --git a/deployments/helm-chart/templates/controller-deployment.yaml b/deployments/helm-chart/templates/controller-deployment.yaml index 47ec2824c6..4349141f9f 100644 --- a/deployments/helm-chart/templates/controller-deployment.yaml +++ b/deployments/helm-chart/templates/controller-deployment.yaml @@ -62,6 +62,15 @@ spec: {{- if .Values.prometheus.create }} - name: prometheus containerPort: {{ .Values.prometheus.port }} +{{- end }} +{{- if .Values.controller.readyStatus.enable }} + - name: readiness-port + containerPort: {{ .Values.controller.readyStatus.port}} + readinessProbe: + httpGet: + path: /nginx-ready + port: readiness-port + periodSeconds: 1 {{- end }} resources: {{ toYaml .Values.controller.resources | indent 10 }} @@ -135,4 +144,6 @@ spec: - -global-configuration=$(POD_NAMESPACE)/{{ .Release.Name }} {{- end }} {{- end }} + - -ready-status={{ .Values.controller.readyStatus.enable }} + - -ready-status-port={{ .Values.controller.readyStatus.port }} {{- end }} diff --git a/deployments/helm-chart/values.yaml b/deployments/helm-chart/values.yaml index 9f490ce6c9..16e14ba373 100644 --- a/deployments/helm-chart/values.yaml +++ b/deployments/helm-chart/values.yaml @@ -260,6 +260,13 @@ controller: ## The PriorityClass of the ingress controller pods. priorityClassName: + readyStatus: + ## Enables readiness endpoint "/nginx-ready". The endpoint returns a success code when NGINX has loaded all the config after startup. + enable: true + + ## Set the port where the readiness endpoint is exposed. + port: 8081 + rbac: ## Configures RBAC. create: true diff --git a/docs-web/configuration/global-configuration/command-line-arguments.md b/docs-web/configuration/global-configuration/command-line-arguments.md index 4f206125c8..75bfc3512c 100644 --- a/docs-web/configuration/global-configuration/command-line-arguments.md +++ b/docs-web/configuration/global-configuration/command-line-arguments.md @@ -188,5 +188,15 @@ Below we describe the available command-line arguments: Requires :option:`-nginx-plus` - If the argument is set, but `nginx-plus` is set to false, the Ingress Controller will fail to start. - + +.. option:: -ready-status + + Enables the readiness endpoint "/nginx-ready". The endpoint returns a success code when NGINX has loaded all the config after the startup. (default true) + +.. option:: -ready-status-port + + The HTTP port for the readiness endpoint. + + Format: ``[1024 - 65535]`` (default 8081) + ``` diff --git a/docs-web/installation/installation-with-helm.md b/docs-web/installation/installation-with-helm.md index d493aaa657..3531a7b28c 100644 --- a/docs-web/installation/installation-with-helm.md +++ b/docs-web/installation/installation-with-helm.md @@ -385,6 +385,12 @@ The following tables lists the configurable parameters of the NGINX Ingress cont * - ``controller.pod.annotations`` - The annotations of the Ingress Controller pod. - {} + * - ``controller.readyStatus.enable`` + - Enables the readiness endpoint `"/nginx-ready"`. The endpoint returns a success code when NGINX has loaded all the config after the startup. This also configures a readiness probe for the Ingress Controller pods that uses the readiness endpoint. + - true + * - ``controller.readyStaus.port`` + - The HTTP port for the readiness endpoint. + - 8081 * - ``rbac.create`` - Configures RBAC. - true diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index f1d3818ded..3ba023cf27 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -145,6 +145,7 @@ type LoadBalancerController struct { transportServerValidator *validation.TransportServerValidator spiffeController *spiffeController syncLock sync.Mutex + isNginxReady bool } var keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc @@ -271,7 +272,6 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc } lbc.updateIngressMetrics() - return lbc } @@ -835,6 +835,11 @@ func (lbc *LoadBalancerController) sync(task task) { case appProtectLogConf: lbc.syncAppProtectLogConf(task) } + + if !lbc.isNginxReady && lbc.syncQueue.Len() == 0 { + lbc.isNginxReady = true + glog.V(3).Infof("NGINX is ready") + } } func (lbc *LoadBalancerController) syncPolicy(task task) { @@ -935,6 +940,7 @@ func (lbc *LoadBalancerController) syncPolicy(task task) { } } } + } func (lbc *LoadBalancerController) syncTransportServer(task task) { @@ -3255,3 +3261,8 @@ func (lbc *LoadBalancerController) findIngressesForAppProtectResource(namespace } return apIngs } + +// IsNginxReady returns ready status of NGINX +func (lbc *LoadBalancerController) IsNginxReady() bool { + return lbc.isNginxReady +} diff --git a/internal/k8s/task_queue.go b/internal/k8s/task_queue.go index b36686a70a..13512e65d3 100644 --- a/internal/k8s/task_queue.go +++ b/internal/k8s/task_queue.go @@ -65,6 +65,12 @@ func (tq *taskQueue) Requeue(task task, err error) { tq.queue.Add(task) } +// Len returns the length of the queue +func (tq *taskQueue) Len() int { + glog.V(3).Infof("The queue has %v element(s)", tq.queue.Len()) + return tq.queue.Len() +} + // RequeueAfter adds the task to the queue after the given duration func (tq *taskQueue) RequeueAfter(t task, err error, after time.Duration) { glog.Errorf("Requeuing %v after %s, err %v", t.Key, after.String(), err)