diff --git a/deployments/common/crds/k8s.nginx.org_virtualserverroutes.yaml b/deployments/common/crds/k8s.nginx.org_virtualserverroutes.yaml index 7a9b0fcc6d..767cda1d1c 100644 --- a/deployments/common/crds/k8s.nginx.org_virtualserverroutes.yaml +++ b/deployments/common/crds/k8s.nginx.org_virtualserverroutes.yaml @@ -509,10 +509,14 @@ spec: type: string jitter: type: string + mandatory: + type: boolean passes: type: integer path: type: string + persistent: + type: boolean port: type: integer read-timeout: diff --git a/deployments/common/crds/k8s.nginx.org_virtualservers.yaml b/deployments/common/crds/k8s.nginx.org_virtualservers.yaml index e9759cf553..58cb822422 100644 --- a/deployments/common/crds/k8s.nginx.org_virtualservers.yaml +++ b/deployments/common/crds/k8s.nginx.org_virtualservers.yaml @@ -539,10 +539,14 @@ spec: type: string jitter: type: string + mandatory: + type: boolean passes: type: integer path: type: string + persistent: + type: boolean port: type: integer read-timeout: diff --git a/deployments/helm-chart/crds/k8s.nginx.org_virtualserverroutes.yaml b/deployments/helm-chart/crds/k8s.nginx.org_virtualserverroutes.yaml index 7a9b0fcc6d..767cda1d1c 100644 --- a/deployments/helm-chart/crds/k8s.nginx.org_virtualserverroutes.yaml +++ b/deployments/helm-chart/crds/k8s.nginx.org_virtualserverroutes.yaml @@ -509,10 +509,14 @@ spec: type: string jitter: type: string + mandatory: + type: boolean passes: type: integer path: type: string + persistent: + type: boolean port: type: integer read-timeout: diff --git a/deployments/helm-chart/crds/k8s.nginx.org_virtualservers.yaml b/deployments/helm-chart/crds/k8s.nginx.org_virtualservers.yaml index e9759cf553..58cb822422 100644 --- a/deployments/helm-chart/crds/k8s.nginx.org_virtualservers.yaml +++ b/deployments/helm-chart/crds/k8s.nginx.org_virtualservers.yaml @@ -539,10 +539,14 @@ spec: type: string jitter: type: string + mandatory: + type: boolean passes: type: integer path: type: string + persistent: + type: boolean port: type: integer read-timeout: diff --git a/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md b/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md index 22812c9a0a..0e5c80c421 100644 --- a/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md +++ b/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md @@ -324,12 +324,13 @@ Note: This feature is supported only in NGINX Plus. ### Upstream.Healthcheck -The Healthcheck defines an [active health check](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/). In the example below we enable a health check for an upstream and configure all the available parameters: +The Healthcheck defines an [active health check](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/). In the example below we enable a health check for an upstream and configure all the available parameters, including combining with the `slow-start` directive for [mandatory healthchecks](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#mandatory-health-checks): ```yaml name: tea service: tea-svc port: 80 +slow-start: 30s healthCheck: enable: true path: /healthz @@ -347,6 +348,8 @@ healthCheck: - name: Host value: my.service statusMatch: "! 500" + mandatory: true + persistent: true ``` Note: This feature is supported only in NGINX Plus. @@ -369,6 +372,8 @@ Note: This feature is supported only in NGINX Plus. |``statusMatch`` | The expected response status codes of a health check. By default, the response should have status code 2xx or 3xx. Examples: ``"200"``\ , ``"! 500"``\ , ``"301-303 307"``. See the documentation of the [match](https://nginx.org/en/docs/http/ngx_http_upstream_hc_module.html?#match) directive. This not supported for gRPC type upstreams. | ``string`` | No | |``grpcStatus`` | The expected [gRPC status code](https://github.com/grpc/grpc/blob/master/doc/statuscodes.md#status-codes-and-their-use-in-grpc) of the upstream server response to the [Check method](https://github.com/grpc/grpc/blob/master/doc/health-checking.md). Configure this field only if your gRPC services do not implement the gRPC health checking protocol. For example, configure ``12`` if the upstream server responds with `12 (UNIMPLEMENTED)` status code. Only valid on gRPC type upstreams. | ``int`` | No | |``grpcService`` | The gRPC service to be monitored on the upstream server. Only valid on gRPC type upstreams. | ``string`` | No | +|``mandatory`` | Require every newly added server to pass all configured health checks before NGINX Plus sends traffic to it. If this is not specified, or is set to false, the server will be initially considered healthy. When combined with [slow_start](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#slow_start), it gives a new server more time to connect to databases and “warm up” before being asked to handle their full share of traffic. | ``bool`` | No | +|``persistent`` | Set the initial “up” state for a server after reload if the server was considered healthy before reload. Enabling persistent requires that the mandatory parameter is also set to `true`. | ``bool`` | No | {{% /table %}} ### Upstream.SessionCookie diff --git a/internal/configs/version2/http.go b/internal/configs/version2/http.go index ce31fb7b87..1104429d6b 100644 --- a/internal/configs/version2/http.go +++ b/internal/configs/version2/http.go @@ -235,6 +235,8 @@ type HealthCheck struct { GRPCPass string GRPCStatus *int GRPCService string + Mandatory bool + Persistent bool } // TLSRedirect defines a redirect in a Server. diff --git a/internal/configs/version2/nginx-plus.virtualserver.tmpl b/internal/configs/version2/nginx-plus.virtualserver.tmpl index 50d3730e35..bd606f7259 100644 --- a/internal/configs/version2/nginx-plus.virtualserver.tmpl +++ b/internal/configs/version2/nginx-plus.virtualserver.tmpl @@ -219,6 +219,7 @@ server { {{ end }} health_check {{ if $hc.URI }}uri={{ $hc.URI }} {{ end }}port={{ $hc.Port }} interval={{ $hc.Interval }} jitter={{ $hc.Jitter }} fails={{ $hc.Fails }} passes={{ $hc.Passes }}{{ if $hc.Match }} match={{ $hc.Match }}{{ end }} + {{ if $hc.Mandatory }} mandatory{{ if $hc.Persistent }} persistent{{ end }}{{ end }} {{ if $hc.GRPCPass }} type=grpc{{ if $hc.GRPCStatus }} grpc_status={{ $hc.GRPCStatus }}{{ end }} {{ if $hc.GRPCService }} grpc_service={{ $hc.GRPCService }}{{ end }}{{ end }}; } diff --git a/internal/configs/version2/templates_test.go b/internal/configs/version2/templates_test.go index b4096603de..fa66161d8d 100644 --- a/internal/configs/version2/templates_test.go +++ b/internal/configs/version2/templates_test.go @@ -165,14 +165,16 @@ var virtualServerCfg = VirtualServerConfig{ }, HealthChecks: []HealthCheck{ { - Name: "coffee", - URI: "/", - Interval: "5s", - Jitter: "0s", - Fails: 1, - Passes: 1, - Port: 50, - ProxyPass: "http://coffee-v2", + Name: "coffee", + URI: "/", + Interval: "5s", + Jitter: "0s", + Fails: 1, + Passes: 1, + Port: 50, + ProxyPass: "http://coffee-v2", + Mandatory: true, + Persistent: true, }, { Name: "tea", diff --git a/internal/configs/virtualserver.go b/internal/configs/virtualserver.go index 19847dd7b0..34580e8b52 100644 --- a/internal/configs/virtualserver.go +++ b/internal/configs/virtualserver.go @@ -1310,6 +1310,10 @@ func generateHealthCheck( hc.Match = generateStatusMatchName(upstreamName) } + hc.Mandatory = upstream.HealthCheck.Mandatory + + hc.Persistent = upstream.HealthCheck.Persistent + hc.GRPCStatus = upstream.HealthCheck.GRPCStatus hc.GRPCService = upstream.HealthCheck.GRPCService diff --git a/internal/configs/virtualserver_test.go b/internal/configs/virtualserver_test.go index e1c0c9b968..a3a113b7d3 100644 --- a/internal/configs/virtualserver_test.go +++ b/internal/configs/virtualserver_test.go @@ -6465,6 +6465,35 @@ func TestGenerateHealthCheck(t *testing.T) { }, msg: "HealthCheck with time parameters have correct format", }, + { + upstream: conf_v1.Upstream{ + HealthCheck: &conf_v1.HealthCheck{ + Enable: true, + Mandatory: true, + Persistent: true, + }, + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "30s", + ProxySendTimeout: "30s", + }, + upstreamName: upstreamName, + expected: &version2.HealthCheck{ + Name: upstreamName, + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "30s", + ProxySendTimeout: "30s", + ProxyPass: fmt.Sprintf("http://%v", upstreamName), + URI: "/", + Interval: "5s", + Jitter: "0s", + Fails: 1, + Passes: 1, + Headers: make(map[string]string), + Mandatory: true, + Persistent: true, + }, + msg: "HealthCheck with mandatory and persistent set", + }, } baseCfgParams := &ConfigParams{ diff --git a/pkg/apis/configuration/v1/types.go b/pkg/apis/configuration/v1/types.go index fb40e407a9..2a73d72ba5 100644 --- a/pkg/apis/configuration/v1/types.go +++ b/pkg/apis/configuration/v1/types.go @@ -110,6 +110,8 @@ type HealthCheck struct { StatusMatch string `json:"statusMatch"` GRPCStatus *int `json:"grpcStatus"` GRPCService string `json:"grpcService"` + Mandatory bool `json:"mandatory"` + Persistent bool `json:"persistent"` } // Header defines an HTTP Header. diff --git a/pkg/apis/configuration/validation/virtualserver.go b/pkg/apis/configuration/validation/virtualserver.go index 21e32581d9..fa8d46d940 100644 --- a/pkg/apis/configuration/validation/virtualserver.go +++ b/pkg/apis/configuration/validation/virtualserver.go @@ -246,6 +246,10 @@ func validateUpstreamHealthCheck(hc *v1.HealthCheck, typeName string, fieldPath } } + if hc.Persistent && !hc.Mandatory { + allErrs = append(allErrs, field.Required(fieldPath.Child("mandatory"), "must be true when `persistent` is true")) + } + return allErrs } diff --git a/pkg/apis/configuration/validation/virtualserver_test.go b/pkg/apis/configuration/validation/virtualserver_test.go index fc7bd2f405..e66affce16 100644 --- a/pkg/apis/configuration/validation/virtualserver_test.go +++ b/pkg/apis/configuration/validation/virtualserver_test.go @@ -2344,6 +2344,8 @@ func TestValidateUpstreamHealthCheck(t *testing.T) { }, }, StatusMatch: "! 500", + Mandatory: true, + Persistent: true, } allErrs := validateUpstreamHealthCheck(hc, "", field.NewPath("healthCheck")) @@ -2427,6 +2429,13 @@ func TestValidateUpstreamHealthCheckFails(t *testing.T) { GRPCService: "tea-servicev2", }, }, + { + hc: &v1.HealthCheck{ + Enable: true, + Path: "/healthz", + Persistent: true, + }, + }, } for _, test := range tests { diff --git a/tests/data/virtual-server-upstream-options/plus-virtual-server-with-invalid-keys.yaml b/tests/data/virtual-server-upstream-options/plus-virtual-server-with-invalid-keys.yaml index 355b3f534f..748f4d449f 100644 --- a/tests/data/virtual-server-upstream-options/plus-virtual-server-with-invalid-keys.yaml +++ b/tests/data/virtual-server-upstream-options/plus-virtual-server-with-invalid-keys.yaml @@ -30,6 +30,7 @@ spec: statusMatch: "invalid" grpcService: "notimplemented" grpcStatus: 12 + persistent: True slow-start: "-3s" queue: size: -100 diff --git a/tests/suite/test_virtual_server_upstream_options.py b/tests/suite/test_virtual_server_upstream_options.py index b1d24384e0..5b0602a4a8 100644 --- a/tests/suite/test_virtual_server_upstream_options.py +++ b/tests/suite/test_virtual_server_upstream_options.py @@ -302,7 +302,7 @@ def test_openapi_validation_flow(self, kube_apis, ingress_controller_prerequisit class TestOptionsSpecificForPlus: @pytest.mark.parametrize('options, expected_strings', [ ({"lb-method": "least_conn", - "healthCheck": {"enable": True, "port": 8080}, + "healthCheck": {"enable": True, "port": 8080, "mandatory": True, "persistent": True}, "slow-start": "3h", "queue": {"size": 100}, "ntlm": True, @@ -311,7 +311,7 @@ class TestOptionsSpecificForPlus: "path": "/some-valid/path", "expires": "max", "domain": "virtual-server-route.example.com", "httpOnly": True, "secure": True}}, - ["health_check uri=/ port=8080 interval=5s jitter=0s", "fails=1 passes=1", ";", + ["health_check uri=/ port=8080 interval=5s jitter=0s", "fails=1 passes=1", "mandatory persistent", ";", "slow_start=3h", "queue 100 timeout=60s;", "ntlm;", "sticky cookie TestCookie expires=max domain=virtual-server-route.example.com httponly secure path=/some-valid/path;"]), ({"lb-method": "least_conn", @@ -398,7 +398,7 @@ def test_validation_flow(self, kube_apis, ingress_controller_prerequisites, "upstreams[0].healthCheck.read-timeout", "upstreams[0].healthCheck.send-timeout", "upstreams[0].healthCheck.headers[0].name", "upstreams[0].healthCheck.headers[0].value", "upstreams[0].healthCheck.statusMatch", "upstreams[0].healthCheck.grpcStatus", - "upstreams[0].healthCheck.grpcService", + "upstreams[0].healthCheck.grpcService", "upstreams[0].healthCheck.mandatory", "upstreams[0].slow-start", "upstreams[0].queue.size", "upstreams[0].queue.timeout", "upstreams[0].sessionCookie.name", "upstreams[0].sessionCookie.path",