diff --git a/deployments/common/crds/k8s.nginx.org_virtualserverroutes.yaml b/deployments/common/crds/k8s.nginx.org_virtualserverroutes.yaml index a8e897f249..5fe25724a0 100644 --- a/deployments/common/crds/k8s.nginx.org_virtualserverroutes.yaml +++ b/deployments/common/crds/k8s.nginx.org_virtualserverroutes.yaml @@ -515,6 +515,8 @@ spec: type: string jitter: type: string + keepalive-time: + type: string mandatory: type: boolean passes: diff --git a/deployments/common/crds/k8s.nginx.org_virtualservers.yaml b/deployments/common/crds/k8s.nginx.org_virtualservers.yaml index 327dbfaca2..edada506a8 100644 --- a/deployments/common/crds/k8s.nginx.org_virtualservers.yaml +++ b/deployments/common/crds/k8s.nginx.org_virtualservers.yaml @@ -597,6 +597,8 @@ spec: type: string jitter: type: string + keepalive-time: + type: string mandatory: type: boolean passes: diff --git a/deployments/helm-chart/crds/k8s.nginx.org_virtualserverroutes.yaml b/deployments/helm-chart/crds/k8s.nginx.org_virtualserverroutes.yaml index a8e897f249..5fe25724a0 100644 --- a/deployments/helm-chart/crds/k8s.nginx.org_virtualserverroutes.yaml +++ b/deployments/helm-chart/crds/k8s.nginx.org_virtualserverroutes.yaml @@ -515,6 +515,8 @@ spec: type: string jitter: type: string + keepalive-time: + type: string mandatory: type: boolean passes: diff --git a/deployments/helm-chart/crds/k8s.nginx.org_virtualservers.yaml b/deployments/helm-chart/crds/k8s.nginx.org_virtualservers.yaml index 327dbfaca2..edada506a8 100644 --- a/deployments/helm-chart/crds/k8s.nginx.org_virtualservers.yaml +++ b/deployments/helm-chart/crds/k8s.nginx.org_virtualservers.yaml @@ -597,6 +597,8 @@ spec: type: string jitter: type: string + keepalive-time: + type: string mandatory: type: boolean passes: diff --git a/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md b/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md index c51b0feb80..c2407678ed 100644 --- a/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md +++ b/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md @@ -411,6 +411,7 @@ healthCheck: statusMatch: "! 500" mandatory: true persistent: true + keepalive-time: 60s ``` Note: This feature is supported only in NGINX Plus. @@ -435,6 +436,7 @@ Note: This feature is supported only in NGINX Plus. |``grpcService`` | The gRPC service to be monitored on the upstream server. Only valid on gRPC type upstreams. | ``string`` | No | |``mandatory`` | Require every newly added server to pass all configured health checks before NGINX Plus sends traffic to it. If this is not specified, or is set to false, the server will be initially considered healthy. When combined with [slow-start](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#slow_start), it gives a new server more time to connect to databases and “warm up” before being asked to handle their full share of traffic. | ``bool`` | No | |``persistent`` | Set the initial “up” state for a server after reload if the server was considered healthy before reload. Enabling persistent requires that the mandatory parameter is also set to `true`. | ``bool`` | No | +|``keepalive-time`` | Enables [keepalive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) connections for health checks and specifies the time during which requests can be processed through one keepalive connection. The default is ``60s``. | ``string`` | No | {{% /table %}} ### Upstream.SessionCookie diff --git a/examples/custom-resources/health-checks/README.md b/examples/custom-resources/health-checks/README.md index fd7178bc05..6608c286b3 100644 --- a/examples/custom-resources/health-checks/README.md +++ b/examples/custom-resources/health-checks/README.md @@ -26,6 +26,7 @@ spec: path: /healthz interval: 20s jitter: 3s + keep_alive: 120s fails: 5 passes: 5 port: 8080 diff --git a/internal/configs/version2/http.go b/internal/configs/version2/http.go index cb8bfa810f..325dafba7a 100644 --- a/internal/configs/version2/http.go +++ b/internal/configs/version2/http.go @@ -258,6 +258,7 @@ type HealthCheck struct { GRPCService string Mandatory bool Persistent bool + KeepaliveTime string } // TLSRedirect defines a redirect in a Server. diff --git a/internal/configs/version2/nginx-plus.virtualserver.tmpl b/internal/configs/version2/nginx-plus.virtualserver.tmpl index 84f304f526..3936fafd5d 100644 --- a/internal/configs/version2/nginx-plus.virtualserver.tmpl +++ b/internal/configs/version2/nginx-plus.virtualserver.tmpl @@ -292,7 +292,7 @@ server { fails={{ $hc.Fails }} passes={{ $hc.Passes }}{{ if $hc.Match }} match={{ $hc.Match }}{{ end }} {{ if $hc.Mandatory }} mandatory{{ if $hc.Persistent }} persistent{{ end }}{{ end }} {{ if $hc.GRPCPass }} type=grpc{{ if $hc.GRPCStatus }} grpc_status={{ $hc.GRPCStatus }}{{ end }} - {{ if $hc.GRPCService }} grpc_service={{ $hc.GRPCService }}{{ end }}{{ end }}; + {{ if $hc.GRPCService }} grpc_service={{ $hc.GRPCService }}{{ end }}{{ end }} keepalive_time={{ $hc.KeepaliveTime }}; } {{ end }} diff --git a/internal/configs/virtualserver.go b/internal/configs/virtualserver.go index 91631f1a45..83dbb7030f 100644 --- a/internal/configs/virtualserver.go +++ b/internal/configs/virtualserver.go @@ -211,6 +211,7 @@ func newHealthCheckWithDefaults(upstream conf_v1.Upstream, upstreamName string, URI: uri, Interval: "5s", Jitter: "0s", + KeepaliveTime: "60s", Fails: 1, Passes: 1, ProxyPass: fmt.Sprintf("%v://%v", generateProxyPassProtocol(upstream.TLS.Enable), upstreamName), @@ -1396,6 +1397,10 @@ func generateHealthCheck( hc.Jitter = generateTime(upstream.HealthCheck.Jitter) } + if upstream.HealthCheck.KeepaliveTime != "" { + hc.KeepaliveTime = generateTime(upstream.HealthCheck.KeepaliveTime) + } + if upstream.HealthCheck.Fails > 0 { hc.Fails = upstream.HealthCheck.Fails } diff --git a/internal/configs/virtualserver_test.go b/internal/configs/virtualserver_test.go index 7b93b36ee7..85c57f51a9 100644 --- a/internal/configs/virtualserver_test.go +++ b/internal/configs/virtualserver_test.go @@ -7100,6 +7100,7 @@ func TestNewHealthCheckWithDefaults(t *testing.T) { URI: "/", Interval: "5s", Jitter: "0s", + KeepaliveTime: "60s", Fails: 1, Passes: 1, Headers: make(map[string]string), @@ -7128,6 +7129,7 @@ func TestGenerateHealthCheck(t *testing.T) { Path: "/healthz", Interval: "5s", Jitter: "2s", + KeepaliveTime: "120s", Fails: 3, Passes: 2, Port: 8080, @@ -7157,6 +7159,7 @@ func TestGenerateHealthCheck(t *testing.T) { URI: "/healthz", Interval: "5s", Jitter: "2s", + KeepaliveTime: "120s", Fails: 3, Passes: 2, Port: 8080, @@ -7187,6 +7190,7 @@ func TestGenerateHealthCheck(t *testing.T) { URI: "/", Interval: "5s", Jitter: "0s", + KeepaliveTime: "60s", Fails: 1, Passes: 1, Headers: make(map[string]string), @@ -7209,6 +7213,7 @@ func TestGenerateHealthCheck(t *testing.T) { URI: "/", Interval: "5s", Jitter: "0s", + KeepaliveTime: "60s", Fails: 1, Passes: 1, Headers: make(map[string]string), @@ -7227,6 +7232,7 @@ func TestGenerateHealthCheck(t *testing.T) { Enable: true, Interval: "1m 5s", Jitter: "2m 3s", + KeepaliveTime: "1m 6s", ConnectTimeout: "1m 10s", SendTimeout: "1m 20s", ReadTimeout: "1m 30s", @@ -7242,6 +7248,7 @@ func TestGenerateHealthCheck(t *testing.T) { URI: "/", Interval: "1m5s", Jitter: "2m3s", + KeepaliveTime: "1m6s", Fails: 1, Passes: 1, Headers: make(map[string]string), @@ -7269,6 +7276,7 @@ func TestGenerateHealthCheck(t *testing.T) { URI: "/", Interval: "5s", Jitter: "0s", + KeepaliveTime: "60s", Fails: 1, Passes: 1, Headers: make(map[string]string), @@ -7308,6 +7316,7 @@ func TestGenerateGrpcHealthCheck(t *testing.T) { Enable: true, Interval: "5s", Jitter: "2s", + KeepaliveTime: "120s", Fails: 3, Passes: 2, Port: 50051, @@ -7339,6 +7348,7 @@ func TestGenerateGrpcHealthCheck(t *testing.T) { GRPCPass: fmt.Sprintf("grpc://%v", upstreamName), Interval: "5s", Jitter: "2s", + KeepaliveTime: "120s", Fails: 3, Passes: 2, Port: 50051, @@ -7371,6 +7381,7 @@ func TestGenerateGrpcHealthCheck(t *testing.T) { GRPCPass: fmt.Sprintf("grpc://%v", upstreamName), Interval: "5s", Jitter: "0s", + KeepaliveTime: "60s", Fails: 1, Passes: 1, Headers: make(map[string]string), diff --git a/pkg/apis/configuration/v1/types.go b/pkg/apis/configuration/v1/types.go index 6bcfa5ca33..0c5170b6a6 100644 --- a/pkg/apis/configuration/v1/types.go +++ b/pkg/apis/configuration/v1/types.go @@ -141,6 +141,7 @@ type HealthCheck struct { GRPCService string `json:"grpcService"` Mandatory bool `json:"mandatory"` Persistent bool `json:"persistent"` + KeepaliveTime string `json:"keepalive-time"` } // Header defines an HTTP Header. diff --git a/pkg/apis/configuration/validation/virtualserver.go b/pkg/apis/configuration/validation/virtualserver.go index 37f6286e93..1ac7be51e8 100644 --- a/pkg/apis/configuration/validation/virtualserver.go +++ b/pkg/apis/configuration/validation/virtualserver.go @@ -343,6 +343,7 @@ func validateUpstreamHealthCheck(hc *v1.HealthCheck, typeName string, fieldPath allErrs = append(allErrs, validateTime(hc.ReadTimeout, fieldPath.Child("read-timeout"))...) allErrs = append(allErrs, validateTime(hc.SendTimeout, fieldPath.Child("send-timeout"))...) allErrs = append(allErrs, validateStatusMatch(hc.StatusMatch, fieldPath.Child("statusMatch"))...) + allErrs = append(allErrs, validateTime(hc.KeepaliveTime, fieldPath.Child("keepalive-time"))...) for i, header := range hc.Headers { idxPath := fieldPath.Child("headers").Index(i) diff --git a/pkg/apis/configuration/validation/virtualserver_test.go b/pkg/apis/configuration/validation/virtualserver_test.go index 39dd19ee4b..db8f268cba 100644 --- a/pkg/apis/configuration/validation/virtualserver_test.go +++ b/pkg/apis/configuration/validation/virtualserver_test.go @@ -2506,6 +2506,7 @@ func TestValidateUpstreamHealthCheck(t *testing.T) { StatusMatch: "! 500", Mandatory: true, Persistent: true, + KeepaliveTime: "120s", } allErrs := validateUpstreamHealthCheck(hc, "", field.NewPath("healthCheck")) diff --git a/tests/data/virtual-server-upstream-options/plus-virtual-server-with-invalid-keys-openapi.yaml b/tests/data/virtual-server-upstream-options/plus-virtual-server-with-invalid-keys-openapi.yaml index 85bfd52575..c1aa5121ee 100644 --- a/tests/data/virtual-server-upstream-options/plus-virtual-server-with-invalid-keys-openapi.yaml +++ b/tests/data/virtual-server-upstream-options/plus-virtual-server-with-invalid-keys-openapi.yaml @@ -20,6 +20,7 @@ spec: path: 200 interval: 1.5 jitter: 2.0 + keepalive-time: 2.0 fails: "one" passes: "one" port: "80" @@ -49,6 +50,7 @@ spec: path: 200 interval: 1.5 jitter: 2.0 + keepalive-time: 2.0 fails: "one" passes: "one" port: "80" diff --git a/tests/data/virtual-server-upstream-options/plus-virtual-server-with-invalid-keys.yaml b/tests/data/virtual-server-upstream-options/plus-virtual-server-with-invalid-keys.yaml index c65861e0ee..c8bd740be6 100644 --- a/tests/data/virtual-server-upstream-options/plus-virtual-server-with-invalid-keys.yaml +++ b/tests/data/virtual-server-upstream-options/plus-virtual-server-with-invalid-keys.yaml @@ -18,6 +18,7 @@ spec: path: "invalid" interval: 1.5d jitter: invalid + keepalive-time: 1.3d fails: -5 passes: -1 port: -1 @@ -48,6 +49,7 @@ spec: path: "invalid" interval: 1.5d jitter: invalid + keepalive-time: 1.3d fails: -5 passes: -1 port: -1 diff --git a/tests/suite/test_virtual_server_grpc.py b/tests/suite/test_virtual_server_grpc.py index 833fcfbf6e..36760127a1 100644 --- a/tests/suite/test_virtual_server_grpc.py +++ b/tests/suite/test_virtual_server_grpc.py @@ -295,7 +295,7 @@ def test_config_after_enable_healthcheck( param_list = [ "health_check port=50051 interval=1s jitter=2s", "type=grpc grpc_status=12", - "grpc_service=helloworld.Greeter;", + "grpc_service=helloworld.Greeter keepalive_time=60s;", ] for p in param_list: assert p in config diff --git a/tests/suite/test_virtual_server_upstream_options.py b/tests/suite/test_virtual_server_upstream_options.py index c874ae249d..491942fd3e 100644 --- a/tests/suite/test_virtual_server_upstream_options.py +++ b/tests/suite/test_virtual_server_upstream_options.py @@ -507,7 +507,7 @@ class TestOptionsSpecificForPlus: "health_check uri=/ interval=5s jitter=0s", "fails=1 passes=1", "mandatory persistent", - ";", + "keepalive_time=60s;", "slow_start=3h", "queue 100 timeout=60s;", "ntlm;", @@ -531,6 +531,7 @@ class TestOptionsSpecificForPlus: "read-timeout": "45s", "send-timeout": "55s", "headers": [{"name": "Host", "value": "virtual-server.example.com"}], + "keepalive-time": "120s", }, "queue": {"size": 1000, "timeout": "66s"}, "slow-start": "0s", @@ -544,7 +545,8 @@ class TestOptionsSpecificForPlus: "proxy_connect_timeout 35s;", "proxy_read_timeout 45s;", "proxy_send_timeout 55s;", - 'proxy_set_header Host "virtual-server.example.com";', + 'proxy_set_header Host "virtual-server.example.com"', + "keepalive_time=120s;", "slow_start=0s", "queue 1000 timeout=66s;", "ntlm;", @@ -632,6 +634,7 @@ def test_validation_flow( "upstreams[0].healthCheck.path", "upstreams[0].healthCheck.interval", "upstreams[0].healthCheck.jitter", + "upstreams[0].healthCheck.keepalive-time", "upstreams[0].healthCheck.fails", "upstreams[0].healthCheck.passes", "upstreams[0].healthCheck.connect-timeout", @@ -653,6 +656,7 @@ def test_validation_flow( "upstreams[1].healthCheck.path", "upstreams[1].healthCheck.interval", "upstreams[1].healthCheck.jitter", + "upstreams[1].healthCheck.keepalive-time", "upstreams[1].healthCheck.fails", "upstreams[1].healthCheck.passes", "upstreams[1].healthCheck.connect-timeout", @@ -693,6 +697,7 @@ def test_openapi_validation_flow( "healthCheck.path", "healthCheck.interval", "healthCheck.jitter", + "healthCheck.keepalive-time", "healthCheck.fails", "healthCheck.passes", "healthCheck.port",