diff --git a/deployments/common/crds-v1beta1/k8s.nginx.org_transportservers.yaml b/deployments/common/crds-v1beta1/k8s.nginx.org_transportservers.yaml index 1d5440fc45..7e06c61212 100644 --- a/deployments/common/crds-v1beta1/k8s.nginx.org_transportservers.yaml +++ b/deployments/common/crds-v1beta1/k8s.nginx.org_transportservers.yaml @@ -49,10 +49,24 @@ spec: type: string protocol: type: string + sessionParameters: + description: SessionParameters defines session parameters. + type: object + properties: + timeout: + type: string upstreamParameters: description: UpstreamParameters defines parameters for an upstream. type: object properties: + connectTimeout: + type: string + nextUpstream: + type: boolean + nextUpstreamTimeout: + type: string + nextUpstreamTries: + type: integer udpRequests: type: integer udpResponses: diff --git a/deployments/common/crds/k8s.nginx.org_transportservers.yaml b/deployments/common/crds/k8s.nginx.org_transportservers.yaml index 954e849b13..880d4c3bbe 100644 --- a/deployments/common/crds/k8s.nginx.org_transportservers.yaml +++ b/deployments/common/crds/k8s.nginx.org_transportservers.yaml @@ -50,10 +50,24 @@ spec: type: string protocol: type: string + sessionParameters: + description: SessionParameters defines session parameters. + type: object + properties: + timeout: + type: string upstreamParameters: description: UpstreamParameters defines parameters for an upstream. type: object properties: + connectTimeout: + type: string + nextUpstream: + type: boolean + nextUpstreamTimeout: + type: string + nextUpstreamTries: + type: integer udpRequests: type: integer udpResponses: diff --git a/deployments/helm-chart/crds/k8s.nginx.org_transportservers.yaml b/deployments/helm-chart/crds/k8s.nginx.org_transportservers.yaml index 1d5440fc45..7e06c61212 100644 --- a/deployments/helm-chart/crds/k8s.nginx.org_transportservers.yaml +++ b/deployments/helm-chart/crds/k8s.nginx.org_transportservers.yaml @@ -49,10 +49,24 @@ spec: type: string protocol: type: string + sessionParameters: + description: SessionParameters defines session parameters. + type: object + properties: + timeout: + type: string upstreamParameters: description: UpstreamParameters defines parameters for an upstream. type: object properties: + connectTimeout: + type: string + nextUpstream: + type: boolean + nextUpstreamTimeout: + type: string + nextUpstreamTries: + type: integer udpRequests: type: integer udpResponses: diff --git a/docs-web/configuration/transportserver-resource.md b/docs-web/configuration/transportserver-resource.md index e1ab57c45b..dbd923f098 100644 --- a/docs-web/configuration/transportserver-resource.md +++ b/docs-web/configuration/transportserver-resource.md @@ -15,6 +15,7 @@ This document is the reference documentation for the TransportServer resource. T - [Listener](#listener) - [Upstream](#upstream) - [UpstreamParameters](#upstreamparameters) + - [SessionParameters](#sessionparameters) - [Action](#action) - [Using TransportServer](#using-transportserver) - [Validation](#validation) @@ -182,11 +183,15 @@ port: 8443 ### UpstreamParameters -The upstream parameters define various parameters for the upstreams. For now, only UDP-related parameters are supported: +The upstream parameters define various parameters for the upstreams: ```yaml upstreamParameters: udpRequests: 1 udpResponses: 1 + connectTimeout: 60s + nextUpstream: true + nextUpstreamTimeout: 50s + nextUpstreamTries: 1 ``` ```eval_rst @@ -205,6 +210,44 @@ upstreamParameters: - The number of datagrams expected from the proxied server in response to a client datagram. See the `proxy_responses <https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_responses>`_ directive. By default, the number of datagrams is not limited. - ``int`` - No + * - ``connectTimeout`` + - The timeout for establishing a connection with a proxied server. See the `proxy_connect_timeout <http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_connect_timeout>`_ directive. The default is ``60s``. + - ``string`` + - No + * - ``nextUpstream`` + - If a connection to the proxied server cannot be established, determines whether a client connection will be passed to the next server. See the `proxy_next_upstream <http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream>`_ directive. The default is ``true``. + - bool + - No + * - ``nextUpstreamTries`` + - The number of tries for passing a connection to the next server. See the `proxy_next_upstream_tries <http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream_tries>`_ directive. The default is ``0``. + - ``int`` + - No + * - ``nextUpstreamTimeout`` + - The time allowed to pass a connection to the next server. See the `proxy_next_upstream_timeout <http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream_timeout>`_ directive. The default us ``0``. + - ``string`` + - No +``` + +### SessionParameters + +The session parameters define various parameters for TCP connections and UDP sessions. +```yaml +sessionParameters: + timeout: 50s +``` + +```eval_rst +.. list-table:: + :header-rows: 1 + + * - Field + - Description + - Type + - Required + * - ``timeout`` + - The timeout between two succesive read or write operations on client or proxied server connections. See `proxy_timeout <http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_timeout>`_ directive. The default is ``10m``. + - ``string`` + - No ``` ### Action @@ -312,4 +355,4 @@ The [ConfigMap](/nginx-ingress-controller/configuration/global-configuration/con As of Release 1.7, the TransportServer resource is a preview feature. Currently, it comes with the following limitations: * When using TLS Passthrough, it is not possible to configure [Proxy Protocol](https://github.com/nginxinc/kubernetes-ingress/tree/master/examples/proxy-protocol) for port 443 both for regular HTTPS and TLS Passthrough traffic. * If multiple TCP (or UDP) TransportServers reference the same listener, only one of them will receive the traffic. Moreover, until there is only one TransportServer, NGINX will fail to reload. If this happens, the IC will report a warning event with the `AddedOrUpdatedWithError` reason for the resource, which caused the problem, and also report the error in the logs. -* If multiple TLS Passthrough TransportServers have the same hostname, only one of them will receive the traffic. If this happens, the IC will report a warning in the logs like `host "app.example.com" is used by more than one TransportServers`. \ No newline at end of file +* If multiple TLS Passthrough TransportServers have the same hostname, only one of them will receive the traffic. If this happens, the IC will report a warning in the logs like `host "app.example.com" is used by more than one TransportServers`. diff --git a/internal/configs/transportserver.go b/internal/configs/transportserver.go index b0b701327c..6951a1a5bd 100644 --- a/internal/configs/transportserver.go +++ b/internal/configs/transportserver.go @@ -35,10 +35,27 @@ func generateTransportServerConfig(transportServerEx *TransportServerEx, listene upstreams := generateStreamUpstreams(transportServerEx, upstreamNamer, isPlus) var proxyRequests, proxyResponses *int + var connectTimeout, nextUpstreamTimeout string + var nextUpstream bool + var nextUpstreamTries int if transportServerEx.TransportServer.Spec.UpstreamParameters != nil { proxyRequests = transportServerEx.TransportServer.Spec.UpstreamParameters.UDPRequests proxyResponses = transportServerEx.TransportServer.Spec.UpstreamParameters.UDPResponses + + nextUpstream = transportServerEx.TransportServer.Spec.UpstreamParameters.NextUpstream + if nextUpstream { + nextUpstreamTries = transportServerEx.TransportServer.Spec.UpstreamParameters.NextUpstreamTries + nextUpstreamTimeout = transportServerEx.TransportServer.Spec.UpstreamParameters.NextUpstreamTimeout + } + + connectTimeout = transportServerEx.TransportServer.Spec.UpstreamParameters.ConnectTimeout } + + var proxyTimeout string + if transportServerEx.TransportServer.Spec.SessionParameters != nil { + proxyTimeout = transportServerEx.TransportServer.Spec.SessionParameters.Timeout + } + statusZone := "" if transportServerEx.TransportServer.Spec.Listener.Name == conf_v1alpha1.TLSPassthroughListenerName { statusZone = transportServerEx.TransportServer.Spec.Host @@ -48,16 +65,21 @@ func generateTransportServerConfig(transportServerEx *TransportServerEx, listene return version2.TransportServerConfig{ Server: version2.StreamServer{ - TLSPassthrough: transportServerEx.TransportServer.Spec.Listener.Name == conf_v1alpha1.TLSPassthroughListenerName, - UnixSocket: generateUnixSocket(transportServerEx), - Port: listenerPort, - UDP: transportServerEx.TransportServer.Spec.Listener.Protocol == "UDP", - StatusZone: statusZone, - ProxyRequests: proxyRequests, - ProxyResponses: proxyResponses, - ProxyPass: upstreamNamer.GetNameForUpstream(transportServerEx.TransportServer.Spec.Action.Pass), - Name: transportServerEx.TransportServer.Name, - Namespace: transportServerEx.TransportServer.Namespace, + TLSPassthrough: transportServerEx.TransportServer.Spec.Listener.Name == conf_v1alpha1.TLSPassthroughListenerName, + UnixSocket: generateUnixSocket(transportServerEx), + Port: listenerPort, + UDP: transportServerEx.TransportServer.Spec.Listener.Protocol == "UDP", + StatusZone: statusZone, + ProxyRequests: proxyRequests, + ProxyResponses: proxyResponses, + ProxyPass: upstreamNamer.GetNameForUpstream(transportServerEx.TransportServer.Spec.Action.Pass), + Name: transportServerEx.TransportServer.Name, + Namespace: transportServerEx.TransportServer.Namespace, + ProxyConnectTimeout: generateString(connectTimeout, "60s"), + ProxyTimeout: generateString(proxyTimeout, "10m"), + ProxyNextUpstream: nextUpstream, + ProxyNextUpstreamTimeout: generateString(nextUpstreamTimeout, "0"), + ProxyNextUpstreamTries: nextUpstreamTries, }, Upstreams: upstreams, } diff --git a/internal/configs/transportserver_test.go b/internal/configs/transportserver_test.go index a21c1c5a99..8a13df31c9 100644 --- a/internal/configs/transportserver_test.go +++ b/internal/configs/transportserver_test.go @@ -80,6 +80,13 @@ func TestGenerateTransportServerConfigForTCP(t *testing.T) { Port: 5001, }, }, + UpstreamParameters: &conf_v1alpha1.UpstreamParameters{ + ConnectTimeout: "30s", + NextUpstream: false, + }, + SessionParameters: &conf_v1alpha1.SessionParameters{ + Timeout: "50s", + }, Action: &conf_v1alpha1.Action{ Pass: "tcp-app", }, @@ -112,12 +119,17 @@ func TestGenerateTransportServerConfigForTCP(t *testing.T) { }, }, Server: version2.StreamServer{ - Port: 2020, - UDP: false, - StatusZone: "tcp-listener", - ProxyPass: "ts_default_tcp-server_tcp-app", - Name: "tcp-server", - Namespace: "default", + Port: 2020, + UDP: false, + StatusZone: "tcp-listener", + ProxyPass: "ts_default_tcp-server_tcp-app", + Name: "tcp-server", + Namespace: "default", + ProxyConnectTimeout: "30s", + ProxyNextUpstream: false, + ProxyNextUpstreamTries: 0, + ProxyNextUpstreamTimeout: "0", + ProxyTimeout: "50s", }, } @@ -148,6 +160,12 @@ func TestGenerateTransportServerConfigForTLSPasstrhough(t *testing.T) { Port: 5001, }, }, + UpstreamParameters: &conf_v1alpha1.UpstreamParameters{ + ConnectTimeout: "30s", + NextUpstream: false, + NextUpstreamTries: 0, + NextUpstreamTimeout: "", + }, Action: &conf_v1alpha1.Action{ Pass: "tcp-app", }, @@ -180,14 +198,19 @@ func TestGenerateTransportServerConfigForTLSPasstrhough(t *testing.T) { }, }, Server: version2.StreamServer{ - TLSPassthrough: true, - UnixSocket: "unix:/var/lib/nginx/passthrough-default_tcp-server.sock", - Port: 2020, - UDP: false, - StatusZone: "example.com", - ProxyPass: "ts_default_tcp-server_tcp-app", - Name: "tcp-server", - Namespace: "default", + TLSPassthrough: true, + UnixSocket: "unix:/var/lib/nginx/passthrough-default_tcp-server.sock", + Port: 2020, + UDP: false, + StatusZone: "example.com", + ProxyPass: "ts_default_tcp-server_tcp-app", + Name: "tcp-server", + Namespace: "default", + ProxyConnectTimeout: "30s", + ProxyNextUpstream: false, + ProxyNextUpstreamTimeout: "0", + ProxyNextUpstreamTries: 0, + ProxyTimeout: "10m", }, } @@ -221,8 +244,12 @@ func TestGenerateTransportServerConfigForUDP(t *testing.T) { }, }, UpstreamParameters: &conf_v1alpha1.UpstreamParameters{ - UDPRequests: &udpRequests, - UDPResponses: &udpResponses, + UDPRequests: &udpRequests, + UDPResponses: &udpResponses, + ConnectTimeout: "30s", + NextUpstream: true, + NextUpstreamTimeout: "", + NextUpstreamTries: 0, }, Action: &conf_v1alpha1.Action{ Pass: "udp-app", @@ -256,14 +283,19 @@ func TestGenerateTransportServerConfigForUDP(t *testing.T) { }, }, Server: version2.StreamServer{ - Port: 2020, - UDP: true, - StatusZone: "udp-listener", - ProxyRequests: &udpRequests, - ProxyResponses: &udpResponses, - ProxyPass: "ts_default_udp-server_udp-app", - Name: "udp-server", - Namespace: "default", + Port: 2020, + UDP: true, + StatusZone: "udp-listener", + ProxyRequests: &udpRequests, + ProxyResponses: &udpResponses, + ProxyPass: "ts_default_udp-server_udp-app", + Name: "udp-server", + Namespace: "default", + ProxyConnectTimeout: "30s", + ProxyNextUpstream: true, + ProxyNextUpstreamTimeout: "0", + ProxyNextUpstreamTries: 0, + ProxyTimeout: "10m", }, } diff --git a/internal/configs/version2/nginx-plus.transportserver.tmpl b/internal/configs/version2/nginx-plus.transportserver.tmpl index bb92cd0cfe..013d963e39 100644 --- a/internal/configs/version2/nginx-plus.transportserver.tmpl +++ b/internal/configs/version2/nginx-plus.transportserver.tmpl @@ -29,4 +29,13 @@ server { {{ end }} proxy_pass {{ $s.ProxyPass }}; + + proxy_timeout {{ $s.ProxyTimeout }}; + proxy_connect_timeout {{ $s.ProxyConnectTimeout }}; + + {{ if $s.ProxyNextUpstream }} + proxy_next_upstream on; + proxy_next_upstream_timeout {{ $s.ProxyNextUpstreamTimeout }}; + proxy_next_upstream_tries {{ $s.ProxyNextUpstreamTries }}; + {{ end }} } \ No newline at end of file diff --git a/internal/configs/version2/nginx.transportserver.tmpl b/internal/configs/version2/nginx.transportserver.tmpl index d19bfca69b..2f73d6cb04 100644 --- a/internal/configs/version2/nginx.transportserver.tmpl +++ b/internal/configs/version2/nginx.transportserver.tmpl @@ -27,4 +27,13 @@ server { {{ end }} proxy_pass {{ $s.ProxyPass }}; + + proxy_timeout {{ $s.ProxyTimeout }}; + proxy_connect_timeout {{ $s.ProxyConnectTimeout }}; + + {{ if $s.ProxyNextUpstream }} + proxy_next_upstream on; + proxy_next_upstream_timeout {{ $s.ProxyNextUpstreamTimeout }}; + proxy_next_upstream_tries {{ $s.ProxyNextUpstreamTries }}; + {{ end }} } \ No newline at end of file diff --git a/internal/configs/version2/stream.go b/internal/configs/version2/stream.go index e700337389..5bd6bc5888 100644 --- a/internal/configs/version2/stream.go +++ b/internal/configs/version2/stream.go @@ -20,16 +20,21 @@ type StreamUpstreamServer struct { // StreamServer defines a server in the stream module. type StreamServer struct { - TLSPassthrough bool - UnixSocket string - Port int - UDP bool - StatusZone string - ProxyRequests *int - ProxyResponses *int - ProxyPass string - Name string - Namespace string + TLSPassthrough bool + UnixSocket string + Port int + UDP bool + StatusZone string + ProxyRequests *int + ProxyResponses *int + ProxyPass string + Name string + Namespace string + ProxyTimeout string + ProxyConnectTimeout string + ProxyNextUpstream bool + ProxyNextUpstreamTimeout string + ProxyNextUpstreamTries int } // TLSPassthroughHostsConfig defines a mapping between TLS Passthrough hosts and the corresponding unix sockets. diff --git a/internal/configs/version2/templates_test.go b/internal/configs/version2/templates_test.go index 6e9c5d1446..d8cc94a466 100644 --- a/internal/configs/version2/templates_test.go +++ b/internal/configs/version2/templates_test.go @@ -326,12 +326,17 @@ var transportServerCfg = TransportServerConfig{ }, }, Server: StreamServer{ - Port: 1234, - UDP: true, - StatusZone: "udp-app", - ProxyRequests: createPointerFromInt(1), - ProxyResponses: createPointerFromInt(2), - ProxyPass: "udp-upstream", + Port: 1234, + UDP: true, + StatusZone: "udp-app", + ProxyRequests: createPointerFromInt(1), + ProxyResponses: createPointerFromInt(2), + ProxyPass: "udp-upstream", + ProxyTimeout: "10s", + ProxyConnectTimeout: "10s", + ProxyNextUpstream: true, + ProxyNextUpstreamTimeout: "10s", + ProxyNextUpstreamTries: 5, }, } diff --git a/pkg/apis/configuration/v1alpha1/types.go b/pkg/apis/configuration/v1alpha1/types.go index 795cbd02b0..16d04e1bd4 100644 --- a/pkg/apis/configuration/v1alpha1/types.go +++ b/pkg/apis/configuration/v1alpha1/types.go @@ -65,6 +65,7 @@ type TransportServerSpec struct { Host string `json:"host"` Upstreams []Upstream `json:"upstreams"` UpstreamParameters *UpstreamParameters `json:"upstreamParameters"` + SessionParameters *SessionParameters `json:"sessionParameters"` Action *Action `json:"action"` } @@ -85,6 +86,16 @@ type Upstream struct { type UpstreamParameters struct { UDPRequests *int `json:"udpRequests"` UDPResponses *int `json:"udpResponses"` + + ConnectTimeout string `json:"connectTimeout"` + NextUpstream bool `json:"nextUpstream"` + NextUpstreamTimeout string `json:"nextUpstreamTimeout"` + NextUpstreamTries int `json:"nextUpstreamTries"` +} + +// SessionParameters defines session parameters. +type SessionParameters struct { + Timeout string `json:"timeout"` } // Action defines an action. diff --git a/pkg/apis/configuration/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/configuration/v1alpha1/zz_generated.deepcopy.go index dd27745962..d1b7308cdd 100644 --- a/pkg/apis/configuration/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/configuration/v1alpha1/zz_generated.deepcopy.go @@ -121,6 +121,22 @@ func (in *Listener) DeepCopy() *Listener { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SessionParameters) DeepCopyInto(out *SessionParameters) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SessionParameters. +func (in *SessionParameters) DeepCopy() *SessionParameters { + if in == nil { + return nil + } + out := new(SessionParameters) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TransportServer) DeepCopyInto(out *TransportServer) { *out = *in @@ -211,6 +227,11 @@ func (in *TransportServerSpec) DeepCopyInto(out *TransportServerSpec) { *out = new(UpstreamParameters) (*in).DeepCopyInto(*out) } + if in.SessionParameters != nil { + in, out := &in.SessionParameters, &out.SessionParameters + *out = new(SessionParameters) + **out = **in + } if in.Action != nil { in, out := &in.Action, &out.Action *out = new(Action) diff --git a/pkg/apis/configuration/validation/transportserver.go b/pkg/apis/configuration/validation/transportserver.go index 68e280e540..163a493ad9 100644 --- a/pkg/apis/configuration/validation/transportserver.go +++ b/pkg/apis/configuration/validation/transportserver.go @@ -40,6 +40,8 @@ func (tsv *TransportServerValidator) validateTransportServerSpec(spec *v1alpha1. allErrs = append(allErrs, validateTransportServerUpstreamParameters(spec.UpstreamParameters, fieldPath.Child("upstreamParameters"), spec.Listener.Protocol)...) + allErrs = append(validateSessionParameters(spec.SessionParameters, fieldPath.Child("sessionParameters"))) + if spec.Action == nil { allErrs = append(allErrs, field.Required(fieldPath.Child("action"), "must specify action")) } else { @@ -164,6 +166,21 @@ func validateTransportServerUpstreamParameters(upstreamParameters *v1alpha1.Upst allErrs = append(allErrs, validateUDPUpstreamParameter(upstreamParameters.UDPRequests, fieldPath.Child("udpRequests"), protocol)...) allErrs = append(allErrs, validateUDPUpstreamParameter(upstreamParameters.UDPResponses, fieldPath.Child("udpResponses"), protocol)...) + allErrs = append(allErrs, validateTime(upstreamParameters.ConnectTimeout, fieldPath.Child("connectTimeout"))...) + allErrs = append(allErrs, validateTime(upstreamParameters.NextUpstreamTimeout, fieldPath.Child("nextUpstreamTimeout"))...) + allErrs = append(allErrs, validatePositiveIntOrZero(upstreamParameters.NextUpstreamTries, fieldPath.Child("nextUpstreamTries"))...) + + return allErrs +} + +func validateSessionParameters(sessionParameters *v1alpha1.SessionParameters, fieldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if sessionParameters == nil { + return allErrs + } + + allErrs = append(allErrs, validateTime(sessionParameters.Timeout, fieldPath.Child("timeout"))...) return allErrs } diff --git a/pkg/apis/configuration/validation/transportserver_test.go b/pkg/apis/configuration/validation/transportserver_test.go index f0b87d287a..2a30173ce3 100644 --- a/pkg/apis/configuration/validation/transportserver_test.go +++ b/pkg/apis/configuration/validation/transportserver_test.go @@ -410,6 +410,56 @@ func TestValidateUpstreamParameters(t *testing.T) { } } +func TestValidateSessionParameters(t *testing.T) { + tests := []struct { + parameters *v1alpha1.SessionParameters + msg string + }{ + { + parameters: nil, + msg: "nil parameters", + }, + { + parameters: &v1alpha1.SessionParameters{}, + msg: "Non-nil parameters", + }, + { + parameters: &v1alpha1.SessionParameters{ + Timeout: "60s", + }, + msg: "valid parameters", + }, + } + + for _, test := range tests { + allErrs := validateSessionParameters(test.parameters, field.NewPath("sessionParameters")) + if len(allErrs) > 0 { + t.Errorf("validateSessionParameters() returned errors %v for valid input for the case of %s", allErrs, test.msg) + } + } +} + +func TestValidateSessionParametersFails(t *testing.T) { + tests := []struct { + parameters *v1alpha1.SessionParameters + msg string + }{ + { + parameters: &v1alpha1.SessionParameters{ + Timeout: "-1s", + }, + msg: "invalid timeout", + }, + } + + for _, test := range tests { + allErrs := validateSessionParameters(test.parameters, field.NewPath("sessionParameters")) + if len(allErrs) == 0 { + t.Errorf("validateSessionParameters() returned no errors for invalid input: %v", test.msg) + } + } +} + func TestValidateUDPUpstreamParameter(t *testing.T) { validInput := []struct { parameter *int