Add command line argument to manually disable IPV6 listeners for unsu…

…pported clusters (#3040) * Add command line argument to disable IPV6 listeners * update helm chart and documentation * Add automated test * update documentation * fix import Co-authored-by: Venktesh <[email protected]>
nginx · Sep 22, 2022 · d78f57b · d78f57b
1 parent d18631e
commit d78f57b
Show file tree

Hide file tree

Showing 29 changed files with 347 additions and 20 deletions.
diff --git a/cmd/nginx-ingress/flags.go b/cmd/nginx-ingress/flags.go
@@ -180,6 +180,9 @@ var (
 	includeYearInLogs = flag.Bool("include-year", false,
 		"Option to include the year in the log header")
 
+	disableIPV6 = flag.Bool("disable-ipv6", false,
+		`Disable IPV6 listeners explicitly for nodes that do not support the IPV6 stack`)
+
 	startupCheckFn func() error
 )
 

diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go
@@ -75,6 +75,7 @@ func main() {
 	cfgParams = processConfigMaps(kubeClient, cfgParams, nginxManager, templateExecutor)
 
 	staticCfgParams := &configs.StaticConfigParams{
+		DisableIPV6:                    *disableIPV6,
 		HealthStatus:                   *healthStatus,
 		HealthStatusURI:                *healthStatusURI,
 		NginxStatus:                    *nginxStatus,
@@ -149,6 +150,7 @@ func main() {
 		SnippetsEnabled:              *enableSnippets,
 		CertManagerEnabled:           *enableCertManager,
 		ExternalDNSEnabled:           *enableExternalDNS,
+		IsIPV6Disabled:               *disableIPV6,
 	}
 
 	lbc := k8s.NewLoadBalancerController(lbcInput)

diff --git a/deployments/helm-chart/README.md b/deployments/helm-chart/README.md
@@ -245,6 +245,7 @@ Parameter | Description | Default
 `controller.enableLatencyMetrics` | Enable collection of latency metrics for upstreams. Requires `prometheus.create`. | false
 `controller.minReadySeconds` | Specifies the minimum number of seconds for which a newly created Pod should be ready without any of its containers crashing, for it to be considered available. [docs](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#min-ready-seconds) | 0
 `controller.strategy` | Specifies the strategy used to replace old Pods by new ones. [docs](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) | {}
+`controller.disableIPV6` | Disable IPV6 listeners explicitly for nodes that do not support the IPV6 stack. | false
 `rbac.create` | Configures RBAC. | true
 `prometheus.create` | Expose NGINX or NGINX Plus metrics in the Prometheus format. | false
 `prometheus.port` | Configures the port to scrape the metrics. | 9113

diff --git a/deployments/helm-chart/templates/controller-daemonset.yaml b/deployments/helm-chart/templates/controller-daemonset.yaml
@@ -192,6 +192,7 @@ spec:
           - -enable-custom-resources={{ .Values.controller.enableCustomResources }}
           - -enable-snippets={{ .Values.controller.enableSnippets }}
           - -include-year={{ .Values.controller.includeYear }}
+          - -disable-ipv6={{ .Values.controller.disableIPV6 }}
 {{- if .Values.controller.enableCustomResources }}
           - -enable-tls-passthrough={{ .Values.controller.enableTLSPassthrough }}
           - -enable-preview-policies={{ .Values.controller.enablePreviewPolicies }}

diff --git a/deployments/helm-chart/templates/controller-deployment.yaml b/deployments/helm-chart/templates/controller-deployment.yaml
@@ -195,6 +195,7 @@ spec:
           - -enable-custom-resources={{ .Values.controller.enableCustomResources }}
           - -enable-snippets={{ .Values.controller.enableSnippets }}
           - -include-year={{ .Values.controller.includeYear }}
+          - -disable-ipv6={{ .Values.controller.disableIPV6 }}
 {{- if .Values.controller.enableCustomResources }}
           - -enable-tls-passthrough={{ .Values.controller.enableTLSPassthrough }}
           - -enable-preview-policies={{ .Values.controller.enablePreviewPolicies }}

diff --git a/deployments/helm-chart/values.yaml b/deployments/helm-chart/values.yaml
@@ -382,6 +382,9 @@ controller:
   ## Enable collection of latency metrics for upstreams. Requires prometheus.create.
   enableLatencyMetrics: false
 
+  ## Disable IPV6 listeners explicitly for nodes that do not support the IPV6 stack.
+  disableIPV6: false
+
 rbac:
   ## Configures RBAC.
   create: true

diff --git a/docs/content/configuration/global-configuration/command-line-arguments.md b/docs/content/configuration/global-configuration/command-line-arguments.md
@@ -459,3 +459,11 @@ The HTTP port for the readiness endpoint.
 
 Format: `[1024 - 65535]` (default `8081`)
 &nbsp;
+
+### -disable-ipv6
+
+Disable IPV6 listeners explicitly for nodes that do not support the IPV6 stack.
+
+Default `false`.
+&nbsp;
+<a name="cmdoption-disable-ipv6"></a>
diff --git a/docs/content/installation/installation-with-helm.md b/docs/content/installation/installation-with-helm.md
@@ -237,6 +237,7 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont
 |``controller.enableLatencyMetrics`` | Enable collection of latency metrics for upstreams. Requires ``prometheus.create``. | false |
 |``controller.minReadySeconds`` | Specifies the minimum number of seconds for which a newly created Pod should be ready, without any of its containers crashing, for it to be considered available. [docs](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#min-ready-seconds) | 0 |
 |``controller.strategy`` | Specifies the strategy used to replace old Pods with new ones. [docs](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) | {} |
+| `controller.disableIPV6` | Disable IPV6 listeners explicitly for nodes that do not support the IPV6 stack. | false |
 |``rbac.create`` | Configures RBAC. | true |
 |``prometheus.create`` | Expose NGINX or NGINX Plus metrics in the Prometheus format. | false |
 |``prometheus.port`` | Configures the port to scrape the metrics. | 9113 |

diff --git a/internal/configs/config_params.go b/internal/configs/config_params.go
@@ -111,6 +111,7 @@ type ConfigParams struct {
 
 // StaticConfigParams holds immutable NGINX configuration parameters that affect the main NGINX config.
 type StaticConfigParams struct {
+	DisableIPV6                    bool
 	HealthStatus                   bool
 	HealthStatusURI                string
 	NginxStatus                    bool

diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go
@@ -553,6 +553,7 @@ func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *Config
 		AccessLogOff:                       config.MainAccessLogOff,
 		DefaultServerAccessLogOff:          config.DefaultServerAccessLogOff,
 		DefaultServerReturn:                config.DefaultServerReturn,
+		DisableIPV6:                        staticCfgParams.DisableIPV6,
 		ErrorLogLevel:                      config.MainErrorLogLevel,
 		HealthStatus:                       staticCfgParams.HealthStatus,
 		HealthStatusURI:                    staticCfgParams.HealthStatusURI,

diff --git a/internal/configs/transportserver.go b/internal/configs/transportserver.go
@@ -16,6 +16,7 @@ type TransportServerEx struct {
 	TransportServer *conf_v1alpha1.TransportServer
 	Endpoints       map[string][]string
 	PodsByIP        map[string]string
+	DisableIPV6     bool
 }
 
 func (tsEx *TransportServerEx) String() string {
@@ -90,6 +91,7 @@ func generateTransportServerConfig(transportServerEx *TransportServerEx, listene
 			ProxyNextUpstreamTries:   nextUpstreamTries,
 			HealthCheck:              healthCheck,
 			ServerSnippets:           serverSnippets,
+			DisableIPV6:              transportServerEx.DisableIPV6,
 		},
 		Match:          match,
 		Upstreams:      upstreams,

diff --git a/internal/configs/transportserver_test.go b/internal/configs/transportserver_test.go
@@ -95,6 +95,7 @@ func TestGenerateTransportServerConfigForTCPSnippets(t *testing.T) {
 				"10.0.0.20:5001",
 			},
 		},
+		DisableIPV6: false,
 	}
 
 	listenerPort := 2020
@@ -132,6 +133,7 @@ func TestGenerateTransportServerConfigForTCPSnippets(t *testing.T) {
 			ProxyNextUpstreamTimeout: "0s",
 			ProxyTimeout:             "10m",
 			HealthCheck:              nil,
+			DisableIPV6:              false,
 			ServerSnippets:           []string{"deny  192.168.1.1;", "allow 192.168.1.0/24;"},
 		},
 		StreamSnippets: []string{"limit_conn_zone $binary_remote_addr zone=addr:10m;"},
@@ -143,6 +145,86 @@ func TestGenerateTransportServerConfigForTCPSnippets(t *testing.T) {
 	}
 }
 
+func TestGenerateTransportServerConfigForIPV6Disabled(t *testing.T) {
+	t.Parallel()
+	transportServerEx := TransportServerEx{
+		TransportServer: &conf_v1alpha1.TransportServer{
+			ObjectMeta: meta_v1.ObjectMeta{
+				Name:      "tcp-server",
+				Namespace: "default",
+			},
+			Spec: conf_v1alpha1.TransportServerSpec{
+				Listener: conf_v1alpha1.TransportServerListener{
+					Name:     "tcp-listener",
+					Protocol: "TCP",
+				},
+				Upstreams: []conf_v1alpha1.Upstream{
+					{
+						Name:    "tcp-app",
+						Service: "tcp-app-svc",
+						Port:    5001,
+					},
+				},
+				Action: &conf_v1alpha1.Action{
+					Pass: "tcp-app",
+				},
+			},
+		},
+		Endpoints: map[string][]string{
+			"default/tcp-app-svc:5001": {
+				"10.0.0.20:5001",
+			},
+		},
+		DisableIPV6: true,
+	}
+
+	listenerPort := 2020
+
+	expected := &version2.TransportServerConfig{
+		Upstreams: []version2.StreamUpstream{
+			{
+				Name: "ts_default_tcp-server_tcp-app",
+				Servers: []version2.StreamUpstreamServer{
+					{
+						Address:     "10.0.0.20:5001",
+						MaxFails:    1,
+						FailTimeout: "10s",
+					},
+				},
+				UpstreamLabels: version2.UpstreamLabels{
+					ResourceName:      "tcp-server",
+					ResourceType:      "transportserver",
+					ResourceNamespace: "default",
+					Service:           "tcp-app-svc",
+				},
+				LoadBalancingMethod: "random two least_conn",
+			},
+		},
+		Server: version2.StreamServer{
+			Port:                     listenerPort,
+			UDP:                      false,
+			StatusZone:               "tcp-listener",
+			ProxyPass:                "ts_default_tcp-server_tcp-app",
+			Name:                     "tcp-server",
+			Namespace:                "default",
+			ProxyConnectTimeout:      "60s",
+			ProxyNextUpstream:        false,
+			ProxyNextUpstreamTries:   0,
+			ProxyNextUpstreamTimeout: "0s",
+			ProxyTimeout:             "10m",
+			HealthCheck:              nil,
+			ServerSnippets:           []string{},
+			DisableIPV6:              true,
+		},
+		StreamSnippets: []string{},
+	}
+
+	result := generateTransportServerConfig(&transportServerEx, listenerPort, true)
+	if diff := cmp.Diff(expected, result); diff != "" {
+		t.Errorf("generateTransportServerConfigForIPV6Disabled() mismatch (-want +got):\n%s", diff)
+	}
+}
+
 func TestGenerateTransportServerConfigForTCP(t *testing.T) {
 	t.Parallel()
 	transportServerEx := TransportServerEx{
@@ -182,6 +264,7 @@ func TestGenerateTransportServerConfigForTCP(t *testing.T) {
 				"10.0.0.20:5001",
 			},
 		},
+		DisableIPV6: false,
 	}
 
 	listenerPort := 2020
@@ -270,6 +353,7 @@ func TestGenerateTransportServerConfigForTCPMaxConnections(t *testing.T) {
 				"10.0.0.20:5001",
 			},
 		},
+		DisableIPV6: false,
 	}
 
 	listenerPort := 2020
@@ -308,6 +392,7 @@ func TestGenerateTransportServerConfigForTCPMaxConnections(t *testing.T) {
 			ProxyNextUpstreamTimeout: "0s",
 			ProxyTimeout:             "50s",
 			HealthCheck:              nil,
+			DisableIPV6:              false,
 			ServerSnippets:           []string{},
 		},
 		StreamSnippets: []string{},
@@ -356,6 +441,7 @@ func TestGenerateTransportServerConfigForTLSPassthrough(t *testing.T) {
 				"10.0.0.20:5001",
 			},
 		},
+		DisableIPV6: false,
 	}
 
 	listenerPort := 2020
@@ -397,6 +483,7 @@ func TestGenerateTransportServerConfigForTLSPassthrough(t *testing.T) {
 			HealthCheck:              nil,
 			ServerSnippets:           []string{},
 		},
+		DisableIPV6:    false,
 		StreamSnippets: []string{},
 	}
 
@@ -448,6 +535,7 @@ func TestGenerateTransportServerConfigForUDP(t *testing.T) {
 				"10.0.0.20:5001",
 			},
 		},
+		DisableIPV6: false,
 	}
 
 	listenerPort := 2020
@@ -487,6 +575,7 @@ func TestGenerateTransportServerConfigForUDP(t *testing.T) {
 			ProxyNextUpstreamTries:   0,
 			ProxyTimeout:             "10m",
 			HealthCheck:              nil,
+			DisableIPV6:              false,
 			ServerSnippets:           []string{},
 		},
 		StreamSnippets: []string{},

diff --git a/internal/configs/version1/config.go b/internal/configs/version1/config.go
@@ -15,6 +15,7 @@ type IngressNginxConfig struct {
 	Keepalive         string
 	Ingress           Ingress
 	SpiffeClientCerts bool
+	DisableIPV6       bool
 }
 
 // Ingress holds information about an Ingress resource.
@@ -110,6 +111,8 @@ type Server struct {
 	AppProtectDosAccessLogDst    string
 
 	SpiffeCerts bool
+
+	DisableIPV6 bool
 }
 
 // JWTRedirectLocation describes a location for redirecting client requests to a login URL for JWT Authentication.
@@ -162,6 +165,7 @@ type MainConfig struct {
 	AccessLogOff                       bool
 	DefaultServerAccessLogOff          bool
 	DefaultServerReturn                string
+	DisableIPV6                        bool
 	ErrorLogLevel                      string
 	HealthStatus                       bool
 	HealthStatusURI                    string

diff --git a/internal/configs/version1/nginx-plus.ingress.tmpl b/internal/configs/version1/nginx-plus.ingress.tmpl
@@ -22,14 +22,14 @@ upstream {{$upstream.Name}} {
 server {
 	{{if $server.SpiffeCerts}}
 	listen 443 ssl;
-	listen [::]:443 ssl;
+	{{if not $server.DisableIPV6}}listen [::]:443 ssl;{{end}}
 	ssl_certificate /etc/nginx/secrets/spiffe_cert.pem;
 	ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem;
 	{{else}}
 	{{if not $server.GRPCOnly}}
 	{{range $port := $server.Ports}}
 	listen {{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
-	listen [::]:{{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
+	{{if not $server.DisableIPV6}}listen [::]:{{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};{{end}}
 	{{- end}}
 	{{end}}
 
@@ -41,7 +41,7 @@ server {
 	{{else}}
 	{{- range $port := $server.SSLPorts}}
 	listen {{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
-	listen [::]:{{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
+	{{if not $server.DisableIPV6}}listen [::]:{{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};{{end}}
 	{{- end}}
 	{{end}}
 	{{if $server.SSLRejectHandshake}}

diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl
@@ -153,15 +153,15 @@ http {
         set $service "";
 
         listen 80 default_server{{if .ProxyProtocol}} proxy_protocol{{end}};
-        listen [::]:80 default_server{{if .ProxyProtocol}} proxy_protocol{{end}};
+        {{if not .DisableIPV6}}listen [::]:80 default_server{{if .ProxyProtocol}} proxy_protocol{{end}};{{end}}
 
         {{if .TLSPassthrough}}
         listen unix:/var/lib/nginx/passthrough-https.sock ssl default_server{{if .HTTP2}} http2{{end}} proxy_protocol;
         set_real_ip_from unix:;
         real_ip_header proxy_protocol;
         {{else}}
         listen 443 ssl default_server{{if .HTTP2}} http2{{end}}{{if .ProxyProtocol}} proxy_protocol{{end}};
-        listen [::]:443 ssl default_server{{if .HTTP2}} http2{{end}}{{if .ProxyProtocol}} proxy_protocol{{end}};
+        {{if not .DisableIPV6}}listen [::]:443 ssl default_server{{if .HTTP2}} http2{{end}}{{if .ProxyProtocol}} proxy_protocol{{end}};{{end}}
         {{end}}
 
         {{if .SSLRejectHandshake}}
@@ -202,7 +202,7 @@ http {
     # NGINX Plus APIs
     server {
         listen {{.NginxStatusPort}};
-        listen [::]:{{.NginxStatusPort}};
+        {{if not .DisableIPV6}}listen [::]:{{.NginxStatusPort}};{{end}}
 
         root /usr/share/nginx/html;
 
@@ -262,7 +262,7 @@ http {
     {{if .InternalRouteServer}}
     server {
         listen 443 ssl;
-        listen [::]:443 ssl;
+        {{if not .DisableIPV6}}listen [::]:443 ssl;{{end}}
         server_name {{.InternalRouteServerName}};
         ssl_certificate /etc/nginx/secrets/spiffe_cert.pem;
         ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem;
@@ -298,7 +298,7 @@ stream {
 
     server {
         listen 443{{if .ProxyProtocol}} proxy_protocol{{end}};
-        listen [::]:443{{if .ProxyProtocol}} proxy_protocol{{end}};
+        {{if not .DisableIPV6}}listen [::]:443{{if .ProxyProtocol}} proxy_protocol{{end}};{{end}}
 
         ssl_preread on;
 

diff --git a/internal/configs/version1/nginx.ingress.tmpl b/internal/configs/version1/nginx.ingress.tmpl
@@ -13,7 +13,7 @@ server {
 	{{if not $server.GRPCOnly}}
 	{{range $port := $server.Ports}}
 	listen {{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
-	listen [::]:{{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
+	{{if not $server.DisableIPV6}}listen [::]:{{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};{{end}}
 	{{- end}}
 	{{end}}
 
@@ -25,7 +25,7 @@ server {
 	{{else}}
 	{{- range $port := $server.SSLPorts}}
 	listen {{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
-	listen [::]:{{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
+	{{if not $server.DisableIPV6}}listen [::]:{{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};{{end}}
 	{{- end}}
 	{{end}}
 	{{if $server.SSLRejectHandshake}}