Merge pull request #1239 from aledbf/pc

Add flags to customize listen ports and detect port collisions

controllers/nginx/README.md

@@ -47,29 +47,36 @@ Anytime we reference a tls secret, we mean (x509, pem encoded, RSA 2048, etc). Y
 Usage of :
       --alsologtostderr                  log to standard error as well as files
       --apiserver-host string            The address of the Kubernetes Apiserver to connect to in the format of protocol://address:port, e.g., http://localhost:8080. If not specified, the assumption is that the binary runs inside a Kubernetes cluster and local discovery is attempted.
-      --configmap string                 Name of the ConfigMap that contains the custom configuration use
-      --default-backend-service string   Service used to serve a 404 page for the default backend. Takes the form namespace/name. The controller uses the first node port of this Service for the default backend.
+      --configmap string                 Name of the ConfigMap that contains the custom configuration to use
+      --default-backend-service string   Service used to serve a 404 page for the default backend. Takes the form
+    	namespace/name. The controller uses the first node port of this Service for
+    	the default backend.
+      --default-server-port int          Default port to use for exposing the default server (catch all) (default 8181)
       --default-ssl-certificate string   Name of the secret that contains a SSL certificate to be used as default for a HTTPS catch-all server
       --election-id string               Election id to use for status update. (default "ingress-controller-leader")
+      --enable-ssl-passthrough           Enable SSL passthrough feature. Default is disabled
       --force-namespace-isolation        Force namespace isolation. This flag is required to avoid the reference of secrets or configmaps located in a different namespace than the specified in the flag --watch-namespace.
       --health-check-path string         Defines the URL to be used as health check inside in the default server in NGINX. (default "/healthz")
       --healthz-port int                 port for healthz endpoint. (default 10254)
+      --http-port int                    Indicates the port to use for HTTP traffic (default 80)
+      --https-port int                   Indicates the port to use for HTTPS traffic (default 443)
       --ingress-class string             Name of the ingress class to route through this controller.
       --kubeconfig string                Path to kubeconfig file with authorization and master location information.
       --log_backtrace_at traceLocation   when logging hits line file:N, emit a stack trace (default :0)
       --log_dir string                   If non-empty, write log files in this directory
       --logtostderr                      log to standard error instead of files
       --profiling                        Enable profiling via web interface host:port/debug/pprof/ (default true)
       --publish-service string           Service fronting the ingress controllers. Takes the form namespace/name. The controller will set the endpoint records on the ingress objects to reflect those on the service.
+      --sort-backends                    Defines if backends and it's endpoints should be sorted
+      --ssl-passtrough-proxy-port int    Default port to use internally for SSL when SSL Passthgough is enabled (default 442)
+      --status-port int                  Indicates the TCP port to use for exposing the nginx status page (default 18080)
       --stderrthreshold severity         logs at or above this threshold go to stderr (default 2)
-      --sync-period duration             Relist and confirm cloud resources this often. (default 1m0s)
-      --tcp-services-configmap string    Name of the ConfigMap that contains the definition of the TCP services to expose.
-		  The key in the map indicates the external port to be used. The value is the name of the service with the format namespace/serviceName and the port of the service could be a number of the name of the port.
-		  The ports 80 and 443 are not allowed as external ports. This ports are reserved for the backend
-      --udp-services-configmap string    Name of the ConfigMap that contains the definition of the UDP services to expose.
-		  The key in the map indicates the external port to be used. The value is the name of the service with the format namespace/serviceName and the port of the service could be a number of the name of the port.
+      --sync-period duration             Relist and confirm cloud resources this often. Default is 10 minutes (default 10m0s)
+      --tcp-services-configmap string    Name of the ConfigMap that contains the definition of the TCP services to expose. The key in the map indicates the external port to be used. The value is the name of theservice with the format namespace/serviceName and the port of the service could be a number of the name of the port. The ports 80 and 443 are not allowed as external ports. This ports are reserved for the backend
+      --udp-services-configmap string    Name of the ConfigMap that contains the definition of the UDP services to expose. The key in the map indicates the external port to be used. The value is the name of theservice with the format namespace/serviceName and the port of the service could be a number of the name of the port.
       --update-status                    Indicates if the ingress controller should update the Ingress status IP/hostname. Default is true (default true)
--v, --v Level                          log level for V logs
+      --update-status-on-shutdown        Indicates if the ingress controller should update the Ingress status IP/hostname when the controller is being stopped. Default is true (default true)
+  -v, --v Level                          log level for V logs
       --vmodule moduleSpec               comma-separated list of pattern=N settings for file-filtered logging
       --watch-namespace string           Namespace to watch for Ingress. Default is to watch all namespaces
 ```

controllers/nginx/pkg/cmd/controller/metrics.go

@@ -45,6 +45,8 @@ type statsCollector struct {
 
 	namespace  string
 	watchClass string
+
+	healthPort int
 }
 
 func (s *statsCollector) stop(sm statusModule) {
@@ -61,17 +63,17 @@ func (s *statsCollector) stop(sm statusModule) {
 func (s *statsCollector) start(sm statusModule) {
 	switch sm {
 	case defaultStatusModule:
-		s.basic = collector.NewNginxStatus(s.namespace, s.watchClass, ngxHealthPort, ngxStatusPath)
+		s.basic = collector.NewNginxStatus(s.namespace, s.watchClass, s.healthPort, ngxStatusPath)
 		prometheus.Register(s.basic)
 		break
 	case vtsStatusModule:
-		s.vts = collector.NewNGINXVTSCollector(s.namespace, s.watchClass, ngxHealthPort, ngxVtsPath)
+		s.vts = collector.NewNGINXVTSCollector(s.namespace, s.watchClass, s.healthPort, ngxVtsPath)
 		prometheus.Register(s.vts)
 		break
 	}
 }
 
-func newStatsCollector(ns, class, binary string) *statsCollector {
+func newStatsCollector(ns, class, binary string, hz int) *statsCollector {
 	glog.Infof("starting new nginx stats collector for Ingress controller running in namespace %v (class %v)", ns, class)
 	pc, err := collector.NewNamedProcess(true, collector.BinaryNameMatcher{
 		Name:   "nginx",
@@ -89,5 +91,6 @@ func newStatsCollector(ns, class, binary string) *statsCollector {
 		namespace:  ns,
 		watchClass: class,
 		process:    pc,
+		healthPort: hz,
 	}
 }

controllers/nginx/pkg/cmd/controller/nginx.go

@@ -34,6 +34,7 @@ import (
 	"github.com/spf13/pflag"
 
 	proxyproto "github.com/armon/go-proxyproto"
+	api "k8s.io/api/core/v1"
 	api_v1 "k8s.io/api/core/v1"
 	extensions "k8s.io/api/extensions/v1beta1"
 
@@ -50,7 +51,6 @@ import (
 type statusModule string
 
 const (
-	ngxHealthPort = 18080
 	ngxHealthPath = "/healthz"
 
 	defaultStatusModule statusModule = "default"
@@ -87,6 +87,7 @@ func newNGINXController() ingress.Controller {
 		configmap:     &api_v1.ConfigMap{},
 		isIPV6Enabled: isIPv6Enabled(),
 		resolver:      h,
+		ports:         &config.ListenPorts{},
 	}
 
 	fcgiListener, err := net.Listen("unix", fastCGISocket)
@@ -161,6 +162,8 @@ type NGINXController struct {
 	isSSLPassthroughEnabled bool
 
 	proxy *proxy
+
+	ports *config.ListenPorts
 }
 
 // Start start a new NGINX master process running in foreground.
@@ -280,14 +283,42 @@ func (n NGINXController) Info() *ingress.BackendInfo {
 	}
 }
 
+// DefaultEndpoint returns the default endpoint to be use as default server that returns 404.
+func (n NGINXController) DefaultEndpoint() ingress.Endpoint {
+	return ingress.Endpoint{
+		Address: "127.0.0.1",
+		Port:    fmt.Sprintf("%v", n.ports.Default),
+		Target:  &api.ObjectReference{},
+	}
+}
+
 // ConfigureFlags allow to configure more flags before the parsing of
 // command line arguments
 func (n *NGINXController) ConfigureFlags(flags *pflag.FlagSet) {
 	flags.BoolVar(&n.isSSLPassthroughEnabled, "enable-ssl-passthrough", false, `Enable SSL passthrough feature. Default is disabled`)
+	flags.IntVar(&n.ports.HTTP, "http-port", 80, `Indicates the port to use for HTTP traffic`)
+	flags.IntVar(&n.ports.HTTPS, "https-port", 443, `Indicates the port to use for HTTPS traffic`)
+	flags.IntVar(&n.ports.Status, "status-port", 18080, `Indicates the TCP port to use for exposing the nginx status page`)
+	flags.IntVar(&n.ports.SSLProxy, "ssl-passtrough-proxy-port", 442, `Default port to use internally for SSL when SSL Passthgough is enabled`)
+	flags.IntVar(&n.ports.Default, "default-server-port", 8181, `Default port to use for exposing the default server (catch all)`)
 }
 
 // OverrideFlags customize NGINX controller flags
 func (n *NGINXController) OverrideFlags(flags *pflag.FlagSet) {
+	// we check port collisions
+	if !isPortAvailable(n.ports.HTTP) {
+		glog.Fatalf("Port %v is already in use. Please check the flag --http-port", n.ports.HTTP)
+	}
+	if !isPortAvailable(n.ports.HTTPS) {
+		glog.Fatalf("Port %v is already in use. Please check the flag --https-port", n.ports.HTTPS)
+	}
+	if !isPortAvailable(n.ports.Status) {
+		glog.Fatalf("Port %v is already in use. Please check the flag --status-port", n.ports.Status)
+	}
+	if !isPortAvailable(n.ports.Default) {
+		glog.Fatalf("Port %v is already in use. Please check the flag --default-server-port", n.ports.Default)
+	}
+
 	ic, _ := flags.GetString("ingress-class")
 	wc, _ := flags.GetString("watch-namespace")
 
@@ -300,20 +331,24 @@ func (n *NGINXController) OverrideFlags(flags *pflag.FlagSet) {
 	}
 
 	flags.Set("ingress-class", ic)
-	n.stats = newStatsCollector(wc, ic, n.binary)
+	n.stats = newStatsCollector(wc, ic, n.binary, n.ports.Health)
 
 	if n.isSSLPassthroughEnabled {
+		if !isPortAvailable(n.ports.SSLProxy) {
+			glog.Fatalf("Port %v is already in use. Please check the flag --ssl-passtrough-proxy-port", n.ports.SSLProxy)
+		}
+
 		glog.Info("starting TLS proxy for SSL passthrough")
 		n.proxy = &proxy{
 			Default: &server{
 				Hostname:      "localhost",
 				IP:            "127.0.0.1",
-				Port:          442,
+				Port:          n.ports.SSLProxy,
 				ProxyProtocol: true,
 			},
 		}
 
-		listener, err := net.Listen("tcp", ":443")
+		listener, err := net.Listen("tcp", fmt.Sprintf(":%v", n.ports.HTTPS))
 		if err != nil {
 			glog.Fatalf("%v", err)
 		}
@@ -594,6 +629,7 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
 		IsIPV6Enabled:           n.isIPV6Enabled && !cfg.DisableIpv6,
 		RedirectServers:         redirectServers,
 		IsSSLPassthroughEnabled: n.isSSLPassthroughEnabled,
+		ListenPorts:             n.ports,
 	}
 
 	// We need to extract the endpoints to be used in the fastcgi error handler
@@ -651,7 +687,7 @@ func (n NGINXController) Name() string {
 
 // Check returns if the nginx healthz endpoint is returning ok (status code 200)
 func (n NGINXController) Check(_ *http.Request) error {
-	res, err := http.Get(fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxHealthPath))
+	res, err := http.Get(fmt.Sprintf("http://localhost:%v%v", n.ports.Status, ngxHealthPath))
 	if err != nil {
 		return err
 	}

controllers/nginx/pkg/cmd/controller/utils.go

@@ -17,7 +17,9 @@ limitations under the License.
 package main
 
 import (
+	"fmt"
 	"io/ioutil"
+	"net"
 	"os"
 	"os/exec"
 	"syscall"
@@ -74,3 +76,12 @@ func diff(b1, b2 []byte) ([]byte, error) {
 	out, _ := exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput()
 	return out, nil
 }
+
+func isPortAvailable(p int) bool {
+	ln, err := net.Listen("tcp", fmt.Sprintf(":%v", p))
+	if err != nil {
+		return false
+	}
+	ln.Close()
+	return true
+}

controllers/nginx/pkg/config/config.go

@@ -440,4 +440,16 @@ type TemplateConfig struct {
 	IsIPV6Enabled           bool
 	IsSSLPassthroughEnabled bool
 	RedirectServers         map[string]string
+	ListenPorts             *ListenPorts
+}
+
+// ListenPorts describe the ports required to run the
+// NGINX Ingress controller
+type ListenPorts struct {
+	HTTP     int
+	HTTPS    int
+	Status   int
+	Health   int
+	Default  int
+	SSLProxy int
 }

controllers/nginx/pkg/template/template_test.go

@@ -175,7 +175,9 @@ func TestTemplateWithData(t *testing.T) {
 	if err := json.Unmarshal(data, &dat); err != nil {
 		t.Errorf("unexpected error unmarshalling json: %v", err)
 	}
-
+	if dat.ListenPorts == nil {
+		dat.ListenPorts = &config.ListenPorts{}
+	}
 	tf, err := os.Open(path.Join(pwd, "../../rootfs/etc/nginx/template/nginx.tmpl"))
 	if err != nil {
 		t.Errorf("unexpected error reading json file: %v", err)

controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl

@@ -160,9 +160,9 @@ http {
     {{ end }}
 
     {{ if $all.IsSSLPassthroughEnabled }}
-    # map port 442 to 443 for header X-Forwarded-Port
+    # map port {{ $all.ListenPorts.SSLProxy }} to 443 for header X-Forwarded-Port
     map $pass_server_port $pass_port {
-        442              443;
+        {{ $all.ListenPorts.SSLProxy }}              443;
         default          $pass_server_port;
     }
     {{ else }}
@@ -319,11 +319,11 @@ http {
     {{/* Build server redirects (from/to www) */}}
     {{ range $hostname, $to := .RedirectServers }}
     server {
-        listen 80{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }};
-        listen {{ if $all.IsSSLPassthroughEnabled }}442 proxy_protocol{{ else }}443{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} ssl;
+        listen {{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }};
+        listen {{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} ssl;
         {{ if $IsIPV6Enabled }}
-        listen  [::]:80{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }};
-        listen {{ if $all.IsSSLPassthroughEnabled }}[::]:442 proxy_protocol{{ else }}[::]:443{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }};
+        listen [::]:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }};
+        listen [::]:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }};
         {{ end }}
         server_name {{ $hostname }};
         return 301 $scheme://{{ $to }}$request_uri;
@@ -347,11 +347,11 @@ http {
 
     # default server, used for NGINX healthcheck and access to nginx stats
     server {
-        # Use the port 18080 (random value just to avoid known ports) as default port for nginx.
+        # Use the port {{ $all.ListenPorts.Status }} (random value just to avoid known ports) as default port for nginx.
         # Changing this value requires a change in:
         # https://github.com/kubernetes/ingress/blob/master/controllers/nginx/pkg/cmd/controller/nginx.go
-        listen 18080 default_server reuseport backlog={{ .BacklogSize }};
-        {{ if $IsIPV6Enabled }}listen [::]:18080 default_server reuseport backlog={{ .BacklogSize }};{{ end }}
+        listen {{ $all.ListenPorts.Status }} default_server reuseport backlog={{ .BacklogSize }};
+        {{ if $IsIPV6Enabled }}listen [::]:{{ $all.ListenPorts.Status }} default_server reuseport backlog={{ .BacklogSize }};{{ end }}
         set $proxy_upstream_name "-";
 
         location {{ $healthzURI }} {
@@ -394,7 +394,7 @@ http {
 
     # default server for services without endpoints
     server {
-        listen 8181;
+        listen {{ $all.ListenPorts.Default }};
         set $proxy_upstream_name "-";
 
         location / {
@@ -518,14 +518,15 @@ stream {
 {{ define "SERVER" }}
         {{ $all := .First }}
         {{ $server := .Second }}
-        listen 80{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}};
-        {{ if $all.IsIPV6Enabled }}listen [::]:80{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }};{{ end }}
+        listen {{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}};
+        {{ if $all.IsIPV6Enabled }}listen [::]:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }};{{ end }}
         set $proxy_upstream_name "-";
 
-        {{/* Listen on 442 because port 443 is used in the TLS sni server */}}
+        {{/* Listen on {{ $all.ListenPorts.SSLProxy }} because port {{ $all.ListenPorts.HTTPS }} is used in the TLS sni server */}}
         {{/* This listener must always have proxy_protocol enabled, because the SNI listener forwards on source IP info in it. */}}
-        {{ if not (empty $server.SSLCertificate) }}listen {{ if $all.IsSSLPassthroughEnabled }}442 proxy_protocol {{ else }}443{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }};
-        {{ if $all.IsIPV6Enabled }}{{ if not (empty $server.SSLCertificate) }}listen {{ if $all.IsSSLPassthroughEnabled }}[::]:442 proxy_protocol{{ else }}[::]:443{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }};{{ end }}        {{/* comment PEM sha is required to detect changes in the generated configuration and force a reload */}}
+        {{ if not (empty $server.SSLCertificate) }}listen {{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol {{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }};
+        {{ if $all.IsIPV6Enabled }}{{ if not (empty $server.SSLCertificate) }}listen [::]:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }};{{ end }}
+        {{/* comment PEM sha is required to detect changes in the generated configuration and force a reload */}}
         # PEM sha: {{ $server.SSLPemChecksum }}
         ssl_certificate                         {{ $server.SSLCertificate }};
         ssl_certificate_key                     {{ $server.SSLCertificate }};
@@ -708,7 +709,7 @@ stream {
         {{ end }}
 
         {{ if eq $server.Hostname "_" }}
-        # health checks in cloud providers require the use of port 80
+        # health checks in cloud providers require the use of port {{ $all.ListenPorts.HTTP }}
         location {{ $all.HealthzURI }} {
         access_log off;
         return 200;