Merge pull request #266 from nginxinc/passive-healthchecks

Add support for passive health checks

examples/customization/README.md

@@ -52,6 +52,8 @@ The table below summarizes all of the options. For some of them, there are examp
 | `nginx.org/rewrites` | N/A | Configures URI rewriting. | N/A | [Rewrites Support](../rewrites). |
 | `nginx.org/ssl-services` | N/A | Enables HTTPS when connecting to the endpoints of services. | N/A | [SSL Services Support](../ssl-services). |
 | `nginx.org/websocket-services` | N/A | Enables WebSocket for services. | N/A | [WebSocket support](../websocket). |
+| `nginx.org/max-fails` | `max-fails` | Sets the value of the [max_fails](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails) parameter of the `server` directive. | `1` | |
+| `nginx.org/fail-timeout` | `fail-timeout` | Sets the value of the [fail_timeout](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout) parameter of the `server` directive. | `10s` | |
 | `nginx.com/sticky-cookie-services` | N/A | Configures session persistence. | N/A | [Session Persistence](../session-persistence). |
 | `nginx.com/jwt-key` | N/A |  Specifies a Secret resource with keys for validating JSON Web Tokens (JWTs). | N/A | [Support for JSON Web Tokens (JWTs)](../jwt). |
 | `nginx.com/jwt-realm` | N/A | Specifies a realm. | N/A | [Support for JSON Web Tokens (JWTs)](../jwt). |

examples/customization/nginx-config.yaml

@@ -2,6 +2,7 @@ kind: ConfigMap
 apiVersion: v1
 metadata:
   name: nginx-config
+  namespace: nginx-ingress
 data:
   proxy-connect-timeout: "10s" # default is "60s". See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout
   proxy-read-timeout: "10s" # default is "60s". See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout
@@ -53,3 +54,5 @@ data:
   worker-cpu-affinity: "auto" # No default. Sets the value of the worker_cpu_affinity directive. See http://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity
   worker-shutdown-timeout: "5m" # No default. Sets the value of the worker_shutdown_timeout directive. See http://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout
   keepalive: "32" # default is 0. When > 0, sets the value of the keepalive directive and adds 'proxy_set_header Connection "";' to a location block. See http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive
+  max-fails: "0" # default is 1. Sets the value of the max_fails parameter of the `server` directive. See https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails
+  fail-timeout: "5s" # default is 10s. Sets the value of the fail_timeout parameter of the `server` directive. See https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout

nginx-controller/controller/controller.go

@@ -659,6 +659,16 @@ func (lbc *LoadBalancerController) syncCfgm(task Task) {
 				cfg.Keepalive = keepalive
 			}
 		}
+		if maxFails, exists, err := nginx.GetMapKeyAsInt(cfgm.Data, "max-fails", cfgm); exists {
+			if err != nil {
+				glog.Error(err)
+			} else {
+				cfg.MaxFails = maxFails
+			}
+		}
+		if failTimeout, exists := cfgm.Data["fail-timeout"]; exists {
+			cfg.FailTimeout = failTimeout
+		}
 	}
 
 	mergeableIngresses := make(map[string]*nginx.MergeableIngresses)

nginx-controller/nginx/config.go

@@ -33,6 +33,8 @@ type Config struct {
 	MainWorkerConnections         string
 	MainWorkerRlimitNofile        string
 	Keepalive                     int64
+	MaxFails                      int64
+	FailTimeout                   string
 
 	// http://nginx.org/en/docs/http/ngx_http_realip_module.html
 	RealIPHeader    string
@@ -69,5 +71,7 @@ func NewDefaultConfig() *Config {
 		HSTSMaxAge:                 2592000,
 		Ports:                      []int{80},
 		SSLPorts:                   []int{443},
+		MaxFails:                   1,
+		FailTimeout:                "10s",
 	}
 }

nginx-controller/nginx/configurator.go

@@ -169,7 +169,7 @@ func (cnf *Configurator) generateNginxCfg(ingEx *IngressEx, pems map[string]stri
 
 	if ingEx.Ingress.Spec.Backend != nil {
 		name := getNameForUpstream(ingEx.Ingress, emptyHost, ingEx.Ingress.Spec.Backend.ServiceName)
-		upstream := cnf.createUpstream(ingEx, name, ingEx.Ingress.Spec.Backend, ingEx.Ingress.Namespace, spServices[ingEx.Ingress.Spec.Backend.ServiceName], ingCfg.LBMethod)
+		upstream := cnf.createUpstream(ingEx, name, ingEx.Ingress.Spec.Backend, ingEx.Ingress.Namespace, spServices[ingEx.Ingress.Spec.Backend.ServiceName], &ingCfg)
 		upstreams[name] = upstream
 	}
 
@@ -225,7 +225,7 @@ func (cnf *Configurator) generateNginxCfg(ingEx *IngressEx, pems map[string]stri
 			upsName := getNameForUpstream(ingEx.Ingress, rule.Host, path.Backend.ServiceName)
 
 			if _, exists := upstreams[upsName]; !exists {
-				upstream := cnf.createUpstream(ingEx, upsName, &path.Backend, ingEx.Ingress.Namespace, spServices[path.Backend.ServiceName], ingCfg.LBMethod)
+				upstream := cnf.createUpstream(ingEx, upsName, &path.Backend, ingEx.Ingress.Namespace, spServices[path.Backend.ServiceName], &ingCfg)
 				upstreams[upsName] = upstream
 			}
 
@@ -416,6 +416,17 @@ func (cnf *Configurator) createConfig(ingEx *IngressEx) Config {
 		}
 	}
 
+	if maxFails, exists, err := GetMapKeyAsInt(ingEx.Ingress.Annotations, "nginx.org/max-fails", ingEx.Ingress); exists {
+		if err != nil {
+			glog.Error(err)
+		} else {
+			ingCfg.MaxFails = maxFails
+		}
+	}
+	if failTimeout, exists := ingEx.Ingress.Annotations["nginx.org/fail-timeout"]; exists {
+		ingCfg.FailTimeout = failTimeout
+	}
+
 	return ingCfg
 }
 
@@ -577,7 +588,7 @@ func createLocation(path string, upstream Upstream, cfg *Config, websocket bool,
 	return loc
 }
 
-func (cnf *Configurator) createUpstream(ingEx *IngressEx, name string, backend *extensions.IngressBackend, namespace string, stickyCookie string, lbMethod string) Upstream {
+func (cnf *Configurator) createUpstream(ingEx *IngressEx, name string, backend *extensions.IngressBackend, namespace string, stickyCookie string, cfg *Config) Upstream {
 	var ups Upstream
 
 	if cnf.isPlus() {
@@ -591,13 +602,18 @@ func (cnf *Configurator) createUpstream(ingEx *IngressEx, name string, backend *
 		var upsServers []UpstreamServer
 		for _, endp := range endps {
 			addressport := strings.Split(endp, ":")
-			upsServers = append(upsServers, UpstreamServer{addressport[0], addressport[1]})
+			upsServers = append(upsServers, UpstreamServer{
+				Address:     addressport[0],
+				Port:        addressport[1],
+				MaxFails:    cfg.MaxFails,
+				FailTimeout: cfg.FailTimeout,
+			})
 		}
 		if len(upsServers) > 0 {
 			ups.UpstreamServers = upsServers
 		}
 	}
-	ups.LBMethod = lbMethod
+	ups.LBMethod = cfg.LBMethod
 	return ups
 }
 
@@ -737,11 +753,18 @@ func (cnf *Configurator) UpdateEndpointsMergeableIngress(mergeableIngs *Mergeabl
 }
 
 func (cnf *Configurator) updatePlusEndpoints(ingEx *IngressEx) error {
+	ingCfg := cnf.createConfig(ingEx)
+
+	cfg := plus.ServerConfig{
+		MaxFails:    ingCfg.MaxFails,
+		FailTimeout: ingCfg.FailTimeout,
+	}
+
 	if ingEx.Ingress.Spec.Backend != nil {
 		name := getNameForUpstream(ingEx.Ingress, emptyHost, ingEx.Ingress.Spec.Backend.ServiceName)
 		endps, exists := ingEx.Endpoints[ingEx.Ingress.Spec.Backend.ServiceName+ingEx.Ingress.Spec.Backend.ServicePort.String()]
 		if exists {
-			err := cnf.nginxAPI.UpdateServers(name, endps)
+			err := cnf.nginxAPI.UpdateServers(name, endps, cfg)
 			if err != nil {
 				return fmt.Errorf("Couldn't update the endpoints for %v: %v", name, err)
 			}
@@ -755,7 +778,7 @@ func (cnf *Configurator) updatePlusEndpoints(ingEx *IngressEx) error {
 			name := getNameForUpstream(ingEx.Ingress, rule.Host, path.Backend.ServiceName)
 			endps, exists := ingEx.Endpoints[path.Backend.ServiceName+path.Backend.ServicePort.String()]
 			if exists {
-				err := cnf.nginxAPI.UpdateServers(name, endps)
+				err := cnf.nginxAPI.UpdateServers(name, endps, cfg)
 				if err != nil {
 					return fmt.Errorf("Couldn't update the endpoints for %v: %v", name, err)
 				}

nginx-controller/nginx/nginx.go

@@ -45,8 +45,10 @@ type Upstream struct {
 
 // UpstreamServer describes a server in an NGINX upstream
 type UpstreamServer struct {
-	Address string
-	Port    string
+	Address     string
+	Port        string
+	MaxFails    int64
+	FailTimeout string
 }
 
 // Server describes an NGINX server
@@ -134,8 +136,14 @@ type NginxMainConfig struct {
 // We use it for services that have no endpoints
 func NewUpstreamWithDefaultServer(name string) Upstream {
 	return Upstream{
-		Name:            name,
-		UpstreamServers: []UpstreamServer{UpstreamServer{Address: "127.0.0.1", Port: "8181"}},
+		Name: name,
+		UpstreamServers: []UpstreamServer{
+			UpstreamServer{
+				Address:     "127.0.0.1",
+				Port:        "8181",
+				MaxFails:    1,
+				FailTimeout: "10s",
+			}},
 	}
 }
 

nginx-controller/nginx/plus/nginx_api.go

@@ -11,6 +11,11 @@ type NginxAPIController struct {
 	local  bool
 }
 
+type ServerConfig struct {
+	MaxFails    int64
+	FailTimeout string
+}
+
 func NewNginxAPIController(httpClient *http.Client, endpoint string, local bool) (*NginxAPIController, error) {
 	client, err := NewNginxClient(httpClient, endpoint)
 	if !local && err != nil {
@@ -20,12 +25,22 @@ func NewNginxAPIController(httpClient *http.Client, endpoint string, local bool)
 	return nginx, nil
 }
 
-func (nginx *NginxAPIController) UpdateServers(upstream string, servers []string) error {
+func (nginx *NginxAPIController) UpdateServers(upstream string, servers []string, config ServerConfig) error {
 	if nginx.local {
 		glog.V(3).Infof("Updating endpoints of %v: %v\n", upstream, servers)
 		return nil
 	}
-	added, removed, err := nginx.client.UpdateHTTPServers(upstream, servers)
+
+	var upsServers []UpstreamServer
+	for _, s := range servers {
+		upsServers = append(upsServers, UpstreamServer{
+			Server:      s,
+			MaxFails:    config.MaxFails,
+			FailTimeout: config.FailTimeout,
+		})
+	}
+
+	added, removed, err := nginx.client.UpdateHTTPServers(upstream, upsServers)
 	if err != nil {
 		glog.V(3).Infof("Couldn't update servers of %v upstream: %v", upstream, err)
 		return err