From 6dbef44dbe206ac226dbc0322220bbea4d8f61ae Mon Sep 17 00:00:00 2001 From: Michael Pleshakov Date: Thu, 5 Apr 2018 18:18:24 +0100 Subject: [PATCH] Add support for passive health checks --- examples/customization/README.md | 2 + examples/customization/nginx-config.yaml | 3 + nginx-controller/controller/controller.go | 10 +++ nginx-controller/nginx/config.go | 4 + nginx-controller/nginx/configurator.go | 37 ++++++-- nginx-controller/nginx/nginx.go | 16 +++- nginx-controller/nginx/plus/nginx_api.go | 19 +++- nginx-controller/nginx/plus/nginx_client.go | 90 +++++++------------ .../nginx/templates/nginx-plus.ingress.tmpl | 2 +- .../nginx/templates/nginx.ingress.tmpl | 2 +- .../nginx/templates/templates_test.go | 7 +- 11 files changed, 119 insertions(+), 73 deletions(-) diff --git a/examples/customization/README.md b/examples/customization/README.md index c95d5b1f48..c83b71c179 100644 --- a/examples/customization/README.md +++ b/examples/customization/README.md @@ -52,6 +52,8 @@ The table below summarizes all of the options. For some of them, there are examp | `nginx.org/rewrites` | N/A | Configures URI rewriting. | N/A | [Rewrites Support](../rewrites). | | `nginx.org/ssl-services` | N/A | Enables HTTPS when connecting to the endpoints of services. | N/A | [SSL Services Support](../ssl-services). | | `nginx.org/websocket-services` | N/A | Enables WebSocket for services. | N/A | [WebSocket support](../websocket). | +| `nginx.org/max-fails` | `max-fails` | Sets the value of the [max_fails](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails) parameter of the `server` directive. | `1` | | +| `nginx.org/fail-timeout` | `fail-timeout` | Sets the value of the [fail_timeout](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout) parameter of the `server` directive. | `10s` | | | `nginx.com/sticky-cookie-services` | N/A | Configures session persistence. | N/A | [Session Persistence](../session-persistence). | | `nginx.com/jwt-key` | N/A | Specifies a Secret resource with keys for validating JSON Web Tokens (JWTs). | N/A | [Support for JSON Web Tokens (JWTs)](../jwt). | | `nginx.com/jwt-realm` | N/A | Specifies a realm. | N/A | [Support for JSON Web Tokens (JWTs)](../jwt). | diff --git a/examples/customization/nginx-config.yaml b/examples/customization/nginx-config.yaml index 18b06cdc35..23c374b2ab 100644 --- a/examples/customization/nginx-config.yaml +++ b/examples/customization/nginx-config.yaml @@ -2,6 +2,7 @@ kind: ConfigMap apiVersion: v1 metadata: name: nginx-config + namespace: nginx-ingress data: proxy-connect-timeout: "10s" # default is "60s". See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout proxy-read-timeout: "10s" # default is "60s". See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout @@ -53,3 +54,5 @@ data: worker-cpu-affinity: "auto" # No default. Sets the value of the worker_cpu_affinity directive. See http://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity worker-shutdown-timeout: "5m" # No default. Sets the value of the worker_shutdown_timeout directive. See http://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout keepalive: "32" # default is 0. When > 0, sets the value of the keepalive directive and adds 'proxy_set_header Connection "";' to a location block. See http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive + max-fails: "0" # default is 1. Sets the value of the max_fails parameter of the `server` directive. See https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails + fail-timeout: "5s" # default is 10s. Sets the value of the fail_timeout parameter of the `server` directive. See https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout diff --git a/nginx-controller/controller/controller.go b/nginx-controller/controller/controller.go index c4b32e4dfe..8246bd230f 100644 --- a/nginx-controller/controller/controller.go +++ b/nginx-controller/controller/controller.go @@ -659,6 +659,16 @@ func (lbc *LoadBalancerController) syncCfgm(task Task) { cfg.Keepalive = keepalive } } + if maxFails, exists, err := nginx.GetMapKeyAsInt(cfgm.Data, "max-fails", cfgm); exists { + if err != nil { + glog.Error(err) + } else { + cfg.MaxFails = maxFails + } + } + if failTimeout, exists := cfgm.Data["fail-timeout"]; exists { + cfg.FailTimeout = failTimeout + } } mergeableIngresses := make(map[string]*nginx.MergeableIngresses) diff --git a/nginx-controller/nginx/config.go b/nginx-controller/nginx/config.go index 5209c19ff3..324509c857 100644 --- a/nginx-controller/nginx/config.go +++ b/nginx-controller/nginx/config.go @@ -33,6 +33,8 @@ type Config struct { MainWorkerConnections string MainWorkerRlimitNofile string Keepalive int64 + MaxFails int64 + FailTimeout string // http://nginx.org/en/docs/http/ngx_http_realip_module.html RealIPHeader string @@ -69,5 +71,7 @@ func NewDefaultConfig() *Config { HSTSMaxAge: 2592000, Ports: []int{80}, SSLPorts: []int{443}, + MaxFails: 1, + FailTimeout: "10s", } } diff --git a/nginx-controller/nginx/configurator.go b/nginx-controller/nginx/configurator.go index 566b63b0c1..d39131a391 100644 --- a/nginx-controller/nginx/configurator.go +++ b/nginx-controller/nginx/configurator.go @@ -169,7 +169,7 @@ func (cnf *Configurator) generateNginxCfg(ingEx *IngressEx, pems map[string]stri if ingEx.Ingress.Spec.Backend != nil { name := getNameForUpstream(ingEx.Ingress, emptyHost, ingEx.Ingress.Spec.Backend.ServiceName) - upstream := cnf.createUpstream(ingEx, name, ingEx.Ingress.Spec.Backend, ingEx.Ingress.Namespace, spServices[ingEx.Ingress.Spec.Backend.ServiceName], ingCfg.LBMethod) + upstream := cnf.createUpstream(ingEx, name, ingEx.Ingress.Spec.Backend, ingEx.Ingress.Namespace, spServices[ingEx.Ingress.Spec.Backend.ServiceName], &ingCfg) upstreams[name] = upstream } @@ -225,7 +225,7 @@ func (cnf *Configurator) generateNginxCfg(ingEx *IngressEx, pems map[string]stri upsName := getNameForUpstream(ingEx.Ingress, rule.Host, path.Backend.ServiceName) if _, exists := upstreams[upsName]; !exists { - upstream := cnf.createUpstream(ingEx, upsName, &path.Backend, ingEx.Ingress.Namespace, spServices[path.Backend.ServiceName], ingCfg.LBMethod) + upstream := cnf.createUpstream(ingEx, upsName, &path.Backend, ingEx.Ingress.Namespace, spServices[path.Backend.ServiceName], &ingCfg) upstreams[upsName] = upstream } @@ -416,6 +416,17 @@ func (cnf *Configurator) createConfig(ingEx *IngressEx) Config { } } + if maxFails, exists, err := GetMapKeyAsInt(ingEx.Ingress.Annotations, "nginx.org/max-fails", ingEx.Ingress); exists { + if err != nil { + glog.Error(err) + } else { + ingCfg.MaxFails = maxFails + } + } + if failTimeout, exists := ingEx.Ingress.Annotations["nginx.org/fail-timeout"]; exists { + ingCfg.FailTimeout = failTimeout + } + return ingCfg } @@ -577,7 +588,7 @@ func createLocation(path string, upstream Upstream, cfg *Config, websocket bool, return loc } -func (cnf *Configurator) createUpstream(ingEx *IngressEx, name string, backend *extensions.IngressBackend, namespace string, stickyCookie string, lbMethod string) Upstream { +func (cnf *Configurator) createUpstream(ingEx *IngressEx, name string, backend *extensions.IngressBackend, namespace string, stickyCookie string, cfg *Config) Upstream { var ups Upstream if cnf.isPlus() { @@ -591,13 +602,18 @@ func (cnf *Configurator) createUpstream(ingEx *IngressEx, name string, backend * var upsServers []UpstreamServer for _, endp := range endps { addressport := strings.Split(endp, ":") - upsServers = append(upsServers, UpstreamServer{addressport[0], addressport[1]}) + upsServers = append(upsServers, UpstreamServer{ + Address: addressport[0], + Port: addressport[1], + MaxFails: cfg.MaxFails, + FailTimeout: cfg.FailTimeout, + }) } if len(upsServers) > 0 { ups.UpstreamServers = upsServers } } - ups.LBMethod = lbMethod + ups.LBMethod = cfg.LBMethod return ups } @@ -737,11 +753,18 @@ func (cnf *Configurator) UpdateEndpointsMergeableIngress(mergeableIngs *Mergeabl } func (cnf *Configurator) updatePlusEndpoints(ingEx *IngressEx) error { + ingCfg := cnf.createConfig(ingEx) + + cfg := plus.ServerConfig{ + MaxFails: ingCfg.MaxFails, + FailTimeout: ingCfg.FailTimeout, + } + if ingEx.Ingress.Spec.Backend != nil { name := getNameForUpstream(ingEx.Ingress, emptyHost, ingEx.Ingress.Spec.Backend.ServiceName) endps, exists := ingEx.Endpoints[ingEx.Ingress.Spec.Backend.ServiceName+ingEx.Ingress.Spec.Backend.ServicePort.String()] if exists { - err := cnf.nginxAPI.UpdateServers(name, endps) + err := cnf.nginxAPI.UpdateServers(name, endps, cfg) if err != nil { return fmt.Errorf("Couldn't update the endpoints for %v: %v", name, err) } @@ -755,7 +778,7 @@ func (cnf *Configurator) updatePlusEndpoints(ingEx *IngressEx) error { name := getNameForUpstream(ingEx.Ingress, rule.Host, path.Backend.ServiceName) endps, exists := ingEx.Endpoints[path.Backend.ServiceName+path.Backend.ServicePort.String()] if exists { - err := cnf.nginxAPI.UpdateServers(name, endps) + err := cnf.nginxAPI.UpdateServers(name, endps, cfg) if err != nil { return fmt.Errorf("Couldn't update the endpoints for %v: %v", name, err) } diff --git a/nginx-controller/nginx/nginx.go b/nginx-controller/nginx/nginx.go index ea7fde0f41..212445102b 100644 --- a/nginx-controller/nginx/nginx.go +++ b/nginx-controller/nginx/nginx.go @@ -45,8 +45,10 @@ type Upstream struct { // UpstreamServer describes a server in an NGINX upstream type UpstreamServer struct { - Address string - Port string + Address string + Port string + MaxFails int64 + FailTimeout string } // Server describes an NGINX server @@ -134,8 +136,14 @@ type NginxMainConfig struct { // We use it for services that have no endpoints func NewUpstreamWithDefaultServer(name string) Upstream { return Upstream{ - Name: name, - UpstreamServers: []UpstreamServer{UpstreamServer{Address: "127.0.0.1", Port: "8181"}}, + Name: name, + UpstreamServers: []UpstreamServer{ + UpstreamServer{ + Address: "127.0.0.1", + Port: "8181", + MaxFails: 1, + FailTimeout: "10s", + }}, } } diff --git a/nginx-controller/nginx/plus/nginx_api.go b/nginx-controller/nginx/plus/nginx_api.go index 54bd01b792..af0b057e78 100644 --- a/nginx-controller/nginx/plus/nginx_api.go +++ b/nginx-controller/nginx/plus/nginx_api.go @@ -11,6 +11,11 @@ type NginxAPIController struct { local bool } +type ServerConfig struct { + MaxFails int64 + FailTimeout string +} + func NewNginxAPIController(httpClient *http.Client, endpoint string, local bool) (*NginxAPIController, error) { client, err := NewNginxClient(httpClient, endpoint) if !local && err != nil { @@ -20,12 +25,22 @@ func NewNginxAPIController(httpClient *http.Client, endpoint string, local bool) return nginx, nil } -func (nginx *NginxAPIController) UpdateServers(upstream string, servers []string) error { +func (nginx *NginxAPIController) UpdateServers(upstream string, servers []string, config ServerConfig) error { if nginx.local { glog.V(3).Infof("Updating endpoints of %v: %v\n", upstream, servers) return nil } - added, removed, err := nginx.client.UpdateHTTPServers(upstream, servers) + + var upsServers []UpstreamServer + for _, s := range servers { + upsServers = append(upsServers, UpstreamServer{ + Server: s, + MaxFails: config.MaxFails, + FailTimeout: config.FailTimeout, + }) + } + + added, removed, err := nginx.client.UpdateHTTPServers(upstream, upsServers) if err != nil { glog.V(3).Infof("Couldn't update servers of %v upstream: %v", upstream, err) return err diff --git a/nginx-controller/nginx/plus/nginx_client.go b/nginx-controller/nginx/plus/nginx_client.go index eb4bf5022e..8ff288080b 100644 --- a/nginx-controller/nginx/plus/nginx_client.go +++ b/nginx-controller/nginx/plus/nginx_client.go @@ -18,19 +18,13 @@ type NginxClient struct { httpClient *http.Client } -type peers struct { - Peers []peer -} - -type peer struct { - ID int - Server string -} - type versions []int -type upstreamServer struct { - Server string `json:"server"` +type UpstreamServer struct { + ID int64 `json:"id,omitempty"` + Server string `json:"server"` + MaxFails int64 `json:"max_fails"` + FailTimeout string `json:"fail_timeout,omitempty"` } type apiErrorResponse struct { @@ -129,16 +123,17 @@ func readAPIErrorResponse(respBody io.ReadCloser) (*apiErrorResponse, error) { // CheckIfUpstreamExists checks if the upstream exists in NGINX. If the upstream doesn't exist, it returns the error. func (client *NginxClient) CheckIfUpstreamExists(upstream string) error { - _, err := client.getUpstreamPeers(upstream) + _, err := client.GetHTTPServers(upstream) return err } -func (client *NginxClient) getUpstreamPeers(upstream string) (*peers, error) { - url := fmt.Sprintf("%v/%v/http/upstreams/%v", client.apiEndpoint, APIVersion, upstream) +// GetHTTPServers returns the servers of the upsteam from NGINX. +func (client *NginxClient) GetHTTPServers(upstream string) ([]UpstreamServer, error) { + url := fmt.Sprintf("%v/%v/http/upstreams/%v/servers", client.apiEndpoint, APIVersion, upstream) resp, err := client.httpClient.Get(url) if err != nil { - return nil, fmt.Errorf("failed to connect to the API to get upstream %v info: %v", upstream, err) + return nil, fmt.Errorf("failed to connect to the API to get upstream servers of upstream %v: %v", upstream, err) } defer resp.Body.Close() @@ -149,36 +144,32 @@ func (client *NginxClient) getUpstreamPeers(upstream string) (*peers, error) { body, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("failed to read the response body with upstream %v info: %v", upstream, err) + return nil, fmt.Errorf("failed to read the response body with upstream servers of upstream %v: %v", upstream, err) } - var prs peers - err = json.Unmarshal(body, &prs) + var servers []UpstreamServer + err = json.Unmarshal(body, &servers) if err != nil { - return nil, fmt.Errorf("error unmarshalling upstream %v: got %q response: %v", upstream, string(body), err) + return nil, fmt.Errorf("error unmarshalling upstream servers of upstream %v: got %q response: %v", upstream, string(body), err) } - return &prs, nil + return servers, nil } // AddHTTPServer adds the server to the upstream. -func (client *NginxClient) AddHTTPServer(upstream string, server string) error { - id, err := client.getIDOfHTTPServer(upstream, server) +func (client *NginxClient) AddHTTPServer(upstream string, server UpstreamServer) error { + id, err := client.getIDOfHTTPServer(upstream, server.Server) if err != nil { - return fmt.Errorf("failed to add %v server to %v upstream: %v", server, upstream, err) + return fmt.Errorf("failed to add %v server to %v upstream: %v", server.Server, upstream, err) } if id != -1 { - return fmt.Errorf("failed to add %v server to %v upstream: server already exists", server, upstream) - } - - upsServer := upstreamServer{ - Server: server, + return fmt.Errorf("failed to add %v server to %v upstream: server already exists", server.Server, upstream) } - jsonServer, err := json.Marshal(upsServer) + jsonServer, err := json.Marshal(server) if err != nil { - return fmt.Errorf("error marshalling upstream server %v: %v", upsServer, err) + return fmt.Errorf("error marshalling upstream server %v: %v", server, err) } url := fmt.Sprintf("%v/%v/http/upstreams/%v/servers/", client.apiEndpoint, APIVersion, upstream) @@ -186,13 +177,13 @@ func (client *NginxClient) AddHTTPServer(upstream string, server string) error { resp, err := client.httpClient.Post(url, "application/json", bytes.NewBuffer(jsonServer)) if err != nil { - return fmt.Errorf("failed to add %v server to %v upstream: %v", server, upstream, err) + return fmt.Errorf("failed to add %v server to %v upstream: %v", server.Server, upstream, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusCreated { mainErr := fmt.Errorf("failed to add %v server to %v upstream: expected %v response, got %v", - server, upstream, http.StatusCreated, resp.StatusCode) + server.Server, upstream, http.StatusCreated, resp.StatusCode) return createResponseMismatchError(resp.Body, mainErr) } @@ -234,7 +225,7 @@ func (client *NginxClient) DeleteHTTPServer(upstream string, server string) erro // UpdateHTTPServers updates the servers of the upstream. // Servers that are in the slice, but don't exist in NGINX will be added to NGINX. // Servers that aren't in the slice, but exist in NGINX, will be removed from NGINX. -func (client *NginxClient) UpdateHTTPServers(upstream string, servers []string) ([]string, []string, error) { +func (client *NginxClient) UpdateHTTPServers(upstream string, servers []UpstreamServer) ([]UpstreamServer, []UpstreamServer, error) { serversInNginx, err := client.GetHTTPServers(upstream) if err != nil { return nil, nil, fmt.Errorf("failed to update servers of %v upstream: %v", upstream, err) @@ -250,7 +241,7 @@ func (client *NginxClient) UpdateHTTPServers(upstream string, servers []string) } for _, server := range toDelete { - err := client.DeleteHTTPServer(upstream, server) + err := client.DeleteHTTPServer(upstream, server.Server) if err != nil { return nil, nil, fmt.Errorf("failed to update servers of %v upstream: %v", upstream, err) } @@ -259,11 +250,11 @@ func (client *NginxClient) UpdateHTTPServers(upstream string, servers []string) return toAdd, toDelete, nil } -func determineUpdates(updatedServers []string, nginxServers []string) (toAdd []string, toRemove []string) { +func determineUpdates(updatedServers []UpstreamServer, nginxServers []UpstreamServer) (toAdd []UpstreamServer, toRemove []UpstreamServer) { for _, server := range updatedServers { found := false for _, serverNGX := range nginxServers { - if server == serverNGX { + if server.Server == serverNGX.Server { found = true break } @@ -276,7 +267,7 @@ func determineUpdates(updatedServers []string, nginxServers []string) (toAdd []s for _, serverNGX := range nginxServers { found := false for _, server := range updatedServers { - if serverNGX == server { + if serverNGX.Server == server.Server { found = true break } @@ -289,30 +280,15 @@ func determineUpdates(updatedServers []string, nginxServers []string) (toAdd []s return } -// GetHTTPServers returns the servers of the upsteam from NGINX. -func (client *NginxClient) GetHTTPServers(upstream string) ([]string, error) { - peers, err := client.getUpstreamPeers(upstream) - if err != nil { - return nil, fmt.Errorf("error getting servers of %v upstream: %v", upstream, err) - } - - var servers []string - for _, peer := range peers.Peers { - servers = append(servers, peer.Server) - } - - return servers, nil -} - -func (client *NginxClient) getIDOfHTTPServer(upstream string, name string) (int, error) { - peers, err := client.getUpstreamPeers(upstream) +func (client *NginxClient) getIDOfHTTPServer(upstream string, name string) (int64, error) { + servers, err := client.GetHTTPServers(upstream) if err != nil { return -1, fmt.Errorf("error getting id of server %v of upstream %v: %v", name, upstream, err) } - for _, p := range peers.Peers { - if p.Server == name { - return p.ID, nil + for _, s := range servers { + if s.Server == name { + return s.ID, nil } } diff --git a/nginx-controller/nginx/templates/nginx-plus.ingress.tmpl b/nginx-controller/nginx/templates/nginx-plus.ingress.tmpl index 47b34d92c0..4b7145e2f0 100644 --- a/nginx-controller/nginx/templates/nginx-plus.ingress.tmpl +++ b/nginx-controller/nginx/templates/nginx-plus.ingress.tmpl @@ -3,7 +3,7 @@ upstream {{$upstream.Name}} { zone {{$upstream.Name}} 256k; {{if $upstream.LBMethod }}{{$upstream.LBMethod}};{{end}} {{range $server := $upstream.UpstreamServers}} - server {{$server.Address}}:{{$server.Port}};{{end}} + server {{$server.Address}}:{{$server.Port}} max_fails={{$server.MaxFails}} fail_timeout={{$server.FailTimeout}};{{end}} {{if $upstream.StickyCookie}} sticky cookie {{$upstream.StickyCookie}}; {{end}} diff --git a/nginx-controller/nginx/templates/nginx.ingress.tmpl b/nginx-controller/nginx/templates/nginx.ingress.tmpl index f45bf76a79..6ac660f9d5 100644 --- a/nginx-controller/nginx/templates/nginx.ingress.tmpl +++ b/nginx-controller/nginx/templates/nginx.ingress.tmpl @@ -2,7 +2,7 @@ upstream {{$upstream.Name}} { {{if $upstream.LBMethod }}{{$upstream.LBMethod}};{{end}} {{range $server := $upstream.UpstreamServers}} - server {{$server.Address}}:{{$server.Port}};{{end}} + server {{$server.Address}}:{{$server.Port}} max_fails={{$server.MaxFails}} fail_timeout={{$server.FailTimeout}};{{end}} {{if $.Keepalive}}keepalive {{$.Keepalive}};{{end}} }{{end}} diff --git a/nginx-controller/nginx/templates/templates_test.go b/nginx-controller/nginx/templates/templates_test.go index 8ccf17a006..ee88cfb8ef 100644 --- a/nginx-controller/nginx/templates/templates_test.go +++ b/nginx-controller/nginx/templates/templates_test.go @@ -16,7 +16,12 @@ const nginxPlusMainTmpl = "nginx-plus.tmpl" var testUps = nginx.Upstream{ Name: "test", UpstreamServers: []nginx.UpstreamServer{ - {"127.0.0.1", "8181"}, + { + Address: "127.0.0.1", + Port: "8181", + MaxFails: 0, + FailTimeout: "1s", + }, }, }