Add gRPC support

nginx · Apr 30, 2018 · 96910e5 · 96910e5
1 parent 4d337d4
commit 96910e5
Show file tree

Hide file tree

Showing 10 changed files with 158 additions and 16 deletions.
diff --git a/examples/customization/README.md b/examples/customization/README.md
@@ -9,12 +9,12 @@ The table below summarizes all of the options. For some of them, there are examp
 | Annotation | ConfigMaps Key | Description | Default | Example |
 | ---------- | -------------- | ----------- | ------- | ------- |
 | `kubernetes.io/ingress.class` | N/A | Specifies which Ingress controller must handle the Ingress resource. Set to `nginx` to make NGINX Ingress controller handle it. | N/A | [Multiple Ingress controllers](../multiple-ingress-controllers). |
-| `nginx.org/proxy-connect-timeout` | `proxy-connect-timeout` | Sets the value of the [proxy_connect_timeout](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout) directive. | `60s` | |
-| `nginx.org/proxy-read-timeout` | `proxy-read-timeout` | Sets the value of the [proxy_read_timeout](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout) directive. | `60s` | |
+| `nginx.org/proxy-connect-timeout` | `proxy-connect-timeout` | Sets the value of the [proxy_connect_timeout](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout) and [grpc_connect_timeout](http://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_connect_timeout) directive. | `60s` | |
+| `nginx.org/proxy-read-timeout` | `proxy-read-timeout` | Sets the value of the [proxy_read_timeout](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout) and [grpc_read_timeout](http://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_read_timeout) directive. | `60s` | |
 | `nginx.org/client-max-body-size` | `client-max-body-size` | Sets the value of the [client_max_body_size](http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size) directive. | `1m` | |
 | `nginx.org/proxy-buffering` | `proxy-buffering` | Enables or disables [buffering of responses](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) from the proxied server. | `True` | |
 | `nginx.org/proxy-buffers` | `proxy-buffers` | Sets the value of the [proxy_buffers](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) directive. | Depends on the platform. | |
-| `nginx.org/proxy-buffer-size` | `proxy-buffer-size` | Sets the value of the [proxy_buffer_size](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) directive | Depends on the platform. | |
+| `nginx.org/proxy-buffer-size` | `proxy-buffer-size` | Sets the value of the [proxy_buffer_size](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) and [grpc_buffer_size](http://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_buffer_size) directives. | Depends on the platform. | |
 | `nginx.org/proxy-max-temp-file-size` | `proxy-max-temp-file-size` | Sets the value of the  [proxy_max_temp_file_size](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_max_temp_file_size) directive. | `1024m` | |
 | `nginx.org/proxy-hide-headers` | `proxy-hide-headers` | Sets the value of one or more  [proxy_hide_header](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header) directives. Example: `"nginx.org/proxy-hide-headers": "header-a,header-b"` | N/A | |
 | `nginx.org/proxy-pass-headers` | `proxy-pass-headers` | Sets the value of one or more   [proxy_pass_header](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass_header) directives. Example: `"nginx.org/proxy-pass-headers": "header-a,header-b"` | N/A | |
@@ -51,6 +51,7 @@ The table below summarizes all of the options. For some of them, there are examp
 | N/A | `proxy-protocol` | Enables PROXY Protocol for incoming connections. | `False` | [Proxy Protocol](../proxy-protocol). |
 | `nginx.org/rewrites` | N/A | Configures URI rewriting. | N/A | [Rewrites Support](../rewrites). |
 | `nginx.org/ssl-services` | N/A | Enables HTTPS when connecting to the endpoints of services. | N/A | [SSL Services Support](../ssl-services). |
+| `nginx.org/grpc-services` | N/A | Enables gRPC for services. | N/A | [GRPC Services Support](../grpc-services).|
 | `nginx.org/websocket-services` | N/A | Enables WebSocket for services. | N/A | [WebSocket support](../websocket). |
 | `nginx.org/max-fails` | `max-fails` | Sets the value of the [max_fails](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails) parameter of the `server` directive. | `1` | |
 | `nginx.org/fail-timeout` | `fail-timeout` | Sets the value of the [fail_timeout](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout) parameter of the `server` directive. | `10s` | |

diff --git a/examples/customization/nginx-config.yaml b/examples/customization/nginx-config.yaml
@@ -4,8 +4,8 @@ metadata:
   name: nginx-config
   namespace: nginx-ingress
 data:
-  proxy-connect-timeout: "10s" # default is "60s". See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout
-  proxy-read-timeout: "10s" # default is "60s". See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout
+  proxy-connect-timeout: "10s" # default is "60s". Sets the value of proxy_connect_timeout and grpc_connect_timeout directives. See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout , http://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_connect_timeout
+  proxy-read-timeout: "10s" # default is "60s". Sets the value of proxy_read_timeout and grpc_read_timeout directives. See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout , http://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_read_timeout
   proxy-hide-headers: "header-a,header-b" # No default. Sets the value of one or more proxy_hide_header directives. See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header
   proxy-pass-headers: "header-a,header-b" # No default. Sets the value of one or more proxy_pass_header directives. See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass_header
   client-max-body-size: "2m" # default is "1m". See http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size
@@ -14,7 +14,7 @@ data:
   http2: "True" # default is "False". Enables HTTP/2 in servers with SSL enabled. See https://nginx.org/en/docs/http/ngx_http_v2_module.html
   proxy-buffering: "False" # default is "True". Enables or disables buffering of responses from the proxied server. See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering
   proxy-buffers: "16 8k" # default value depends on the platform. See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers
-  proxy-buffer-size: "2k" # default value depends on the platform. See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size
+  proxy-buffer-size: "2k" # default value depends on the platform. Sets proxy_buffer_size and grpc_buffer_size. See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size , http://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_buffer_size
   proxy-max-temp-file-size: "0" # default is "1024m". See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_max_temp_file_size
   log-format: '{ "@timestamp": "$time_iso8601", "@version": "1", "clientip": "$remote_addr", "tag": "ingress", "remote_user": "$remote_user", "bytes": $bytes_sent, "duration": $request_time, "status": $status, "request": "$request_uri", "urlpath": "$uri", "urlquery": "$args", "method": "$request_method", "referer": "$http_referer", "useragent": "$http_user_agent", "software": "nginx", "version": "$nginx_version", "host": "$host", "upstream": "$upstream_addr", "upstream-status": "$upstream_status" }'
   # log-format default is set in the nginx.conf.tmpl file. Also see http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format

diff --git a/examples/grpc-services/README.md b/examples/grpc-services/README.md
@@ -0,0 +1,39 @@
+# gRPC support
+
+To add gRPC support to an application with NGINX Ingress controllers, you need to add the **nginx.org/grpc-services** annotation to your Ingress resource definition. The annotation specifies which services are gRPC services. The annotation syntax is as follows:
+```
+nginx.org/grpc-services: "service1[,service2,...]"
+```
+
+In the following example we load balance three applications, one of which is using gRPC:
+```yaml
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: grpc-ingress
+  annotations:
+    nginx.org/grpc-services: "grpc-svc"
+    kubernetes.io/ingress.class: "nginx"
+spec:
+  tls:
+  - hosts:
+    - grpc.example.com
+    secretName: grpc-secret
+  rules:
+  - host: grpc.example.com
+    http:
+      paths:
+      - path: /helloworld.Greeter
+        backend:
+          serviceName: grpc-svc
+          servicePort: 50051
+      - path: /tea
+        backend:
+          serviceName: tea-svc
+          servicePort: 80
+      - path: /coffee
+        backend:
+          serviceName: coffee-svc
+          servicePort: 80
+```
+*grpc-svc* is a service for the gRPC application. The service becomes available at the `/helloworld.Greeter` path. Note how we used the **nginx.org/grpc-services** annotation.
diff --git a/nginx-controller/Dockerfile b/nginx-controller/Dockerfile
@@ -1,4 +1,4 @@
-FROM nginx:1.13.8
+FROM nginx:1.13.12
 
 # forward nginx access and error logs to stdout and stderr of the ingress
 # controller process

diff --git a/nginx-controller/DockerfileForAlpine b/nginx-controller/DockerfileForAlpine
@@ -1,4 +1,4 @@
-FROM nginx:1.13.8-alpine
+FROM nginx:1.13.12-alpine
 
 # forward nginx access and error logs to stdout and stderr of the ingress
 # controller process

diff --git a/nginx-controller/DockerfileForPlus b/nginx-controller/DockerfileForPlus
@@ -2,7 +2,7 @@ FROM debian:stretch-slim
 
 LABEL maintainer="NGINX Docker Maintainers <[email protected]>"
 
-ENV NGINX_PLUS_VERSION 1.13.7-2~stretch
+ENV NGINX_PLUS_VERSION 15-2~stretch
 
 # Download certificate and key from the customer portal (https://cs.nginx.com)
 # and copy to the build context

diff --git a/nginx-controller/nginx/configurator.go b/nginx-controller/nginx/configurator.go
@@ -181,6 +181,13 @@ func (cnf *Configurator) generateNginxCfg(ingEx *IngressEx, pems map[string]stri
 	spServices := getSessionPersistenceServices(ingEx)
 	rewrites := getRewrites(ingEx)
 	sslServices := getSSLServices(ingEx)
+	grpcServices := getGrpcServices(ingEx)
+
+	// HTTP2 is required for gRPC to function
+	if len(grpcServices) > 0 && !ingCfg.HTTP2 {
+		glog.Errorf("Ingress %s/%s: annotation nginx.org/grpc-services requires HTTP2, ignoring", ingEx.Ingress.Namespace, ingEx.Ingress.Name)
+		grpcServices = make(map[string]bool)
+	}
 
 	if ingEx.Ingress.Spec.Backend != nil {
 		name := getNameForUpstream(ingEx.Ingress, emptyHost, ingEx.Ingress.Spec.Backend.ServiceName)
@@ -236,6 +243,14 @@ func (cnf *Configurator) generateNginxCfg(ingEx *IngressEx, pems map[string]stri
 		var locations []Location
 		rootLocation := false
 
+		grpcOnly := true
+		for _, path := range rule.HTTP.Paths {
+			if _, exists := grpcServices[path.Backend.ServiceName]; !exists {
+				grpcOnly = false
+				break
+			}
+		}
+
 		for _, path := range rule.HTTP.Paths {
 			upsName := getNameForUpstream(ingEx.Ingress, rule.Host, path.Backend.ServiceName)
 
@@ -244,7 +259,8 @@ func (cnf *Configurator) generateNginxCfg(ingEx *IngressEx, pems map[string]stri
 				upstreams[upsName] = upstream
 			}
 
-			loc := createLocation(pathOrDefault(path.Path), upstreams[upsName], &ingCfg, wsServices[path.Backend.ServiceName], rewrites[path.Backend.ServiceName], sslServices[path.Backend.ServiceName])
+			loc := createLocation(pathOrDefault(path.Path), upstreams[upsName], &ingCfg, wsServices[path.Backend.ServiceName], rewrites[path.Backend.ServiceName],
+				sslServices[path.Backend.ServiceName], grpcServices[path.Backend.ServiceName])
 			locations = append(locations, loc)
 
 			if loc.Path == "/" {
@@ -254,11 +270,18 @@ func (cnf *Configurator) generateNginxCfg(ingEx *IngressEx, pems map[string]stri
 
 		if rootLocation == false && ingEx.Ingress.Spec.Backend != nil {
 			upsName := getNameForUpstream(ingEx.Ingress, emptyHost, ingEx.Ingress.Spec.Backend.ServiceName)
-			loc := createLocation(pathOrDefault("/"), upstreams[upsName], &ingCfg, wsServices[ingEx.Ingress.Spec.Backend.ServiceName], rewrites[ingEx.Ingress.Spec.Backend.ServiceName], sslServices[ingEx.Ingress.Spec.Backend.ServiceName])
+
+			loc := createLocation(pathOrDefault("/"), upstreams[upsName], &ingCfg, wsServices[ingEx.Ingress.Spec.Backend.ServiceName], rewrites[ingEx.Ingress.Spec.Backend.ServiceName],
+				sslServices[ingEx.Ingress.Spec.Backend.ServiceName], grpcServices[ingEx.Ingress.Spec.Backend.ServiceName])
 			locations = append(locations, loc)
+
+			if _, exists := grpcServices[ingEx.Ingress.Spec.Backend.ServiceName]; !exists {
+				grpcOnly = false
+			}
 		}
 
 		server.Locations = locations
+		server.GRPCOnly = grpcOnly
 
 		servers = append(servers, server)
 	}
@@ -517,6 +540,18 @@ func getSSLServices(ingEx *IngressEx) map[string]bool {
 	return sslServices
 }
 
+func getGrpcServices(ingEx *IngressEx) map[string]bool {
+	grpcServices := make(map[string]bool)
+
+	if services, exists := ingEx.Ingress.Annotations["nginx.org/grpc-services"]; exists {
+		for _, svc := range strings.Split(services, ",") {
+			grpcServices[svc] = true
+		}
+	}
+
+	return grpcServices
+}
+
 func getSessionPersistenceServices(ingEx *IngressEx) map[string]string {
 	spServices := make(map[string]string)
 
@@ -595,7 +630,7 @@ func parsePort(value string) (int, error) {
 	return int(port), nil
 }
 
-func createLocation(path string, upstream Upstream, cfg *Config, websocket bool, rewrite string, ssl bool) Location {
+func createLocation(path string, upstream Upstream, cfg *Config, websocket bool, rewrite string, ssl bool, grpc bool) Location {
 	loc := Location{
 		Path:                 path,
 		Upstream:             upstream,
@@ -605,6 +640,7 @@ func createLocation(path string, upstream Upstream, cfg *Config, websocket bool,
 		Websocket:            websocket,
 		Rewrite:              rewrite,
 		SSL:                  ssl,
+		GRPC:                 grpc,
 		ProxyBuffering:       cfg.ProxyBuffering,
 		ProxyBuffers:         cfg.ProxyBuffers,
 		ProxyBufferSize:      cfg.ProxyBufferSize,

diff --git a/nginx-controller/nginx/nginx.go b/nginx-controller/nginx/nginx.go
@@ -60,6 +60,7 @@ type Server struct {
 	SSL                   bool
 	SSLCertificate        string
 	SSLCertificateKey     string
+	GRPCOnly              bool
 	StatusZone            string
 	HTTP2                 bool
 	RedirectToHTTPS       bool
@@ -99,6 +100,7 @@ type Location struct {
 	Websocket            bool
 	Rewrite              string
 	SSL                  bool
+	GRPC                 bool
 	ProxyBuffering       bool
 	ProxyBuffers         string
 	ProxyBufferSize      string

diff --git a/nginx-controller/nginx/templates/nginx-plus.ingress.tmpl b/nginx-controller/nginx/templates/nginx-plus.ingress.tmpl
@@ -12,9 +12,11 @@ upstream {{$upstream.Name}} {
 
 {{range $server := .Servers}}
 server {
+	{{if not $server.GRPCOnly}}
 	{{range $port := $server.Ports}}
 	listen {{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
 	{{- end}}
+	{{end}}
 	{{if $server.SSL}}
 	{{- range $port := $server.SSLPorts}}
 	listen {{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
@@ -33,19 +35,23 @@ server {
 
 	status_zone {{$server.StatusZone}};
 
+	{{if not $server.GRPCOnly}}
 	{{range $proxyHideHeader := $server.ProxyHideHeaders}}
 	proxy_hide_header {{$proxyHideHeader}};{{end}}
 	{{range $proxyPassHeader := $server.ProxyPassHeaders}}
 	proxy_pass_header {{$proxyPassHeader}};{{end}}
+	{{end}}
 
 	{{if $server.SSL}}
+	{{if not $server.GRPCOnly}}
 	{{- if $server.SSLRedirect}}
 	if ($scheme = http) {
 		return 301 https://$host:{{index $server.SSLPorts 0}}$request_uri;
 	}
 	{{- end}}
 	{{- if $server.HSTS}}
 	add_header Strict-Transport-Security "max-age={{$server.HSTSMaxAge}}; {{if $server.HSTSIncludeSubdomains}}includeSubDomains; {{end}}preload" always;{{end}}
+	{{end}}
 	{{- end}}
 	{{- if $server.RedirectToHTTPS}}
 	if ($http_x_forwarded_proto = 'http') {
@@ -56,9 +62,9 @@ server {
 	{{ if $server.JWTKey}}
 	auth_jwt_key_file {{$server.JWTKey}};
 	auth_jwt "{{$server.JWTRealm}}"{{if $server.JWTToken}} token={{$server.JWTToken}}{{end}};
-	
+
 	{{- if $server.JWTLoginURL}}
-	error_page 401 @login_url;	
+	error_page 401 @login_url;
 	location @login_url {
 		internal;
 		return 302 {{$server.JWTLoginURL}};
@@ -73,6 +79,32 @@ server {
 
 	{{range $location := $server.Locations}}
 	location {{$location.Path}} {
+		{{if $location.GRPC}}
+
+		{{- if $location.LocationSnippets}}
+		{{range $value := $location.LocationSnippets}}
+		{{$value}}{{end}}
+		{{- end}}
+
+		grpc_connect_timeout {{$location.ProxyConnectTimeout}};
+		grpc_read_timeout {{$location.ProxyReadTimeout}};
+		grpc_set_header Host $host;
+		grpc_set_header X-Real-IP $remote_addr;
+		grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		grpc_set_header X-Forwarded-Host $host;
+		grpc_set_header X-Forwarded-Port $server_port;
+		grpc_set_header X-Forwarded-Proto $scheme;
+
+		{{- if $location.ProxyBufferSize}}
+		grpc_buffer_size {{$location.ProxyBufferSize}};
+		{{- end}}
+
+		{{if $location.SSL}}
+		grpc_pass grpcs://{{$location.Upstream.Name}}
+		{{else}}
+		grpc_pass grpc://{{$location.Upstream.Name}};
+		{{end}}
+		{{else}}
 		proxy_http_version 1.1;
 		{{if $location.Websocket}}
 		proxy_set_header Upgrade $http_upgrade;
@@ -110,5 +142,6 @@ server {
 		{{else}}
 		proxy_pass http://{{$location.Upstream.Name}}{{$location.Rewrite}};
 		{{end}}
+		{{end}}
 	}{{end}}
 }{{end}}
diff --git a/nginx-controller/nginx/templates/nginx.ingress.tmpl b/nginx-controller/nginx/templates/nginx.ingress.tmpl
@@ -9,9 +9,11 @@ upstream {{$upstream.Name}} {
 {{range $server := .Servers}}
 {{if $server.IngressResource}} # *Master*, configured in Ingress Resource: {{$server.IngressResource}}{{end}}
 server {
+	{{if not $server.GRPCOnly}}
 	{{range $port := $server.Ports}}
 	listen {{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
 	{{- end}}
+	{{end}}
 	{{if $server.SSL}}
 	{{- range $port := $server.SSLPorts}}
 	listen {{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
@@ -27,12 +29,13 @@ server {
 	server_tokens {{$server.ServerTokens}};
 
 	server_name {{$server.Name}};
-	
+
 	{{range $proxyHideHeader := $server.ProxyHideHeaders}}
 	proxy_hide_header {{$proxyHideHeader}};{{end}}
 	{{range $proxyPassHeader := $server.ProxyPassHeaders}}
 	proxy_pass_header {{$proxyPassHeader}};{{end}}
 	{{if $server.SSL}}
+	{{if not $server.GRPCOnly}}
 	{{- if $server.SSLRedirect}}
 	if ($scheme = http) {
 		return 301 https://$host:{{index $server.SSLPorts 0}}$request_uri;
@@ -41,6 +44,7 @@ server {
 	{{- if $server.HSTS}}
 	proxy_hide_header Strict-Transport-Security;
 	add_header Strict-Transport-Security "max-age={{$server.HSTSMaxAge}}; {{if $server.HSTSIncludeSubdomains}}includeSubDomains; {{end}}preload" always;{{end}}
+	{{end}}
 	{{- end}}
 	{{- if $server.RedirectToHTTPS}}
 	if ($http_x_forwarded_proto = 'http') {
@@ -56,6 +60,32 @@ server {
 	{{range $location := $server.Locations}}
 	{{if $location.IngressResource}} # *Minion*, configured in Ingress Resource: {{$location.IngressResource}}{{end}}
 	location {{$location.Path}} {
+		{{if $location.GRPC}}
+		{{- if $location.LocationSnippets}}
+		{{range $value := $location.LocationSnippets}}
+		{{$value}}{{end}}
+		{{- end}}
+
+		grpc_connect_timeout {{$location.ProxyConnectTimeout}};
+		proxy_read_timeout {{$location.ProxyReadTimeout}};
+		client_max_body_size {{$location.ClientMaxBodySize}};
+		grpc_set_header Host $host;
+		grpc_set_header X-Real-IP $remote_addr;
+		grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		grpc_set_header X-Forwarded-Host $host;
+		grpc_set_header X-Forwarded-Port $server_port;
+		grpc_set_header X-Forwarded-Proto {{if $server.RedirectToHTTPS}}https{{else}}$scheme{{end}};
+
+		{{- if $location.ProxyBufferSize}}
+		grpc_buffer_size {{$location.ProxyBufferSize}};
+		{{- end}}
+
+		{{if $location.SSL}}
+		grpc_pass grpcs://{{$location.Upstream.Name}}{{$location.Rewrite}};
+		{{else}}
+		grpc_pass grpc://{{$location.Upstream.Name}}{{$location.Rewrite}};
+		{{end}}
+		{{else}}
 		proxy_http_version 1.1;
 		{{if $location.Websocket}}
 		proxy_set_header Upgrade $http_upgrade;
@@ -78,8 +108,8 @@ server {
 		proxy_set_header X-Forwarded-Host $host;
 		proxy_set_header X-Forwarded-Port $server_port;
 		proxy_set_header X-Forwarded-Proto {{if $server.RedirectToHTTPS}}https{{else}}$scheme{{end}};
-
 		proxy_buffering {{if $location.ProxyBuffering}}on{{else}}off{{end}};
+
 		{{- if $location.ProxyBuffers}}
 		proxy_buffers {{$location.ProxyBuffers}};
 		{{- end}}
@@ -94,5 +124,6 @@ server {
 		{{else}}
 		proxy_pass http://{{$location.Upstream.Name}}{{$location.Rewrite}};
 		{{end}}
+		{{end}}
 	}{{end}}
 }{{end}}