From 46b807707b384fadc68d0c06d7d747a6e8f294f0 Mon Sep 17 00:00:00 2001 From: Michael Pleshakov Date: Wed, 12 Sep 2018 11:15:52 +0100 Subject: [PATCH] Support custom annotations Now it is possible to get access to the annotations of an Ingress resource from the template for Ingress resources. This allows users to implement custom annotations by modifying the template to insert NGINX configuration based on the presence on an annotation or its value. Additionally, the Ingress name and namespace is also avaiable in the template. --- docs/custom-annotations.md | 91 ++++++++++ examples/custom-annotations/README.md | 159 ++++++++++++++++++ internal/nginx/configurator.go | 9 +- internal/nginx/configurator_test.go | 34 +++- internal/nginx/nginx.go | 14 +- .../nginx/templates/nginx-plus.ingress.tmpl | 5 +- internal/nginx/templates/nginx.ingress.tmpl | 6 +- internal/nginx/templates/templates_test.go | 10 +- 8 files changed, 314 insertions(+), 14 deletions(-) create mode 100644 docs/custom-annotations.md create mode 100644 examples/custom-annotations/README.md diff --git a/docs/custom-annotations.md b/docs/custom-annotations.md new file mode 100644 index 0000000000..877f1d6a44 --- /dev/null +++ b/docs/custom-annotations.md @@ -0,0 +1,91 @@ +# Custom Annotations + +Custom annotations enable you to quickly extend the Ingress Controller to support many advanced features of NGINX, such as rate limiting, caching, etc. + +## What are Custom Annotations + +NGINX Ingress Controller supports a number of annotations that fine tune NGINX configuration (for example, connection timeouts) or enable additional features (for example, JWT validation). The complete list of annotations is available [here](../examples/customization). + +The annotations are provided only for the most common features and use cases, meaning that not every NGINX feature or a customization option is available through the annotations. Additionally, even if an annotation is available, it might not give you the satisfactory level of control of a particular NGINX feature. + +Custom annotations allow you to add an annotation for an NGINX feature that is not available as a regular annotation. In contrast with regular annotations, to add a custom annotation, you don't need to modify the Ingress Controller source code -- just modify the template. Additionally, with a custom annotation, you get full control of how the feature is implemented in NGINX configuration. + +## Usage + +The Ingress Controller generates NGINX configuration for Ingress resources by executing a configuration template. See [NGINX template](../internal/nginx/templates/nginx.ingress.tmpl) or [NGINX Plus template](../internal/nginx/templates/nginx-plus.ingress.tmpl). + +To support custom annotations, the template has access to the information about the Ingress resource - its *name*, *namespace* and *annotations*. It is possible to check if a particular annotation present in the Ingress resource and conditionally insert NGINX configuration directives at multiple NGINX contexts - `http`, `server`, `location` or `upstream`. Additionally, you can get the value that is set to the annotation. + +Consider the following excerpt from the template, which was extended to support two custom annotations: + +``` +# This is the configuration for {{$.Ingress.Name}}/{{$.Ingress.Namespace}} + +{{if index $.Ingress.Annotations "custom.nginx.org/feature-a"}} +# Insert config for feature A if the annotation is set +{{end}} + +{{with $value := index $.Ingress.Annotations "custom.nginx.org/feature-b"}} +# Insert config for feature B if the annotation is set +# Print the value assigned to the annotation: {{$value}} +{{end}} +``` + +Consider the following Ingress resource and note how we set two annotations: +```yaml +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: example-ingress + namespace: production + annotations: + custom.nginx.org/feature-a: "on" + custom.nginx.org/feature-b: "512" +spec: + rules: + - host: example.com + . . . +``` + +Assuming that the Ingress Controller is using that customized template, it will generate a config for the Ingress resource that will include the following part, generated by our template excerpt: +``` +# This is the configuration for cafe-ingress/default + +# Insert config for feature A if the annotation is set + + + +# Insert config for feature B if the annotation is set +# Print the value assigned to the annotation: 512 +``` + +**Notes**: +* You can customize the template to insert you custom annotations via [custom templates](../examples/custom-templates). +* The Ingress Controller uses go templates to generate NGINX config. You can read more information about go templates [here](https://golang.org/pkg/text/template/). + +See the examples in the next section that use custom annotations to configure NGINX features. + +### Custom Annotations with Mergeable Ingress Resources + +A Mergeable Ingress resource consists of multiple Ingress resources - one master and one or several minions. Read more about Mergeable Ingress resources [here](../examples/mergeable-ingress-types). + +If you'd like to use custom annotations with Mergeable Ingress resources, please keep the following in mind: +* Custom annotations can be used in the Master and in Minions. For Minions, you can access them in the template only when processing locations. + + If you access `$.Ingress` anywhere in the Ingress template, you will get the master Ingress resource. To access a Minion Ingress resource, use `$location.MinionIngress`. However, it is only available when processing locations: + ``` + {{range $location := $server.Locations}} + location {{$location.Path}} { + {{with $location.MinionIngress}} + # location for minion {{$location.MinionIngress.Namespace}}/{{$location.MinionIngress.Name}} + {{end}} + . . . + } {{end}} + ``` + **Note**: `$location.MinionIngress` is a pointer. When a regular Ingress resource is processed in the template, the value of the pointer is `nil`. Thus, it is important that you check that `$location.MinionIngress` is not `nil` as in the example above using the `with` action. + +* Minions do not inherent custom annotations of the master. + +## Example + +See the [custom annotations example](../examples/custom-annotations). \ No newline at end of file diff --git a/examples/custom-annotations/README.md b/examples/custom-annotations/README.md new file mode 100644 index 0000000000..83d8061cb0 --- /dev/null +++ b/examples/custom-annotations/README.md @@ -0,0 +1,159 @@ +# Custom Annotations + +Custom annotations enable you to quickly extend the Ingress Controller to support many advanced features of NGINX, such as rate limiting, caching, etc. + +Let's create a set of custom annotations to support [rate-limiting](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html): +* `custom.nginx.org/rate-limiting` - enables rate-limiting. +* `custom.nginx.org/rate-limiting-rate` - configures the rate of rate-limiting, with the default of `1r/s`. +* `custom.nginx.org/rate-limiting-burst` - configures the maximum bursts size of requests with the default of `3`. + +## Prerequisites + +* Read the [custom annotations doc](../../docs/custom-annotations.md) before going through this example first. +* Read about [custom templates](../custom-templates). + +## Step 1 - Customize the Template + +Customize the template for Ingress resources to include the logic to handle and apply the annotations. + +1. Create a ConfigMap file with the customized template (`nginx-config.yaml`): + ```yaml + kind: ConfigMap + apiVersion: v1 + metadata: + name: nginx-config + namespace: nginx-ingress + data: + ingress-template: | + . . . + # handling custom.nginx.org/rate-limiting` and custom.nginx.org/rate-limiting-rate + + {{if index $.Ingress.Annotations "custom.nginx.org/rate-limiting"}} + {{$rate := index $.Ingress.Annotations "custom.nginx.org/rate-limiting-rate"}} + limit_req_zone $binary_remote_addr zone={{$.Ingress.Namespace}}-{{$.Ingress.Name}}:10m rate={{if $rate}}{{$rate}}{{else}}1r/s{{end}}; + {{end}} + + . . . + + {{range $server := .Servers}} + server { + + . . . + + {{range $location := $server.Locations}} + location {{$location.Path}} { + + . . . + + # handling custom.nginx.org/rate-limiting and custom.nginx.org/rate-limiting-burst + + {{if index $.Ingress.Annotations "custom.nginx.org/rate-limiting"}} + {{$burst := index $.Ingress.Annotations "custom.nginx.org/rate-limiting-burst"}} + limit_req zone={{$.Ingress.Namespace}}-{{$.Ingress.Name}} burst={{if $burst}}{{$burst}}{{else}}3{{end}} nodelay; + {{end}} + + . . . + ``` + + The customization above consists of two parts: + * handling the `custom.nginx.org/rate-limiting` and `custom.nginx.org/rate-limiting-rate` annotations in the `http` context. + * handling the `custom.nginx.org/rate-limiting` and `custom.nginx.org/rate-limiting-burst` annotation in the `location` context. + + **Note**: for the brevity, the unimportant for the example parts of the template are replaced with `. . .`. + +1. Apply the customized template: + ``` + $ kubectl apply -f nginx-config.yaml + ``` + +1. If the Ingress Controller fails to parse the customized template, it will attach an error event with the corresponding ConfigMap resource. You can see the events by running: + ``` + $ kubectl describe configmap nginx-config -n nginx-ingress + . . . + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Updated 12s (x2 over 25s) nginx-ingress-controller Configuration from nginx-ingress/nginx-config was updated + ``` + In this case, we got the `Updated` event meaning that the template was parsed successfully. + +### Step 2 - Use Custom Annotations in an Ingress Resource + +1. Create a file with the following Ingress resource (`cafe-ingress.yaml`) and use the custom annotations to enable rate-limiting: + ```yaml + apiVersion: extensions/v1beta1 + kind: Ingress + metadata: + name: cafe-ingress + annotations: + kubernetes.io/ingress.class: "nginx" + custom.nginx.org/rate-limiting: "on" + custom.nginx.org/rate-limiting-rate: "5r/s" + custom.nginx.org/rate-limiting-burst: "1" + spec: + rules: + - host: "cafe.example.com" + http: + paths: + - path: /tea + backend: + serviceName: tea-svc + servicePort: 80 + - path: /coffee + backend: + serviceName: coffee-svc + servicePort: 80 + ``` + +1. Apply the Ingress resource: + ``` + $ kubectl apply -f cafe-ingress.yaml + ``` + +1. Since it is possible that the value we put in `custom.nginx.org/rate-limiting-rate` or `custom.nginx.org/rate-limiting-burst` annotation might be considered invalid by NGINX, make sure to run the following command to check if the configuration for the Ingress resource was successfully applied. As with the ConfigMap resource, in case of an error, the Ingress Controller will attach an error event to the Ingress resource: + ``` + $ kubectl describe ingress cafe-ingress + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal AddedOrUpdated 2m nginx-ingress-controller Configuration for default/cafe-ingress was added or updated + ``` + In this case, the config was successfully applied. + +### Step 3 -- Take a Look at the Generated NGINX Config + +Take a look at the generated config for the cafe-ingress Ingress resource to see how the rate-limiting feature is enabled: +``` +$ kubectl exec -n nginx-ingress -- cat /etc/nginx/conf.d/default-cafe-ingress.conf +``` + +```nginx +# configuration for default/cafe-ingress + +. . . + +limit_req_zone $binary_remote_addr zone=default-cafe-ingress:10m rate=5r/s; + +server { + listen 80; + + . . . + + location /tea { + + limit_req zone=default-cafe-ingress burst=1 nodelay; + + . . . + } + + location /coffee { + + limit_req zone=default-cafe-ingress burst=1 nodelay; + + . . . + } + +. . . +} +``` +**Note**: the unimportant parts are replaced with `. . .`. \ No newline at end of file diff --git a/internal/nginx/configurator.go b/internal/nginx/configurator.go index 8e2ab6f472..28eb52d571 100644 --- a/internal/nginx/configurator.go +++ b/internal/nginx/configurator.go @@ -127,7 +127,6 @@ func (cnf *Configurator) generateNginxCfgForMergeableIngresses(mergeableIngs *Me masterNginxCfg := cnf.generateNginxCfg(mergeableIngs.Master, pems, jwtKeyFileName, isMinion) masterServer = masterNginxCfg.Servers[0] - masterServer.IngressResource = objectMetaToFileName(&mergeableIngs.Master.Ingress.ObjectMeta) masterServer.Locations = []Location{} for _, val := range masterNginxCfg.Upstreams { @@ -157,7 +156,7 @@ func (cnf *Configurator) generateNginxCfgForMergeableIngresses(mergeableIngs *Me for _, server := range nginxCfg.Servers { for _, loc := range server.Locations { - loc.IngressResource = objectMetaToFileName(&minion.Ingress.ObjectMeta) + loc.MinionIngress = &nginxCfg.Ingress locations = append(locations, loc) } for hcName, healthCheck := range server.HealthChecks { @@ -178,6 +177,7 @@ func (cnf *Configurator) generateNginxCfgForMergeableIngresses(mergeableIngs *Me Servers: []Server{masterServer}, Upstreams: upstreams, Keepalive: keepalive, + Ingress: masterNginxCfg.Ingress, } } @@ -379,6 +379,11 @@ func (cnf *Configurator) generateNginxCfg(ingEx *IngressEx, pems map[string]stri Upstreams: upstreamMapToSlice(upstreams), Servers: servers, Keepalive: keepalive, + Ingress: Ingress{ + Name: ingEx.Ingress.Name, + Namespace: ingEx.Ingress.Namespace, + Annotations: ingEx.Ingress.Annotations, + }, } } diff --git a/internal/nginx/configurator_test.go b/internal/nginx/configurator_test.go index 28aa0eadf2..2b1bffcb21 100644 --- a/internal/nginx/configurator_test.go +++ b/internal/nginx/configurator_test.go @@ -281,6 +281,13 @@ func createExpectedConfigForCafeIngressEx() IngressNginxConfig { HealthChecks: make(map[string]HealthCheck), }, }, + Ingress: Ingress{ + Name: "cafe-ingress", + Namespace: "default", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + }, + }, } return expected } @@ -453,7 +460,14 @@ func createExpectedConfigForMergeableCafeIngress() IngressNginxConfig { ProxyReadTimeout: "60s", ClientMaxBodySize: "1m", ProxyBuffering: true, - IngressResource: "default-cafe-ingress-coffee-minion", + MinionIngress: &Ingress{ + Name: "cafe-ingress-coffee-minion", + Namespace: "default", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + "nginx.org/mergeable-ingress-type": "minion", + }, + }, }, { Path: "/tea", @@ -462,7 +476,14 @@ func createExpectedConfigForMergeableCafeIngress() IngressNginxConfig { ProxyReadTimeout: "60s", ClientMaxBodySize: "1m", ProxyBuffering: true, - IngressResource: "default-cafe-ingress-tea-minion", + MinionIngress: &Ingress{ + Name: "cafe-ingress-tea-minion", + Namespace: "default", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + "nginx.org/mergeable-ingress-type": "minion", + }, + }, }, }, SSL: true, @@ -474,7 +495,14 @@ func createExpectedConfigForMergeableCafeIngress() IngressNginxConfig { SSLPorts: []int{443}, SSLRedirect: true, HealthChecks: make(map[string]HealthCheck), - IngressResource: "default-cafe-ingress-master", + }, + }, + Ingress: Ingress{ + Name: "cafe-ingress-master", + Namespace: "default", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + "nginx.org/mergeable-ingress-type": "master", }, }, } diff --git a/internal/nginx/nginx.go b/internal/nginx/nginx.go index 8bc642c722..15fad8bdf2 100644 --- a/internal/nginx/nginx.go +++ b/internal/nginx/nginx.go @@ -29,6 +29,14 @@ type IngressNginxConfig struct { Upstreams []Upstream Servers []Server Keepalive string + Ingress Ingress +} + +// Ingress holds information about an Ingress resource +type Ingress struct { + Name string + Namespace string + Annotations map[string]string } // Upstream describes an NGINX upstream @@ -96,9 +104,6 @@ type Server struct { Ports []int SSLPorts []int - - // Used for mergeable types - IngressResource string } // JWTRedirectLocation describes a location for redirecting client requests to a login URL for JWT Authentication @@ -133,8 +138,7 @@ type Location struct { ProxyMaxTempFileSize string JWTAuth *JWTAuth - // Used for mergeable types - IngressResource string + MinionIngress *Ingress } // MainConfig describe the main NGINX configuration file diff --git a/internal/nginx/templates/nginx-plus.ingress.tmpl b/internal/nginx/templates/nginx-plus.ingress.tmpl index 135aae8c88..599abbca2b 100644 --- a/internal/nginx/templates/nginx-plus.ingress.tmpl +++ b/internal/nginx/templates/nginx-plus.ingress.tmpl @@ -1,3 +1,4 @@ +# configuration for {{.Ingress.Namespace}}/{{.Ingress.Name}} {{range $upstream := .Upstreams}} upstream {{$upstream.Name}} { zone {{$upstream.Name}} 256k; @@ -18,7 +19,6 @@ upstream {{$upstream.Name}} { {{- end}} {{range $server := .Servers}} -{{if $server.IngressResource}} # *Master*, configured in Ingress Resource: {{$server.IngressResource}}{{end}} server { {{if not $server.GRPCOnly}} {{range $port := $server.Ports}} @@ -104,6 +104,9 @@ server { {{range $location := $server.Locations}} location {{$location.Path}} { + {{with $location.MinionIngress}} + # location for minion {{$location.MinionIngress.Namespace}}/{{$location.MinionIngress.Name}} + {{end}} {{if $location.GRPC}} {{if not $server.GRPCOnly}} error_page 400 @grpcerror400; diff --git a/internal/nginx/templates/nginx.ingress.tmpl b/internal/nginx/templates/nginx.ingress.tmpl index ffa287de60..7a0c8c8cf8 100644 --- a/internal/nginx/templates/nginx.ingress.tmpl +++ b/internal/nginx/templates/nginx.ingress.tmpl @@ -1,3 +1,4 @@ +# configuration for {{.Ingress.Namespace}}/{{.Ingress.Name}} {{range $upstream := .Upstreams}} upstream {{$upstream.Name}} { {{if $upstream.LBMethod }}{{$upstream.LBMethod}};{{end}} @@ -7,7 +8,6 @@ upstream {{$upstream.Name}} { }{{end}} {{range $server := .Servers}} -{{if $server.IngressResource}} # *Master*, configured in Ingress Resource: {{$server.IngressResource}}{{end}} server { {{if not $server.GRPCOnly}} {{range $port := $server.Ports}} @@ -58,8 +58,10 @@ server { {{- end}} {{range $location := $server.Locations}} - {{if $location.IngressResource}} # *Minion*, configured in Ingress Resource: {{$location.IngressResource}}{{end}} location {{$location.Path}} { + {{with $location.MinionIngress}} + # location for minion {{$location.MinionIngress.Namespace}}/{{$location.MinionIngress.Name}} + {{end}} {{if $location.GRPC}} {{if not $server.GRPCOnly}} error_page 400 @grpcerror400; diff --git a/internal/nginx/templates/templates_test.go b/internal/nginx/templates/templates_test.go index 221b64040f..6da5c310c0 100644 --- a/internal/nginx/templates/templates_test.go +++ b/internal/nginx/templates/templates_test.go @@ -55,7 +55,7 @@ var ingCfg = nginx.IngressNginxConfig{ SSLRedirect: true, Locations: []nginx.Location{ nginx.Location{ - Path: "/", + Path: "/tea", Upstream: testUps, ProxyConnectTimeout: "10s", ProxyReadTimeout: "10s", @@ -65,6 +65,10 @@ var ingCfg = nginx.IngressNginxConfig{ Realm: "closed site", Token: "$cookie_auth_token", }, + MinionIngress: &nginx.Ingress{ + Name: "tea-minion", + Namespace: "default", + }, }, }, HealthChecks: map[string]nginx.HealthCheck{"test": healthCheck}, @@ -78,6 +82,10 @@ var ingCfg = nginx.IngressNginxConfig{ }, Upstreams: []nginx.Upstream{testUps}, Keepalive: "16", + Ingress: nginx.Ingress{ + Name: "cafe-ingress", + Namespace: "default", + }, } var mainCfg = nginx.MainConfig{