Add VirtualServer/VirtualServerRoute snippets support

cmd/nginx-ingress/main.go

@@ -132,6 +132,9 @@ var (
 	enableCustomResources = flag.Bool("enable-custom-resources", true,
 		"Enable custom resources")
 
+	enableSnippets = flag.Bool("enable-snippets", false,
+		"Enable custom NGINX configuration snippets in VirtualServer and VirtualServerRoute resources.")
+
 	globalConfiguration = flag.String("global-configuration", "",
 		`A GlobalConfiguration resource for global configuration of the Ingress Controller. Requires -enable-custom-resources. If the flag is set,
 		but the Ingress controller is not able to fetch the corresponding resource from Kubernetes API, the Ingress Controller 
@@ -392,6 +395,7 @@ func main() {
 		NginxStatusPort:                *nginxStatusPort,
 		StubStatusOverUnixSocketForOSS: *enablePrometheusMetrics,
 		TLSPassthrough:                 *enableTLSPassthrough,
+		EnableSnippets:                 *enableSnippets,
 		SpiffeCerts:                    *spireAgentAddress != "",
 	}
 

deployments/common/vs-definition.yaml

@@ -59,6 +59,8 @@ spec:
           properties:
             host:
               type: string
+            http-snippets:
+              type: string
             ingressClassName:
               type: string
             routes:
@@ -186,6 +188,8 @@ spec:
                                     type: string
                             type:
                               type: string
+                  location-snippets:
+                    type: string
                   matches:
                     type: array
                     items:
@@ -474,6 +478,8 @@ spec:
                                   type: string
                         weight:
                           type: integer
+            server-snippets:
+              type: string
             tls:
               description: TLS defines TLS configuration for a VirtualServer.
               type: object

deployments/common/vsr-definition.yaml

@@ -184,6 +184,8 @@ spec:
                                     type: string
                             type:
                               type: string
+                  location-snippets:
+                    type: string
                   matches:
                     type: array
                     items:

deployments/helm-chart/README.md

@@ -210,6 +210,7 @@ Parameter | Description | Default
 `controller.watchNamespace` | Namespace to watch for Ingress resources. By default the Ingress controller watches all namespaces. | ""
 `controller.enableCustomResources` | Enable the custom resources. | true
 `controller.enableTLSPassthrough` | Enable TLS Passthrough on port 443. Requires `controller.enableCustomResources`. | false 
+`controller.enableSnippets` | Enable custom NGINX configuration snippets in VirtualServer and VirtualServerRoute resources. | false
 `controller.healthStatus` | Add a location "/nginx-health" to the default server. The location responds with the 200 status code for any request. Useful for external health-checking of the Ingress controller. | false
 `controller.healthStatusURI` | Sets the URI of health status location in the default server. Requires `contoller.healthStatus`. | "/nginx-health"
 `controller.nginxStatus.enable` | Enable the NGINX stub_status, or the NGINX Plus API. | true

deployments/helm-chart/crds/virtualserver.yaml

@@ -61,10 +61,12 @@ spec:
           description: VirtualServerSpec is the spec of the VirtualServer resource.
           type: object
           properties:
-            ingressClassName:
-              type: string
             host:
               type: string
+            http-snippets:
+              type: string
+            ingressClassName:
+              type: string
             routes:
               type: array
               items:
@@ -190,6 +192,8 @@ spec:
                                     type: string
                             type:
                               type: string
+                  location-snippets:
+                    type: string
                   matches:
                     type: array
                     items:
@@ -478,6 +482,8 @@ spec:
                                   type: string
                         weight:
                           type: integer
+            server-snippets:
+              type: string
             tls:
               description: TLS defines TLS configuration for a VirtualServer.
               type: object

deployments/helm-chart/crds/virtualserverroute.yaml

@@ -188,6 +188,8 @@ spec:
                                     type: string
                             type:
                               type: string
+                  location-snippets:
+                    type: string
                   matches:
                     type: array
                     items:

deployments/helm-chart/templates/controller-daemonset.yaml

@@ -130,5 +130,6 @@ spec:
           - -enable-custom-resources={{ .Values.controller.enableCustomResources }}
 {{- if .Values.controller.enableCustomResources  }}
           - -enable-tls-passthrough={{ .Values.controller.enableTLSPassthrough }}
+          - -enable-snippets={{ .Values.controller.enableSnippets }}
 {{- end }}
 {{- end }}

deployments/helm-chart/templates/controller-deployment.yaml

@@ -128,5 +128,6 @@ spec:
           - -enable-custom-resources={{ .Values.controller.enableCustomResources }}
 {{- if .Values.controller.enableCustomResources  }}
           - -enable-tls-passthrough={{ .Values.controller.enableTLSPassthrough }}
+          - -enable-snippets={{ .Values.controller.enableSnippets }}
 {{- end }}
 {{- end }}

deployments/helm-chart/values.yaml

@@ -124,6 +124,9 @@ controller:
   ## Enable TLS Passthrough on port 443. Requires controller.enableCustomResources.
   enableTLSPassthrough: false
 
+  ## Enable custom NGINX configuration snippets in VirtualServer and VirtualServerRoute resources.
+  enableSnippets: false
+
   ## Add a location based on the value of health-status-uri to the default server. The location responds with the 200 status code for any request.
   ## Useful for external health-checking of the Ingress controller.
   healthStatus: false

docs-web/configuration/global-configuration/command-line-arguments.md

@@ -6,6 +6,10 @@ The Ingress Controller supports several command-line arguments. Setting the argu
 
 Below we describe the available command-line arguments:
 ```eval_rst
+.. option:: -enable-snippets
+
+	Enable custom NGINX configuration snippets in VirtualServer and VirtualServerRoute resources. (default false)
+
 .. option:: -default-server-tls-secret <string>
 
 	Secret with a TLS certificate and key for TLS termination of the default server.
@@ -27,11 +31,11 @@ Below we describe the available command-line arguments:
 
 .. option:: -enable-custom-resources
 
-	Enables custom resources (default true)
+	Enables custom resources. (default true)
 
 .. option:: -enable-leader-election
 
-	Enables Leader election to avoid multiple replicas of the controller reporting the status of Ingress, VirtualServer and VirtualServerRoute resources -- only one replica will report status (default true).
+	Enables Leader election to avoid multiple replicas of the controller reporting the status of Ingress, VirtualServer and VirtualServerRoute resources -- only one replica will report status. (default true)
 
 	See :option:`-report-ingress-status` flag.
 
@@ -161,13 +165,13 @@ Below we describe the available command-line arguments:
 
 	Enables exposing NGINX or NGINX Plus metrics in the Prometheus format.
 
-.. option:: -prometheus-metrics-listen-port
+.. option:: -prometheus-metrics-listen-port <int>
 
 	Sets the port where the Prometheus metrics are exposed.
 
 	Format: ``[1023 - 65535]`` (default 9113)
 
-.. option:: -spire-agent-address
+.. option:: -spire-agent-address <string>
 
 	Specifies the address of a running Spire agent. **For use with NGINX Service Mesh only**.
 

docs-web/configuration/virtualserver-and-virtualserverroute-resources.md

@@ -33,6 +33,7 @@ This document is the reference documentation for the resources. To see additiona
     - [ErrorPage.Redirect](#errorpage-redirect)
     - [ErrorPage.Return](#errorpage-return)
   - [Using VirtualServer and VirtualServerRoute](#using-virtualserver-and-virtualserverroute)
+    - [Using Snippets](#using-snippets)
     - [Validation](#validation)
       - [Structural Validation](#structural-validation)
       - [Comprehensive Validation](#comprehensive-validation)
@@ -100,6 +101,14 @@ spec:
      - Specifies which Ingress controller must handle the VirtualServer resource.
      - ``string``
      - No
+   * - ``http-snippets``
+     - Sets a custom snippet in the http context.
+     - ``string``
+     - No
+   * - ``server-snippets``
+     - Sets a custom snippet in server context. Overrides the ``server-snippets`` ConfigMap key.
+     - ``string``
+     - No
 ```
 
 ### VirtualServer.TLS
@@ -200,6 +209,10 @@ The route defines rules for matching client requests to actions like passing a r
      - The custom responses for error codes. NGINX will use those responses instead of returning the error responses from the upstream servers or the default responses generated by NGINX. A custom response can be a redirect or a canned response. For example, a redirect to another URL if an upstream server responded with a 404 status code.
      - `[]errorPage <#errorpage>`_
      - No
+   * - ``location-snippets``
+     - Sets a custom snippet in the location context. Overrides the ``location-snippets`` ConfigMap key.
+     - ``string``
+     - No
 ```
 
 \* -- a route must include exactly one of the following: `action`, `splits`, or `route`.
@@ -321,6 +334,10 @@ action:
      - The custom responses for error codes. NGINX will use those responses instead of returning the error responses from the upstream servers or the default responses generated by NGINX. A custom response can be a redirect or a canned response. For example, a redirect to another URL if an upstream server responded with a 404 status code.
      - `[]errorPage <#errorpage>`_
      - No
+   * - ``location-snippets``
+     - Sets a custom snippet in the location context. Overrides the ``location-snippets`` of the VirtualServer (if set) or the ``location-snippets`` ConfigMap key.
+     - ``string``
+     - No
 ```
 
 \* -- a subroute must include exactly one of the following: `action` or `splits`.
@@ -1263,6 +1280,59 @@ In the kubectl get and similar commands, you can also use the short name `vs` in
 
 Working with VirtualServerRoute resources is analogous. In the kubectl commands, use `virtualserverroute` or the short name `vsr`.
 
+### Using Snippets
+
+Snippets allow you to insert raw NGINX config into different contexts of NGINX configuration. In the example below, we use snippets to configure several NGINX features in a VirtualServer:
+
+```yaml
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServer
+metadata:
+  name: cafe
+  namespace: cafe
+spec:
+  http-snippets: |
+    limit_req_zone $binary_remote_addr zone=mylimit:10m rate=1r/s;
+    proxy_cache_path /tmp keys_zone=one:10m;
+  host: cafe.example.com
+  tls:
+    secret: cafe-secret
+  server-snippets: |
+    limit_req zone=mylimit burst=20;
+  upstreams:
+  - name: tea
+    service: tea-svc
+    port: 80
+  - name: coffee
+    service: coffee-svc
+    port: 80
+  routes:
+  - path: /tea
+    location-snippets: |
+      proxy_cache one;
+      proxy_cache_valid 200 10m;
+    action:
+      pass: tea
+  - path: /coffee
+    action:
+      pass: coffee
+```
+
+Snippets are intended to be used by advanced NGINX users who need more control over the generated NGINX configuration.
+
+However, because of the disadvantages described below, snippets are disabled by default. To use snippets, set the [`enable-snippets`](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments#cmdoption-enable-snippets) command-line argument.
+
+Disadvantages of using snippets:
+* *Complexity*. To use snippets, you will need to:
+  * Understand NGINX configuration primitives and implement a correct NGINX configuration.
+  * Understand how the IC generates NGINX configuration so that a snippet doesn't interfere with the other features in the configuration.
+* *Decreased robustness*. An incorrect snippet makes the NGINX config invalid which will lead to a failed reload. This will prevent any new configuration updates, including updates for the other VirtualServer and VirtualServerRoute resources until the snippet is fixed.
+* *Security implications*. Snippets give access to NGINX configuration primitives and those primitives are not validated by the Ingress Controller. For example, a snippet can configure NGINX to serve the TLS certificates and keys used for TLS termination for Ingress and VirtualServer resources.
+
+To help catch errors when using snippets, the Ingress Controller reports config reload errors in the logs as well as in the events and status field of VirtualServer and VirtualServerRoute resources. Additionally, a number of Prometheus metrics show the stats about failed reloads – `controller_nginx_last_reload_status` and `controller_nginx_reload_errors_total`.
+
+> Note that during a period when the NGINX config includes an invalid snippet, NGINX will continue to operate with the latest valid configuration.
+
 ### Validation
 
 Two types of validation are available for VirtualServer and VirtualServerRoute resources:

docs-web/installation/installation-with-helm.md

@@ -273,6 +273,9 @@ The following tables lists the configurable parameters of the NGINX Ingress cont
    * - ``controller.enableTLSPassthrough``
      - Enable TLS Passthrough on port 443. Requires ``controller.enableCustomResources``.
      - false
+   * - ``controller.enableSnippets``
+     - Enable custom NGINX configuration snippets in VirtualServer and VirtualServerRoute resources.
+     - false
    * - ``controller.healthStatus``
      - Add a location "/nginx-health" to the default server. The location responds with the 200 status code for any request. Useful for external health-checking of the Ingress controller.
      - false

internal/configs/config_params.go

@@ -99,6 +99,7 @@ type StaticConfigParams struct {
 	NginxStatusPort                int
 	StubStatusOverUnixSocketForOSS bool
 	TLSPassthrough                 bool
+	EnableSnippets                 bool
 	SpiffeCerts                    bool
 }
 

internal/configs/version2/http.go

@@ -7,6 +7,7 @@ type VirtualServerConfig struct {
 	SplitClients  []SplitClient
 	Maps          []Map
 	StatusMatches []StatusMatch
+	HTTPSnippets  []string
 	SpiffeCerts   bool
 }
 

internal/configs/version2/nginx-plus.virtualserver.tmpl

@@ -40,6 +40,10 @@ map {{ $m.Source }} {{ $m.Variable }} {
 }
 {{ end }}
 
+{{ range $snippet := .HTTPSnippets }}
+{{ $snippet }}
+{{ end }}
+
 {{ range $m := .StatusMatches }}
 match {{ $m.Name }} {
     status {{ $m.Code }};

internal/configs/version2/nginx.virtualserver.tmpl

@@ -30,6 +30,10 @@ map {{ $m.Source }} {{ $m.Variable }} {
 }
 {{ end }}
 
+{{ range $snippet := .HTTPSnippets }}
+{{ $snippet }}
+{{ end }}
+
 {{ $s := .Server }}
 server {
     listen 80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};
@@ -137,7 +141,7 @@ server {
         proxy_buffer_size {{ $l.ProxyBufferSize }};
             {{ end }}
             {{ if $l.ProxyInterceptErrors }}
-                proxy_intercept_errors on;
+        proxy_intercept_errors on;
             {{ end }}
         proxy_http_version 1.1;
 

internal/configs/version2/templates_test.go

@@ -99,6 +99,7 @@ var virtualServerCfg = VirtualServerConfig{
 			},
 		},
 	},
+	HTTPSnippets: []string{"# HTTP snippet"},
 	Server: Server{
 		ServerName:    "example.com",
 		StatusZone:    "example.com",