Add support for IngressClass resources

Co-authored-by: Dean Coakley <[email protected]>
Co-authored-by: Michael Pleshakov <[email protected]>

cmd/nginx-ingress/main.go

@@ -30,6 +30,7 @@ import (
 	api_v1 "k8s.io/api/core/v1"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/validation"
+	util_version "k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -68,13 +69,23 @@ var (
 	appProtect = flag.Bool("enable-app-protect", false, "Enable support for NGINX App Protect. Requires -nginx-plus.")
 
 	ingressClass = flag.String("ingress-class", "nginx",
-		`A class of the Ingress controller. The Ingress controller only processes Ingress resources that belong to its class
-	- i.e. have the annotation "kubernetes.io/ingress.class" or the "ingressClassName" field in VirtualServer/VirtualServerRoute equal to the class. Additionally,
-	the Ingress controller processes Ingress resources that do not have that annotation,
-	which can be disabled by setting the "-use-ingress-class-only" flag`)
+		`A class of the Ingress controller.
+
+	For Kubernetes >= 1.18, a corresponding IngressClass resource with the name equal to the class must be deployed. Otherwise,
+	the Ingress Controller will fail to start.
+	The Ingress controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class.
+
+	For Kubernetes < 1.18, the Ingress Controller only processes resources that belong to its class
+	- i.e have the annotation "kubernetes.io/ingress.class" equal to the class.
+	Additionally, the Ingress Controller processes resources that do not have the class set,
+	which can be disabled by setting the "-use-ingress-class-only" flag
+	
+	The Ingress Controller processes all the VirtualServer/VirtualServerRoute resources that do not have the "ingressClassName" field for all versions of kubernetes.`)
 
 	useIngressClassOnly = flag.Bool("use-ingress-class-only", false,
-		`Ignore Ingress resources without the "kubernetes.io/ingress.class" annotation or the "ingressClassName" field in VirtualServer/VirtualServerRoute`)
+		`For kubernetes versions >= 1.18 this flag will be IGNORED.
+
+	Ignore Ingress resources without the "kubernetes.io/ingress.class" annotation`)
 
 	defaultServerSecret = flag.String("default-server-tls-secret", "",
 		`A Secret with a TLS certificate and key for TLS termination of the default server. Format: <namespace>/<name>.
@@ -256,6 +267,32 @@ func main() {
 		glog.Fatalf("Failed to create client: %v.", err)
 	}
 
+	k8sVersion, err := k8s.GetK8sVersion(kubeClient)
+	if err != nil {
+		glog.Fatalf("error retrieving k8s version: %v", err)
+	}
+
+	minK8sVersion := minVersion("1.14.0")
+	if !k8sVersion.AtLeast(minK8sVersion) {
+		glog.Fatalf("Versions of Kubernetes < %v are not supported, please refer to the documentation for details on supported versions.", minK8sVersion)
+	}
+
+	// Ingress V1 is only available from k8s > 1.18
+	ingressV1Version := minVersion("1.18.0")
+	if k8sVersion.AtLeast(ingressV1Version) {
+		*useIngressClassOnly = true
+		glog.Warningln("The '-use-ingress-class-only' flag will be deprecated and has no effect on versions of kubernetes >= 1.18.0. Processing ONLY resources that have the 'ingressClassName' field in Ingress equal to the class.")
+
+		ingressClassRes, err := kubeClient.NetworkingV1beta1().IngressClasses().Get(context.TODO(), *ingressClass, meta_v1.GetOptions{})
+		if err != nil {
+			glog.Fatalf("Error when getting IngressClass %v: %v", *ingressClass, err)
+		}
+
+		if ingressClassRes.Spec.Controller != k8s.IngressControllerName {
+			glog.Fatalf("IngressClass with name %v has an invalid Spec.Controller %v", ingressClassRes.Name, ingressClassRes.Spec.Controller)
+		}
+	}
+
 	var dynClient dynamic.Interface
 	if *appProtect {
 		dynClient, err = dynamic.NewForConfig(config)
@@ -789,3 +826,12 @@ func ready(lbc *k8s.LoadBalancerController) http.HandlerFunc {
 		fmt.Fprintln(w, "Ready")
 	}
 }
+
+func minVersion(min string) (v *util_version.Version) {
+	minVer, err := util_version.ParseGeneric(min)
+	if err != nil {
+		glog.Fatalf("unexpected error parsing minimum supported version: %v", err)
+	}
+
+	return minVer
+}

deployments/common/ingress-class.yaml

@@ -0,0 +1,8 @@
+apiVersion: networking.k8s.io/v1beta1
+kind: IngressClass
+metadata:
+  name: nginx
+  # annotations:
+  #   ingressclass.kubernetes.io/is-default-class: "true"
+spec:
+  controller: nginx.org/ingress-controller 

deployments/helm-chart/Chart.yaml

@@ -2,6 +2,7 @@ name: nginx-ingress
 version: edge
 appVersion: edge
 apiVersion: v1
+kubeVersion: ">= 1.14.0"
 description: NGINX Ingress Controller
 icon: https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/master/deployments/helm-chart/chart-icon.png
 sources:

deployments/helm-chart/README.md

@@ -214,8 +214,9 @@ Parameter | Description | Default
 `controller.volumeMounts` | The volumeMounts of the Ingress controller pods. | []
 `controller.resources` | The resources of the Ingress controller pods. | {}
 `controller.replicaCount` | The number of replicas of the Ingress controller deployment. | 1
-`controller.ingressClass` | A class of the Ingress controller. The Ingress controller only processes Ingress resources that belong to its class - i.e. have the annotation `"kubernetes.io/ingress.class"` or the `"ingressClassName"` field in VirtualServer/VirtualServerRoute equal to the class. Additionally, the Ingress controller processes Ingress resources that do not have that annotation which can be disabled by setting the "-use-ingress-class-only" flag. | nginx
-`controller.useIngressClassOnly` | Ignore Ingress resources without the `"kubernetes.io/ingress.class"` annotation or the `"ingressClassName"` field in VirtualServer/VirtualServerRoute. | false
+`controller.ingressClass` | A class of the Ingress controller. For Kubernetes >= 1.18, the Ingress controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class. Additionally the Ingress Controller processes all the VirtualServer/VirtualServerRoute resources that do not have the "ingressClassName" field. For Kubernetes < 1.18, the Ingress Controller only processes resources that belong to its class - i.e have the annotation "kubernetes.io/ingress.class" (for Ingress resources) or field "ingressClassName" (for VirtualServer/VirtualServerRoute resources) equal to the class. Additionally, the Ingress Controller processes resources that do not have the class set, which can be disabled by setting the "-use-ingress-class-only" flag. | nginx
+`controller.useIngressClassOnly` | Ignore Ingress resources without the `"kubernetes.io/ingress.class"` annotation. For kubernetes versions >= 1.18 this flag will be IGNORED. | false
+`controller.setAsDefaultIngress` | New Ingresses without an `"ingressClassName"` field specified will be assigned the class specified in `controller.ingressClass`. Only for kubernetes versions >= 1.18. | false
 `controller.watchNamespace` | Namespace to watch for Ingress resources. By default the Ingress controller watches all namespaces. | ""
 `controller.enableCustomResources` | Enable the custom resources. | true
 `controller.enableTLSPassthrough` | Enable TLS Passthrough on port 443. Requires `controller.enableCustomResources`. | false

deployments/helm-chart/templates/controller-daemonset.yaml

@@ -108,7 +108,9 @@ spec:
           - -default-server-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.defaultTLSName" . }}
 {{- end }}
           - -ingress-class={{ .Values.controller.ingressClass }}
+{{- if semverCompare "<1.18.0" .Capabilities.KubeVersion.GitVersion }}
           - -use-ingress-class-only={{ .Values.controller.useIngressClassOnly }}
+{{- end }}
 {{- if .Values.controller.watchNamespace }}
           - -watch-namespace={{ .Values.controller.watchNamespace }}
 {{- end }}

deployments/helm-chart/templates/controller-deployment.yaml

@@ -106,7 +106,9 @@ spec:
           - -default-server-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.defaultTLSName" . }}
 {{- end }}
           - -ingress-class={{ .Values.controller.ingressClass }}
+{{- if semverCompare "<1.18.0" .Capabilities.KubeVersion.GitVersion }}
           - -use-ingress-class-only={{ .Values.controller.useIngressClassOnly }}
+{{- end }}
 {{- if .Values.controller.watchNamespace }}
           - -watch-namespace={{ .Values.controller.watchNamespace }}
 {{- end }}

deployments/helm-chart/templates/controller-ingress-class..yaml

@@ -0,0 +1,12 @@
+{{- if semverCompare ">=1.18.0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1beta1
+kind: IngressClass
+metadata:
+  name: {{ .Values.controller.ingressClass }}
+{{- if .Values.controller.setAsDefaultIngress }}
+  annotations:
+    ingressclass.kubernetes.io/is-default-class: "true"
+{{- end }}
+spec:
+  controller: nginx.org/ingress-controller
+{{- end }}

deployments/helm-chart/templates/rbac.yaml

@@ -69,6 +69,12 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingressclasses
+  verbs:
+  - get
 {{- if .Values.controller.reportIngressStatus.enable }}
 - apiGroups:
   - networking.k8s.io

deployments/helm-chart/values.yaml

@@ -117,14 +117,27 @@ controller:
   ## The number of replicas of the Ingress controller deployment.
   replicaCount: 1
 
-  ## A class of the Ingress controller. The Ingress controller only processes Ingress resources that belong to its class
-  ## i.e. have the annotation "kubernetes.io/ingress.class" or the "ingressClassName" field in VirtualServer/VirtualServerRoute equal to the class.
-  ## Additionally, the Ingress controller processes Ingress resources that do not have that annotation which can be disabled by setting the "-use-ingress-class-only" flag.
+  ## A class of the Ingress controller.
+
+  ## For Kubernetes >= 1.18, a corresponding IngressClass resource with the name equal to the class must be deployed. Otherwise,
+  ## the Ingress Controller will fail to start.
+  ## The Ingress controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class.
+
+  ## For Kubernetes < 1.18, the Ingress Controller only processes resources that belong to its class
+  ## - i.e have the annotation "kubernetes.io/ingress.class" equal to the class.
+  ## Additionally, the Ingress Controller processes resources that do not have the class set, which can be disabled by setting the "-use-ingress-class-only" flag
+
+  ## The Ingress Controller processes all the VirtualServer/VirtualServerRoute resources that do not have the "ingressClassName" field for all versions of kubernetes.
   ingressClass: nginx
 
-  ## Ignore Ingress resources without the "kubernetes.io/ingress.class" annotation or the "ingressClassName" field in VirtualServer/VirtualServerRoute.
+  ## For kubernetes versions >= 1.18 this flag will be IGNORED.
+  ## Ignore Ingress resources without the "kubernetes.io/ingress.class" annotation
   useIngressClassOnly: false
 
+  ## Only for Kubernetes >= 1.18
+  ## New Ingresses without an ingressClassName field specified will be assigned the class specified in `controller.ingressClass`.
+  setAsDefaultIngress: false
+
   ## Namespace to watch for Ingress resources. By default the Ingress controller watches all namespaces.
   watchNamespace: ""
 

deployments/rbac/rbac.yaml

@@ -78,6 +78,12 @@ rules:
   - virtualserverroutes/status
   verbs:
   - update
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingressclasses
+  verbs:
+  - get
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

docs-web/configuration/global-configuration/command-line-arguments.md

@@ -66,8 +66,17 @@ Below we describe the available command-line arguments:
 
 .. option:: -ingress-class <string>
 
-	A class of the Ingress controller. The Ingress controller only processes Ingress resources that belong to its class (i.e. have the annotation "kubernetes.io/ingress.class" or the "ingressClassName" field in VirtualServer/VirtualServerRoute").
-	Additionally, the Ingress controller processes Ingress resources that do not have that annotation, which can be disabled by setting the :option:`-use-ingress-class-only` flag (default "nginx").
+	A class of the Ingress controller.
+
+	For Kubernetes >= 1.18, a corresponding IngressClass resource with the name equal to the class must be deployed. Otherwise, the Ingress Controller will fail to start.
+	The Ingress controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class.
+
+	For Kubernetes < 1.18, the Ingress Controller only processes resources that belong to its class - i.e have the annotation "kubernetes.io/ingress.class" (for Ingress resources) or field "ingressClassName" (for VirtualServer/VirtualServerRoute resources) equal to the class.
+	Additionally, the Ingress Controller processes resources that do not have the class set, which can be disabled by setting the "-use-ingress-class-only" flag.
+
+	The Ingress Controller processes all the VirtualServer/VirtualServerRoute resources that do not have the "ingressClassName" field.
+
+	(default "nginx")
 
 .. option:: -ingress-template-path <string>
 
@@ -140,7 +149,9 @@ Below we describe the available command-line arguments:
 
 .. option:: -use-ingress-class-only
 
-	Ignore Ingress resources without the "kubernetes.io/ingress.class" annotation or the "ingressClassName" field in VirtualServer/VirtualServerRoute.
+	For kubernetes versions >= 1.18 this flag will be IGNORED.
+
+	Ignore Ingress resources without the "kubernetes.io/ingress.class" annotation.
 
 .. option:: -v <value>
 

docs-web/configuration/ingress-resources/basic-configuration.md

@@ -63,6 +63,23 @@ Starting from Kubernetes 1.18, you can use the following new features:
         serviceName: coffee-svc
         servicePort: 80
   ```
+* The `ingressClassName` field is now supported:
+  ```yaml
+    apiVersion: networking.k8s.io/v1beta1
+    kind: Ingress
+    metadata:
+      name: cafe-ingress
+    spec:
+      ingressClassName: nginx
+      tls:
+      - hosts:
+        - cafe.example.com
+        secretName: cafe-secret
+      rules:
+      - host: cafe.example.com
+    . . .
+  ```
+  When using this filed you need to create the `IngressClass` resource with the corresponding `name`. See Step 3 *Create an IngressClass resource* of the [Create Common Resources](/nginx-ingress-controller/installation/installation-with-manifests/#create-common-resources) section.
 
 ## Restrictions
 

docs-web/installation/installation-with-helm.md

@@ -274,10 +274,13 @@ The following tables lists the configurable parameters of the NGINX Ingress cont
      - The number of replicas of the Ingress controller deployment.
      - 1
    * - ``controller.ingressClass``
-     - A class of the Ingress controller. The Ingress controller only processes Ingress resources that belong to its class - i.e. have the annotation ``"kubernetes.io/ingress.class"``  or the ``"ingressClassName"`` field in VirtualServer/VirtualServerRoute equal to the class. Additionally, the Ingress controller processes Ingress resources that do not have that annotation which can be disabled by setting the "-use-ingress-class-only" flag.
+     - A class of the Ingress controller. For Kubernetes >= 1.18, the Ingress controller only processes resources that belong to its class - i.e. have the ``"ingressClassName"`` field resource equal to the class. For Kubernetes < 1.18, the Ingress Controller only processes resources that belong to its class - i.e have the annotation ``"kubernetes.io/ingress.class"`` equal to the class. Additionally, the Ingress Controller processes resources that do not have the class set, which can be disabled by setting the ``controller.useIngressClassOnly`` flag. The Ingress Controller processes all the VirtualServer/VirtualServerRoute resources that do not have the ``"ingressClassName"`` field for all versions of kubernetes.
      - nginx
    * - ``controller.useIngressClassOnly``
-     - Ignore Ingress resources without the ``"kubernetes.io/ingress.class"`` annotation or the ``"ingressClassName"`` field in VirtualServer/VirtualServerRoute.
+     - Ignore Ingress resources without the ``"kubernetes.io/ingress.class"`` annotation. For kubernetes versions >= 1.18 this flag will be IGNORED.
+     - false
+   * - ``controller.setAsDefaultIngress``
+     - New Ingresses without an ingressClassName field specified will be assigned the class specified in `controller.ingressClass`.
      - false
    * - ``controller.watchNamespace``
      - Namespace to watch for Ingress resources. By default the Ingress controller watches all namespaces.

docs-web/installation/installation-with-manifests.md

@@ -47,6 +47,14 @@ In this section, we create resources common for most of the Ingress Controller i
     $ kubectl apply -f common/nginx-config.yaml
     ```
 
+1. Create an IngressClass resource (for Kubernetes >= 1.18):
+    ```
+    $ kubectl apply -f common/ingress-class.yaml
+    ```
+    If you would like to set the Ingress Controller as the default one, uncomment the annotation `ingressclass.kubernetes.io/is-default-class`. With this annotation set to true all the new Ingresses without an ingressClassName field specified will be assigned this IngressClass.
+
+    **Note**: The Ingress Controller will fail to start without an IngressClass resource.
+
 ### Create Custom Resources
 
 **Note**: If you're using Kubernetes 1.14, make sure to add `--validate=false` to the `kubectl apply` commands below. Otherwise, you will get an error validating data:

docs-web/installation/running-multiple-ingress-controllers.md

@@ -11,14 +11,18 @@ This document explains the following topics:
 
 The smooth coexistence of multiple Ingress Controllers in one cluster is provided by the Ingress class concept, which mandates the following:
 * Every Ingress Controller must only handle Ingress resources for its particular class. 
-* Ingress resources should be annotated with the `kubernetes.io/ingress.class` annotation set to the value, which corresponds to the class of the Ingress Controller the user wants to use. 
+* For Kubernetes < 1.18, Ingress resources should be annotated with the `kubernetes.io/ingress.class` annotation set to the value, which corresponds to the class of the Ingress Controller the user wants to use. 
+* When using versions of Kubernetes >= 1.18, Ingress resources should have the `ingressClassName` field set to the value, which corresponds to the class of the Ingress Controller the user wants to use. 
 * VirtualServer and VirtualServerRoute resources should have the `ingressClassName` field set to the value, which corresponds to the class of the Ingress Controller the user wants to use.
 
 ### Configuring Ingress Class
 
 The default Ingress class of NGINX Ingress Controller is `nginx`, which means that it only handles configuration resources with the `class` set to `nginx`. You can customize the class through the `-ingress-class` command-line argument.
 
-**Note**: By default, if the `class` is not set in a configuration resource, the Ingress Controller will handle the resource. This is controlled via the `-use-ingress-class-only` argument.
+**Notes**: 
+* For Kubernetes < 1.18, if the class is not set in an Ingress configuration resource, the Ingress Controller will handle the resource. This is controlled via the `-use-ingress-class-only` argument. 
+* For Kubernetes >= 1.18, if the class is not set in an Ingress resource, Kubernetes will set it to the class of the default Ingress Controller. To make the Ingress Controller the default one, the `ingressclass.kubernetes.io/is-default-class` must be set on the IngressClass resource. See Step 3 *Create an IngressClass resource* of the [Create Common Resources](/nginx-ingress-controller/installation/installation-with-manifests/#create-common-resources) section.
+* For VirtualServer and VirtualServerRoute resources, the Ingress Controller will always handle resources with an empty class.
 
 ## Running NGINX Ingress Controller and Another Ingress Controller
 

examples/complete-example/README.md

@@ -31,7 +31,7 @@ $ kubectl create -f cafe.yaml
     $ kubectl create -f cafe-secret.yaml
     ```
 
-2. Create an Ingress resource:
+2. Create an Ingress resource (for Kubernetes >= 1.18, uncomment the ```ingressClassName``` field in the YAML file):
     ```
     $ kubectl create -f cafe-ingress.yaml
     ```

examples/complete-example/cafe-ingress.yaml

@@ -3,6 +3,7 @@ kind: Ingress
 metadata:
   name: cafe-ingress
 spec:
+  # ingressClassName: nginx # use only with k8s version >= 1.18.0
   tls:
   - hosts:
     - cafe.example.com