feature: Support Dynamic namespaces using Labels (#3299)

* Add labels flag and create namespace watcher when present

* Create namespaced watchers dynamically for simple case

* Handle unwatched app protect resources

* Add dynamic watchers to externaldns and cert-manager controllers

* Add initial python tests

* Add app protect waf tests

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add python tests for cert-manager and external-dns

* Add helm config and docs

* Update logging for new and deleted watched namespaces

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

cmd/nginx-ingress/flags.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/golang/glog"
 	api_v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/validation"
 )
 
@@ -27,15 +28,18 @@ var (
 	The Ingress Controller does not start NGINX and does not write any generated NGINX configuration files to disk`)
 
 	watchNamespace = flag.String("watch-namespace", api_v1.NamespaceAll,
-		`Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces`)
+		`Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces. Mutually exclusive with "watch-namespace-label".`)
 
 	watchNamespaces []string
 
 	watchSecretNamespace = flag.String("watch-secret-namespace", "",
-		`Comma separated list of namespaces the Ingress Controller should watch for secrets. If this arg is not configured, the Ingress Controller watches the same namespaces for all resources. See "watch-namespace". `)
+		`Comma separated list of namespaces the Ingress Controller should watch for secrets. If this arg is not configured, the Ingress Controller watches the same namespaces for all resources. See "watch-namespace" and "watch-namespace-label". `)
 
 	watchSecretNamespaces []string
 
+	watchNamespaceLabel = flag.String("watch-namespace-label", "",
+		`Configures the Ingress Controller to watch only those namespaces with label foo=bar. By default the Ingress Controller watches all namespaces. Mutually exclusive with "watch-namespace". `)
+
 	nginxConfigMaps = flag.String("nginx-configmaps", "",
 		`A ConfigMap resource for customizing NGINX configuration. If a ConfigMap is set,
 	but the Ingress Controller is not able to fetch it from Kubernetes API, the Ingress Controller will fail to start.
@@ -192,17 +196,7 @@ func parseFlags() {
 
 	initialChecks()
 
-	watchNamespaces = strings.Split(*watchNamespace, ",")
-	glog.Infof("Namespaces watched: %v", watchNamespaces)
-
-	if len(*watchSecretNamespace) > 0 {
-		watchSecretNamespaces = strings.Split(*watchSecretNamespace, ",")
-	} else {
-		// empty => default to watched namespaces
-		watchSecretNamespaces = watchNamespaces
-	}
-
-	glog.Infof("Namespaces watched for secrets: %v", watchSecretNamespaces)
+	validateWatchedNamespaces()
 
 	validationChecks()
 
@@ -295,6 +289,42 @@ func initialChecks() {
 	}
 }
 
+func validateWatchedNamespaces() {
+	if *watchNamespace != "" && *watchNamespaceLabel != "" {
+		glog.Fatal("watch-namespace and -watch-namespace-label are mutually exclusive")
+	}
+
+	watchNamespaces = strings.Split(*watchNamespace, ",")
+
+	if *watchNamespace != "" {
+		glog.Infof("Namespaces watched: %v", watchNamespaces)
+		namespacesNameValidationError := validateNamespaceNames(watchNamespaces)
+		if namespacesNameValidationError != nil {
+			glog.Fatalf("Invalid values for namespaces: %v", namespacesNameValidationError)
+		}
+	}
+
+	if len(*watchSecretNamespace) > 0 {
+		watchSecretNamespaces = strings.Split(*watchSecretNamespace, ",")
+		glog.Infof("Namespaces watched for secrets: %v", watchSecretNamespaces)
+		namespacesNameValidationError := validateNamespaceNames(watchSecretNamespaces)
+		if namespacesNameValidationError != nil {
+			glog.Fatalf("Invalid values for secret namespaces: %v", namespacesNameValidationError)
+		}
+	} else {
+		// empty => default to watched namespaces
+		watchSecretNamespaces = watchNamespaces
+	}
+
+	if *watchNamespaceLabel != "" {
+		var err error
+		_, err = labels.Parse(*watchNamespaceLabel)
+		if err != nil {
+			glog.Fatalf("Unable to parse label %v for watch namespace label: %v", *watchNamespaceLabel, err)
+		}
+	}
+}
+
 // validationChecks checks the values for various flags
 func validationChecks() {
 	healthStatusURIValidationError := validateLocation(*healthStatusURI)
@@ -307,16 +337,6 @@ func validationChecks() {
 		glog.Fatalf("Invalid value for leader-election-lock-name: %v", statusLockNameValidationError)
 	}
 
-	namespacesNameValidationError := validateNamespaceNames(watchNamespaces)
-	if namespacesNameValidationError != nil {
-		glog.Fatalf("Invalid values for namespaces: %v", namespacesNameValidationError)
-	}
-
-	namespacesNameValidationError = validateNamespaceNames(watchSecretNamespaces)
-	if namespacesNameValidationError != nil {
-		glog.Fatalf("Invalid values for secret namespaces: %v", namespacesNameValidationError)
-	}
-
 	statusPortValidationError := validatePort(*nginxStatusPort)
 	if statusPortValidationError != nil {
 		glog.Fatalf("Invalid value for nginx-status-port: %v", statusPortValidationError)

cmd/nginx-ingress/main.go

@@ -53,9 +53,7 @@ func main() {
 
 	validateIngressClass(kubeClient)
 
-	checkNamespaceExists(kubeClient, watchNamespaces)
-
-	checkNamespaceExists(kubeClient, watchSecretNamespaces)
+	checkNamespaces(kubeClient)
 
 	dynClient, confClient := createCustomClients(config)
 
@@ -160,6 +158,7 @@ func main() {
 		CertManagerEnabled:           *enableCertManager,
 		ExternalDNSEnabled:           *enableExternalDNS,
 		IsIPV6Disabled:               *disableIPV6,
+		WatchNamespaceLabel:          *watchNamespaceLabel,
 	}
 
 	lbc := k8s.NewLoadBalancerController(lbcInput)
@@ -243,6 +242,25 @@ func validateIngressClass(kubeClient kubernetes.Interface) {
 	}
 }
 
+func checkNamespaces(kubeClient kubernetes.Interface) {
+	if *watchNamespaceLabel != "" {
+		// bootstrap the watched namespace list
+		var newWatchNamespaces []string
+		nsList, err := kubeClient.CoreV1().Namespaces().List(context.TODO(), meta_v1.ListOptions{LabelSelector: *watchNamespaceLabel})
+		if err != nil {
+			glog.Errorf("error when getting Namespaces with the label selector %v: %v", watchNamespaceLabel, err)
+		}
+		for _, ns := range nsList.Items {
+			newWatchNamespaces = append(newWatchNamespaces, ns.Name)
+		}
+		watchNamespaces = newWatchNamespaces
+		glog.Infof("Namespaces watched using label %v: %v", *watchNamespaceLabel, watchNamespaces)
+	} else {
+		checkNamespaceExists(kubeClient, watchNamespaces)
+	}
+	checkNamespaceExists(kubeClient, watchSecretNamespaces)
+}
+
 func checkNamespaceExists(kubeClient kubernetes.Interface, namespaces []string) {
 	for _, ns := range namespaces {
 		if ns != "" {

deployments/helm-chart/README.md

@@ -186,8 +186,9 @@ Parameter | Description | Default
 `controller.replicaCount` | The number of replicas of the Ingress Controller deployment. | 1
 `controller.ingressClass` | A class of the Ingress Controller. An IngressClass resource with the name equal to the class must be deployed. Otherwise, the Ingress Controller will fail to start. The Ingress Controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class. The Ingress Controller processes all the VirtualServer/VirtualServerRoute/TransportServer resources that do not have the "ingressClassName" field for all versions of kubernetes. | nginx
 `controller.setAsDefaultIngress` | New Ingresses without an `"ingressClassName"` field specified will be assigned the class specified in `controller.ingressClass`. | false
-`controller.watchNamespace` | Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces. Please note that if configuring multiple namespaces using the Helm cli `--set` option, the string needs to wrapped in double quotes and the commas escaped using a backslash - e.g. `--set controller.watchNamespace="default\,nginx-ingress"`. | ""
-`controller.watchSecretNamespace` | Comma separated list of namespaces the Ingress Controller should watch for resources of type Secret. If this arg is not configured, the Ingress Controller watches the same namespaces for all resources. See `watch-namespace`. Please note that if configuring multiple namespaces using the Helm cli `--set` option, the string needs to wrapped in double quotes and the commas escaped using a backslash - e.g. `--set controller.watchSecretNamespace="default\,nginx-ingress"`. | ""
+`controller.watchNamespace` | Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces. Mutually exclusive with `controller.watchNamespaceLabel`. Please note that if configuring multiple namespaces using the Helm cli `--set` option, the string needs to wrapped in double quotes and the commas escaped using a backslash - e.g. `--set controller.watchNamespace="default\,nginx-ingress"`. | ""
+`controller.watchNamespaceLabel` | Configures the Ingress Controller to watch only those namespaces with label foo=bar. By default the Ingress Controller watches all namespaces. Mutually exclusive with `controller.watchNamespace`. | ""
+`controller.watchSecretNamespace` | Comma separated list of namespaces the Ingress Controller should watch for resources of type Secret. If this arg is not configured, the Ingress Controller watches the same namespaces for all resources. See `controller.watchNamespace` and `controller.watchNamespaceLabel`. Please note that if configuring multiple namespaces using the Helm cli `--set` option, the string needs to wrapped in double quotes and the commas escaped using a backslash - e.g. `--set controller.watchSecretNamespace="default\,nginx-ingress"`. | ""
 `controller.enableCustomResources` | Enable the custom resources. | true
 `controller.enablePreviewPolicies` | Enable preview policies. This parameter is deprecated. To enable OIDC Policies please use `controller.enableOIDC` instead. | false
 `controller.enableOIDC` | Enable OIDC policies. | false

deployments/helm-chart/templates/controller-daemonset.yaml

@@ -164,6 +164,9 @@ spec:
 {{- if .Values.controller.watchNamespace }}
           - -watch-namespace={{ .Values.controller.watchNamespace }}
 {{- end }}
+{{- if .Values.controller.watchNamespaceLabel }}
+          - -watch-namespace-label={{ .Values.controller.watchNamespaceLabel }}
+{{- end }}
 {{- if .Values.controller.watchSecretNamespace }}
           - -watch-secret-namespace={{ .Values.controller.watchSecretNamespace }}
 {{- end }}

deployments/helm-chart/templates/controller-deployment.yaml

@@ -167,6 +167,9 @@ spec:
 {{- if .Values.controller.watchNamespace }}
           - -watch-namespace={{ .Values.controller.watchNamespace }}
 {{- end }}
+{{- if .Values.controller.watchNamespaceLabel }}
+          - -watch-namespace-label={{ .Values.controller.watchNamespaceLabel }}
+{{- end }}
 {{- if .Values.controller.watchSecretNamespace }}
           - -watch-secret-namespace={{ .Values.controller.watchSecretNamespace }}
 {{- end }}

deployments/helm-chart/values.yaml

@@ -198,9 +198,12 @@ controller:
   ## New Ingresses without an ingressClassName field specified will be assigned the class specified in `controller.ingressClass`.
   setAsDefaultIngress: false
 
-  ## Comma separated list of namespaces to watch for Ingress resources. By default the Ingress Controller watches all namespaces.
+  ## Comma separated list of namespaces to watch for Ingress resources. By default the Ingress Controller watches all namespaces. Mutually exclusive with "controller.watchNamespaceLabel".
   watchNamespace: ""
 
+  ## Configures the Ingress Controller to watch only those namespaces with label foo=bar. By default the Ingress Controller watches all namespaces. Mutually exclusive with "controller.watchNamespace".
+  watchNamespaceLabel: ""
+
   ## Comma separated list of namespaces to watch for Secret resources. By default the Ingress Controller watches all namespaces.
   watchSecretNamespace: ""
 

docs/content/configuration/global-configuration/command-line-arguments.md

@@ -302,7 +302,19 @@ A comma-separated list of pattern=N settings for file-filtered logging.
 
 ### -watch-namespace `<string>`
 
-Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces.
+Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces. Mutually exclusive with "watch-namespace-label".
+&nbsp;
+<a name="cmdoption-watch-namespace-label"></a>
+
+### -watch-namespace-label `<string>`
+
+Configures the Ingress Controller to watch only those namespaces with label foo=bar. By default the Ingress Controller watches all namespaces. Mutually exclusive with "watch-namespace".
+&nbsp;
+<a name="cmdoption-watch-secret-namespace"></a>
+
+### -watch-secret-namespace `<string>`
+
+Comma separated list of namespaces the Ingress Controller should watch for secrets. If this arg is not configured, the Ingress Controller watches the same namespaces for all resources. See "watch-namespace" and "watch-namespace-label".
 &nbsp;
 <a name="cmdoption-enable-prometheus-metrics"></a>
 

docs/content/installation/installation-with-helm.md

@@ -185,8 +185,9 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont
 |``controller.replicaCount`` | The number of replicas of the Ingress Controller deployment. | 1 |
 |``controller.ingressClass`` | A class of the Ingress Controller. An IngressClass resource with the name equal to the class must be deployed. Otherwise, the Ingress Controller will fail to start. The Ingress Controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class. The Ingress Controller processes all the VirtualServer/VirtualServerRoute/TransportServer resources that do not have the "ingressClassName" field for all versions of kubernetes. | nginx |
 |``controller.setAsDefaultIngress`` | New Ingresses without an ingressClassName field specified will be assigned the class specified in `controller.ingressClass`. | false |
-|``controller.watchNamespace`` | Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces. Please note that if configuring multiple namespaces using the Helm cli `--set` option, the string needs to wrapped in double quotes and the commas escaped using a backslash - e.g. ``--set controller.watchNamespace="default\,nginx-ingress"``. | "" |
-|``controller.watchSecretNamespace`` | Comma separated list of namespaces the Ingress Controller should watch for resources of type Secret. If this arg is not configured, the Ingress Controller watches the same namespaces for all resources. See `watch-namespace`. Please note that if configuring multiple namespaces using the Helm cli `--set` option, the string needs to wrapped in double quotes and the commas escaped using a backslash - e.g. ``--set controller.watchSecretNamespace="default\,nginx-ingress"``. | "" |
+|``controller.watchNamespace`` | Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces. Mutually exclusive with `controller.watchNamespaceLabel`. Please note that if configuring multiple namespaces using the Helm cli `--set` option, the string needs to wrapped in double quotes and the commas escaped using a backslash - e.g. ``--set controller.watchNamespace="default\,nginx-ingress"``. | "" |
+|``controller.watchNamespaceLabel`` | Configures the Ingress Controller to watch only those namespaces with label foo=bar. By default the Ingress Controller watches all namespaces. Mutually exclusive with `controller.watchNamespace`. | "" |
+|``controller.watchSecretNamespace`` | Comma separated list of namespaces the Ingress Controller should watch for resources of type Secret. If this arg is not configured, the Ingress Controller watches the same namespaces for all resources. See `controller.watchNamespace` and `controller.watchNamespaceLabel`. Please note that if configuring multiple namespaces using the Helm cli `--set` option, the string needs to wrapped in double quotes and the commas escaped using a backslash - e.g. ``--set controller.watchSecretNamespace="default\,nginx-ingress"``. | "" |
 |``controller.enableCustomResources`` | Enable the custom resources. | true |
 |``controller.enablePreviewPolicies`` | Enable preview policies. This parameter is deprecated. To enable OIDC Policies please use ``controller.enableOIDC`` instead. | false |
 |``controller.enableOIDC`` | Enable OIDC policies. | false |