feat: Support watching multiple namespaces (#2914)

* Initial commit for multiple namespaces

* add validation

* PR feedback

cmd/nginx-ingress/flags.go

@@ -36,7 +36,9 @@ var (
 	The Ingress Controller does not start NGINX and does not write any generated NGINX configuration files to disk`)
 
 	watchNamespace = flag.String("watch-namespace", api_v1.NamespaceAll,
-		`Namespace to watch for Ingress resources. By default the Ingress Controller watches all namespaces`)
+		`Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces`)
+
+	watchNamespaces []string
 
 	nginxConfigMaps = flag.String("nginx-configmaps", "",
 		`A ConfigMap resource for customizing NGINX configuration. If a ConfigMap is set,
@@ -194,6 +196,8 @@ func parseFlags(versionInfo string, binaryInfo string) {
 	glog.Infof("Starting NGINX Ingress Controller %v PlusFlag=%v", versionInfo, *nginxPlus)
 	glog.Info(binaryInfo)
 
+	watchNamespaces = strings.Split(*watchNamespace, ",")
+
 	validationChecks()
 
 	if *enableTLSPassthrough && !*enableCustomResources {
@@ -302,6 +306,11 @@ func validationChecks() {
 		glog.Fatalf("Invalid value for leader-election-lock-name: %v", statusLockNameValidationError)
 	}
 
+	namespacesNameValidationError := validateNamespaceNames(watchNamespaces)
+	if namespacesNameValidationError != nil {
+		glog.Fatalf("Invalid values for namespaces: %v", namespacesNameValidationError)
+	}
+
 	statusPortValidationError := validatePort(*nginxStatusPort)
 	if statusPortValidationError != nil {
 		glog.Fatalf("Invalid value for nginx-status-port: %v", statusPortValidationError)
@@ -331,11 +340,31 @@ func validationChecks() {
 	}
 }
 
+// validateNamespaceNames validates the namespaces are in the correct format
+func validateNamespaceNames(namespaces []string) error {
+	var allErrs []error
+
+	for _, ns := range namespaces {
+		if ns != "" {
+			ns = strings.TrimSpace(ns)
+			err := validateResourceName(ns)
+			if err != nil {
+				allErrs = append(allErrs, err)
+				fmt.Printf("error %v ", err)
+			}
+		}
+	}
+	if len(allErrs) > 0 {
+		return fmt.Errorf("errors validating namespaces: %v", allErrs)
+	}
+	return nil
+}
+
 // validateResourceName validates the name of a resource
-func validateResourceName(lock string) error {
-	allErrs := validation.IsDNS1123Subdomain(lock)
+func validateResourceName(name string) error {
+	allErrs := validation.IsDNS1123Subdomain(name)
 	if len(allErrs) > 0 {
-		return fmt.Errorf("invalid resource name %v: %v", lock, allErrs)
+		return fmt.Errorf("invalid resource name %v: %v", name, allErrs)
 	}
 	return nil
 }

cmd/nginx-ingress/main.go

@@ -47,6 +47,8 @@ func main() {
 
 	validateIngressClass(kubeClient)
 
+	checkNamespaceExists(kubeClient)
+
 	dynClient, confClient := createCustomClients(config)
 
 	constLabels := map[string]string{"class": *ingressClass}
@@ -117,7 +119,7 @@ func main() {
 		DynClient:                    dynClient,
 		RestConfig:                   config,
 		ResyncPeriod:                 30 * time.Second,
-		Namespace:                    *watchNamespace,
+		Namespace:                    watchNamespaces,
 		NginxConfigurator:            cnf,
 		DefaultServerSecret:          *defaultServerSecret,
 		AppProtectEnabled:            *appProtect,
@@ -230,6 +232,17 @@ func validateIngressClass(kubeClient kubernetes.Interface) {
 	}
 }
 
+func checkNamespaceExists(kubeClient kubernetes.Interface) {
+	for _, ns := range watchNamespaces {
+		if ns != "" {
+			_, err := kubeClient.CoreV1().Namespaces().Get(context.TODO(), ns, meta_v1.GetOptions{})
+			if err != nil {
+				glog.Warningf("Error when getting Namespace %v: %v", ns, err)
+			}
+		}
+	}
+}
+
 func createCustomClients(config *rest.Config) (dynamic.Interface, k8s_nginx.Interface) {
 	var dynClient dynamic.Interface
 	var err error

cmd/nginx-ingress/main_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"errors"
 	"reflect"
+	"strings"
 	"testing"
 )
 
@@ -153,3 +154,21 @@ func TestValidateAppProtectLogLevel(t *testing.T) {
 		}
 	}
 }
+
+func TestValidateNamespaces(t *testing.T) {
+	badNamespaces := []string{"watchns1, watchns2, watchns%$", "watchns1,watchns2,watchns%$"}
+	for _, badNs := range badNamespaces {
+		err := validateNamespaceNames(strings.Split(badNs, ","))
+		if err == nil {
+			t.Errorf("Expected error for invalid namespace %v\n", badNs)
+		}
+	}
+
+	goodNamespaces := []string{"watched-namespace", "watched-namespace,", "watched-namespace1,watched-namespace2", "watched-namespace1, watched-namespace2"}
+	for _, goodNs := range goodNamespaces {
+		err := validateNamespaceNames(strings.Split(goodNs, ","))
+		if err != nil {
+			t.Errorf("Error for valid namespace:  %v err: %v\n", goodNs, err)
+		}
+	}
+}

deployments/helm-chart/README.md

@@ -181,7 +181,7 @@ Parameter | Description | Default
 `controller.replicaCount` | The number of replicas of the Ingress Controller deployment. | 1
 `controller.ingressClass` | A class of the Ingress Controller. An IngressClass resource with the name equal to the class must be deployed. Otherwise, the Ingress Controller will fail to start. The Ingress Controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class. The Ingress Controller processes all the VirtualServer/VirtualServerRoute/TransportServer resources that do not have the "ingressClassName" field for all versions of kubernetes. | nginx
 `controller.setAsDefaultIngress` | New Ingresses without an `"ingressClassName"` field specified will be assigned the class specified in `controller.ingressClass`. | false
-`controller.watchNamespace` | Namespace to watch for Ingress resources. By default the Ingress Controller watches all namespaces. | ""
+`controller.watchNamespace` | Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces. | ""
 `controller.enableCustomResources` | Enable the custom resources. | true
 `controller.enablePreviewPolicies` | Enable preview policies. This parameter is deprecated. To enable OIDC Policies please use `controller.enableOIDC` instead. | false
 `controller.enableOIDC` | Enable OIDC policies. | false

deployments/helm-chart/values.yaml

@@ -166,7 +166,7 @@ controller:
   ## New Ingresses without an ingressClassName field specified will be assigned the class specified in `controller.ingressClass`.
   setAsDefaultIngress: false
 
-  ## Namespace to watch for Ingress resources. By default the Ingress Controller watches all namespaces.
+  ## Comma separated list of namespaces to watch for Ingress resources. By default the Ingress Controller watches all namespaces.
   watchNamespace: ""
 
   ## Enable the custom resources.

deployments/rbac/rbac.yaml

@@ -37,6 +37,14 @@ rules:
   verbs:
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:

docs/content/configuration/global-configuration/command-line-arguments.md

@@ -302,7 +302,7 @@ A comma-separated list of pattern=N settings for file-filtered logging.
 
 ### -watch-namespace `<string>`
 
-Namespace to watch for Ingress resources. By default the Ingress Controller watches all namespaces.
+Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces.
 &nbsp;
 <a name="cmdoption-enable-prometheus-metrics"></a>
 

docs/content/installation/installation-with-helm.md

@@ -183,7 +183,7 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont
 |``controller.replicaCount`` | The number of replicas of the Ingress Controller deployment. | 1 |
 |``controller.ingressClass`` | A class of the Ingress Controller. An IngressClass resource with the name equal to the class must be deployed. Otherwise, the Ingress Controller will fail to start. The Ingress Controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class. The Ingress Controller processes all the VirtualServer/VirtualServerRoute/TransportServer resources that do not have the "ingressClassName" field for all versions of kubernetes. | nginx |
 |``controller.setAsDefaultIngress`` | New Ingresses without an ingressClassName field specified will be assigned the class specified in `controller.ingressClass`. | false |
-|``controller.watchNamespace`` | Namespace to watch for Ingress resources. By default the Ingress Controller watches all namespaces. | "" |
+|``controller.watchNamespace`` | Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces. | "" |
 |``controller.enableCustomResources`` | Enable the custom resources. | true |
 |``controller.enablePreviewPolicies`` | Enable preview policies. This parameter is deprecated. To enable OIDC Policies please use ``controller.enableOIDC`` instead. | false |
 |``controller.enableOIDC`` | Enable OIDC policies. | false |

docs/content/installation/running-multiple-ingress-controllers.md

@@ -44,7 +44,7 @@ To make sure that NGINX Ingress Controller handles particular configuration reso
 
 When running NGINX Ingress Controller, you have the following options with regards to which configuration resources it handles:
 * **Cluster-wide Ingress Controller (default)**. The Ingress Controller handles configuration resources created in any namespace of the cluster. As NGINX is a high-performance load balancer capable of serving many applications at the same time, this option is used by default in our installation manifests and Helm chart.
-* **Single-namespace Ingress Controller**. You can configure the Ingress Controller to handle configuration resources only from a particular namespace, which is controlled through the `-watch-namespace` command-line argument. This can be useful if you want to use different NGINX Ingress Controllers for different applications, both in terms of isolation and/or operation.
+* **Defined-namespace Ingress Controller**. You can configure the Ingress Controller to handle configuration resources only from particular namespaces, which is controlled through the `-watch-namespace` command-line argument. This can be useful if you want to use different NGINX Ingress Controllers for different applications, both in terms of isolation and/or operation.
 * **Ingress Controller for Specific Ingress Class**. This option works in conjunction with either of the options above. You can further customize which configuration resources are handled by the Ingress Controller by configuring the class of the Ingress Controller and using that class in your configuration resources. See the section [Configuring Ingress Class](#configuring-ingress-class).
 
 Considering the options above, you can run multiple NGINX Ingress Controllers, each handling a different set of configuration resources.

internal/certmanager/cm_controller.go

@@ -23,6 +23,7 @@ import (
 	cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
 	cm_clientset "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned"
 	cm_informers "github.com/cert-manager/cert-manager/pkg/client/informers/externalversions"
+	cmlisters "github.com/cert-manager/cert-manager/pkg/client/listers/certmanager/v1"
 	controllerpkg "github.com/cert-manager/cert-manager/pkg/controller"
 	"github.com/golang/glog"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,6 +35,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
 
+	conf_v1 "github.com/nginxinc/kubernetes-ingress/pkg/apis/configuration/v1"
 	k8s_nginx "github.com/nginxinc/kubernetes-ingress/pkg/client/clientset/versioned"
 	vsinformers "github.com/nginxinc/kubernetes-ingress/pkg/client/informers/externalversions"
 	listers_v1 "github.com/nginxinc/kubernetes-ingress/pkg/client/listers/configuration/v1"
@@ -54,14 +56,14 @@ const (
 // and creates/ updates certificates for VS resources as required,
 // and VS resources when certificate objects are created/ updated
 type CmController struct {
-	vsLister                  listers_v1.VirtualServerLister
+	vsLister                  []listers_v1.VirtualServerLister
 	sync                      SyncFn
 	ctx                       context.Context
 	mustSync                  []cache.InformerSynced
 	queue                     workqueue.RateLimitingInterface
-	vsSharedInformerFactory   vsinformers.SharedInformerFactory
-	cmSharedInformerFactory   cm_informers.SharedInformerFactory
-	kubeSharedInformerFactory kubeinformers.SharedInformerFactory
+	vsSharedInformerFactory   []vsinformers.SharedInformerFactory
+	cmSharedInformerFactory   []cm_informers.SharedInformerFactory
+	kubeSharedInformerFactory []kubeinformers.SharedInformerFactory
 	recorder                  record.EventRecorder
 	cmClient                  *cm_clientset.Clientset
 }
@@ -71,27 +73,31 @@ type CmOpts struct {
 	context       context.Context
 	kubeConfig    *rest.Config
 	kubeClient    kubernetes.Interface
-	namespace     string
+	namespace     []string
 	eventRecorder record.EventRecorder
 	vsClient      k8s_nginx.Interface
 }
 
 func (c *CmController) register() workqueue.RateLimitingInterface {
-	c.vsLister = c.vsSharedInformerFactory.K8s().V1().VirtualServers().Lister()
-	c.vsSharedInformerFactory.K8s().V1().VirtualServers().Informer().AddEventHandler(&controllerpkg.QueuingEventHandler{
-		Queue: c.queue,
-	})
+	var cmLister []cmlisters.CertificateLister
+	for _, sif := range c.vsSharedInformerFactory {
+		c.vsLister = append(c.vsLister, sif.K8s().V1().VirtualServers().Lister())
+		sif.K8s().V1().VirtualServers().Informer().AddEventHandler(&controllerpkg.QueuingEventHandler{
+			Queue: c.queue,
+		})
+		c.mustSync = append(c.mustSync, sif.K8s().V1().VirtualServers().Informer().HasSynced)
+	}
 
-	c.sync = SyncFnFor(c.recorder, c.cmClient, c.cmSharedInformerFactory.Certmanager().V1().Certificates().Lister())
+	for _, cif := range c.cmSharedInformerFactory {
+		cif.Certmanager().V1().Certificates().Informer().AddEventHandler(&controllerpkg.BlockingEventHandler{
+			WorkFunc: certificateHandler(c.queue),
+		})
+		cmLister = append(cmLister, cif.Certmanager().V1().Certificates().Lister())
+		c.mustSync = append(c.mustSync, cif.Certmanager().V1().Certificates().Informer().HasSynced)
+	}
 
-	c.cmSharedInformerFactory.Certmanager().V1().Certificates().Informer().AddEventHandler(&controllerpkg.BlockingEventHandler{
-		WorkFunc: certificateHandler(c.queue),
-	})
+	c.sync = SyncFnFor(c.recorder, c.cmClient, cmLister)
 
-	c.mustSync = []cache.InformerSynced{
-		c.vsSharedInformerFactory.K8s().V1().VirtualServers().Informer().HasSynced,
-		c.cmSharedInformerFactory.Certmanager().V1().Certificates().Informer().HasSynced,
-	}
 	return c.queue
 }
 
@@ -103,7 +109,13 @@ func (c *CmController) processItem(ctx context.Context, key string) error {
 		return err
 	}
 
-	vs, err := c.vsLister.VirtualServers(namespace).Get(name)
+	var vs *conf_v1.VirtualServer
+	for _, vl := range c.vsLister {
+		vs, err = vl.VirtualServers(namespace).Get(name)
+		if err == nil {
+			break
+		}
+	}
 	if err != nil {
 		return err
 	}
@@ -156,9 +168,15 @@ func NewCmController(opts *CmOpts) *CmController {
 	// Create a cert-manager api client
 	intcl, _ := cm_clientset.NewForConfig(opts.kubeConfig)
 
-	cmSharedInformerFactory := cm_informers.NewSharedInformerFactoryWithOptions(intcl, resyncPeriod, cm_informers.WithNamespace(opts.namespace))
-	kubeSharedInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(opts.kubeClient, resyncPeriod, kubeinformers.WithNamespace(opts.namespace))
-	vsSharedInformerFactory := vsinformers.NewSharedInformerFactoryWithOptions(opts.vsClient, resyncPeriod, vsinformers.WithNamespace(opts.namespace))
+	var vsSharedInformerFactory []vsinformers.SharedInformerFactory
+	var cmSharedInformerFactory []cm_informers.SharedInformerFactory
+	var kubeSharedInformerFactory []kubeinformers.SharedInformerFactory
+
+	for _, ns := range opts.namespace {
+		cmSharedInformerFactory = append(cmSharedInformerFactory, cm_informers.NewSharedInformerFactoryWithOptions(intcl, resyncPeriod, cm_informers.WithNamespace(ns)))
+		kubeSharedInformerFactory = append(kubeSharedInformerFactory, kubeinformers.NewSharedInformerFactoryWithOptions(opts.kubeClient, resyncPeriod, kubeinformers.WithNamespace(ns)))
+		vsSharedInformerFactory = append(vsSharedInformerFactory, vsinformers.NewSharedInformerFactoryWithOptions(opts.vsClient, resyncPeriod, vsinformers.WithNamespace(ns)))
+	}
 
 	cm := &CmController{
 		ctx:                       opts.context,
@@ -183,9 +201,15 @@ func (c *CmController) Run(stopCh <-chan struct{}) {
 
 	glog.Infof("Starting cert-manager control loop")
 
-	go c.vsSharedInformerFactory.Start(c.ctx.Done())
-	go c.cmSharedInformerFactory.Start(c.ctx.Done())
-	go c.kubeSharedInformerFactory.Start(c.ctx.Done())
+	for _, vif := range c.vsSharedInformerFactory {
+		go vif.Start(c.ctx.Done())
+	}
+	for _, cif := range c.cmSharedInformerFactory {
+		go cif.Start(c.ctx.Done())
+	}
+	for _, kif := range c.kubeSharedInformerFactory {
+		go kif.Start(c.ctx.Done())
+	}
 	// // wait for all the informer caches we depend on are synced
 	glog.V(3).Infof("Waiting for %d caches to sync", len(c.mustSync))
 	if !cache.WaitForNamedCacheSync(ControllerName, stopCh, c.mustSync...) {
@@ -234,7 +258,7 @@ func (c *CmController) runWorker(ctx context.Context) {
 }
 
 // BuildOpts builds a CmOpts from the given parameters
-func BuildOpts(ctx context.Context, kc *rest.Config, cl kubernetes.Interface, ns string, er record.EventRecorder, vsc k8s_nginx.Interface) *CmOpts {
+func BuildOpts(ctx context.Context, kc *rest.Config, cl kubernetes.Interface, ns []string, er record.EventRecorder, vsc k8s_nginx.Interface) *CmOpts {
 	return &CmOpts{
 		context:       ctx,
 		kubeClient:    cl,

internal/certmanager/cm_controller_test.go

@@ -23,16 +23,19 @@ import (
 
 	cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
 	cmclient "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned"
+	cm_informers "github.com/cert-manager/cert-manager/pkg/client/informers/externalversions"
 	controllerpkg "github.com/cert-manager/cert-manager/pkg/controller"
 	testpkg "github.com/nginxinc/kubernetes-ingress/internal/certmanager/test_files"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/util/workqueue"
 
 	vsapi "github.com/nginxinc/kubernetes-ingress/pkg/apis/configuration/v1"
 	k8s_nginx "github.com/nginxinc/kubernetes-ingress/pkg/client/clientset/versioned"
+	vsinformers "github.com/nginxinc/kubernetes-ingress/pkg/client/informers/externalversions"
 )
 
 func Test_controller_Register(t *testing.T) {
@@ -138,10 +141,10 @@ func Test_controller_Register(t *testing.T) {
 			cm := &CmController{
 				ctx:                       b.RootContext,
 				queue:                     workqueue.NewNamedRateLimitingQueue(controllerpkg.DefaultItemBasedRateLimiter(), ControllerName),
-				cmSharedInformerFactory:   b.FakeCMInformerFactory(),
-				kubeSharedInformerFactory: b.FakeKubeInformerFactory(),
+				cmSharedInformerFactory:   []cm_informers.SharedInformerFactory{b.FakeCMInformerFactory()},
+				kubeSharedInformerFactory: []kubeinformers.SharedInformerFactory{b.FakeKubeInformerFactory()},
 				recorder:                  b.Recorder,
-				vsSharedInformerFactory:   b.VsSharedInformerFactory,
+				vsSharedInformerFactory:   []vsinformers.SharedInformerFactory{b.VsSharedInformerFactory},
 			}
 
 			queue := cm.register()