add default NginxIngressController reconciler

go.mod

@@ -16,11 +16,13 @@ require (
 	github.com/prometheus/common v0.44.0
 	github.com/stretchr/testify v1.8.4
 	go.uber.org/zap v1.25.0
+	gomodules.xyz/jsonpatch/v2 v2.4.0
 	k8s.io/api v0.28.1
 	k8s.io/apiextensions-apiserver v0.28.1
 	k8s.io/apimachinery v0.28.1
 	k8s.io/client-go v0.28.1
 	k8s.io/klog/v2 v2.100.1
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
 	sigs.k8s.io/controller-runtime v0.16.1
 	sigs.k8s.io/secrets-store-csi-driver v1.3.4
 	sigs.k8s.io/yaml v1.3.0
@@ -73,15 +75,13 @@ require (
 	golang.org/x/term v0.11.0 // indirect
 	golang.org/x/text v0.12.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/component-base v0.28.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230816210353-14e408962443 // indirect
-	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
 )

pkg/controller/controller.go

@@ -115,9 +115,12 @@ func NewManagerForRestConfig(conf *config.Config, rc *rest.Config) (ctrl.Manager
 		return nil, fmt.Errorf("loading CRDs: %w", err)
 	}
 
-	if err := setupControllers(m, conf); err != nil {
-		return nil, fmt.Errorf("setting up controllers: %w", err)
-	}
+	go func() {
+		if err := setupControllers(m, conf, webhooksReady, setupLog); err != nil {
+			setupLog.Error(err, "unable to setup controllers")
+			os.Exit(1)
+		}
+	}()
 
 	webhookCfg, err := webhook.New(conf)
 	if err != nil {
@@ -147,8 +150,8 @@ func NewManagerForRestConfig(conf *config.Config, rc *rest.Config) (ctrl.Manager
 			setupLog.Error(err, "failed to setup webhooks")
 			os.Exit(1)
 		}
-		setupLog.Info("webhooks are ready")
 
+		setupLog.Info("webhooks are ready")
 		close(webhooksReady)
 	}()
 
@@ -163,18 +166,27 @@ func setupWebhooks(mgr ctrl.Manager, addWebhooksFn func(mgr ctrl.Manager) error)
 	return nil
 }
 
-func setupControllers(mgr ctrl.Manager, conf *config.Config) error {
-	var selfDeploy *appsv1.Deployment = nil // self deploy doesn't work because operator isn't in same resources as child resources
-
+func setupControllers(mgr ctrl.Manager, conf *config.Config, webhooksReady <-chan struct{}, lgr logr.Logger) error {
+	lgr.Info("settup up ExternalDNS controller")
 	if err := dns.NewExternalDns(mgr, conf); err != nil {
 		return fmt.Errorf("setting up external dns controller: %w", err)
 	}
 
+	lgr.Info("setting up Nginx Ingress Controller reconciler")
 	if err := nginxingress.NewReconciler(conf, mgr); err != nil {
 		return fmt.Errorf("setting up nginx ingress controller reconciler: %w", err)
 	}
 
-	nginxConfigs, err := nginx.New(mgr, conf, selfDeploy)
+	lgr.Info("waiting for webhooks to be ready")
+	<-webhooksReady // some controllers need to wait for webhooks to be ready because they interact directly with our CRDs
+	lgr.Info("finished waiting for webhooks to be ready")
+
+	lgr.Info("setting up default Nginx Ingress Controller reconciler")
+	if err := nginxingress.NewDefaultReconciler(mgr); err != nil {
+		return fmt.Errorf("setting up nginx ingress default controller reconciler: %w", err)
+	}
+
+	nginxConfigs, err := nginx.New(mgr, conf, nil)
 	if err != nil {
 		return fmt.Errorf("getting nginx configs: %w", err)
 	}
@@ -187,6 +199,7 @@ func setupControllers(mgr ctrl.Manager, conf *config.Config) error {
 		})
 	}
 
+	lgr.Info("setting up ingress concurrency watchdog")
 	if err := ingress.NewConcurrencyWatchdog(mgr, conf, watchdogTargets); err != nil {
 		return fmt.Errorf("setting up ingress concurrency watchdog: %w", err)
 	}
@@ -201,20 +214,25 @@ func setupControllers(mgr ctrl.Manager, conf *config.Config) error {
 	}
 
 	ingressManager := keyvault.NewIngressManager(ics)
+	lgr.Info("setting up keyvault secret provider class reconciler")
 	if err := keyvault.NewIngressSecretProviderClassReconciler(mgr, conf, ingressManager); err != nil {
 		return fmt.Errorf("setting up ingress secret provider class reconciler: %w", err)
 	}
+	lgr.Info("setting up keyvault placeholder pod controller")
 	if err := keyvault.NewPlaceholderPodController(mgr, conf, ingressManager); err != nil {
 		return fmt.Errorf("setting up placeholder pod controller: %w", err)
 	}
+	lgr.Info("setting up keyvault event mirror")
 	if err = keyvault.NewEventMirror(mgr, conf); err != nil {
 		return fmt.Errorf("setting up event mirror: %w", err)
 	}
 
 	ingressControllerNamer := osm.NewIngressControllerNamer(icToController)
+	lgr.Info("setting up ingress backend reconciler")
 	if err := osm.NewIngressBackendReconciler(mgr, conf, ingressControllerNamer); err != nil {
 		return fmt.Errorf("setting up ingress backend reconciler: %w", err)
 	}
+	lgr.Info("setting up ingress cert config reconciler")
 	if err = osm.NewIngressCertConfigReconciler(mgr, conf); err != nil {
 		return fmt.Errorf("setting up ingress cert config reconciler: %w", err)
 	}

pkg/controller/keyvault/placeholder_pod.go

@@ -138,8 +138,7 @@ func (p *PlaceholderPodController) Reconcile(ctx context.Context, req ctrl.Reque
 }
 
 func (p *PlaceholderPodController) buildDeployment(dep *appsv1.Deployment, spc *secv1.SecretProviderClass, ing *netv1.Ingress) {
-	labels := util.MergeMaps(map[string]string{"app": spc.Name}, manifests.GetTopLevelLabels())
-
+	labels := map[string]string{"app": spc.Name}
 	dep.Spec = appsv1.DeploymentSpec{
 		Replicas:             util.Int32Ptr(1),
 		RevisionHistoryLimit: util.Int32Ptr(2),

pkg/controller/keyvault/placeholder_pod_test.go

@@ -5,9 +5,10 @@ package keyvault
 
 import (
 	"context"
-	"k8s.io/apimachinery/pkg/api/errors"
 	"testing"
 
+	"k8s.io/apimachinery/pkg/api/errors"
+
 	"github.com/go-logr/logr"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -92,7 +93,7 @@ func TestPlaceholderPodControllerIntegration(t *testing.T) {
 	replicas := int32(1)
 	historyLimit := int32(2)
 
-	expectedLabels := util.MergeMaps(spc.Labels, map[string]string{"app": spc.Name})
+	expectedLabels := map[string]string{"app": spc.Name}
 	expected := appsv1.DeploymentSpec{
 		Replicas:             &replicas,
 		RevisionHistoryLimit: &historyLimit,
@@ -217,7 +218,7 @@ func TestPlaceholderPodControllerNoManagedByLabels(t *testing.T) {
 	replicas := int32(1)
 	historyLimit := int32(2)
 
-	expectedLabels := util.MergeMaps(map[string]string{"app": spc.Name}, manifests.GetTopLevelLabels())
+	expectedLabels := map[string]string{"app": spc.Name}
 	expected := appsv1.DeploymentSpec{
 		Replicas:             &replicas,
 		RevisionHistoryLimit: &historyLimit,

pkg/controller/nginxingress/default.go

@@ -0,0 +1,140 @@
+package nginxingress
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	approutingv1alpha1 "github.com/Azure/aks-app-routing-operator/api/v1alpha1"
+	"github.com/Azure/aks-app-routing-operator/pkg/controller/controllername"
+	"github.com/Azure/aks-app-routing-operator/pkg/controller/metrics"
+	"github.com/Azure/aks-app-routing-operator/pkg/util"
+	"github.com/go-logr/logr"
+	netv1 "k8s.io/api/networking/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	// DefaultcName is the default Ingress class name
+	DefaultIcName = "webapprouting.kubernetes.azure.com"
+	// DefaultNicName is the default Nginx Ingress Controller resource name
+	DefaultNicName         = "default"
+	DefaultNicResourceName = "nginx"
+)
+
+func NewDefaultReconciler(mgr ctrl.Manager) error {
+	name := controllername.New("default", "nginx", "ingress", "controller", "reconciler")
+	if err := mgr.Add(&defaultNicReconciler{
+		name:   name,
+		lgr:    name.AddToLogger(mgr.GetLogger()),
+		client: mgr.GetClient(),
+	}); err != nil {
+		return fmt.Errorf("adding default nginx ingress controller: %w", err)
+	}
+
+	return nil
+}
+
+type defaultNicReconciler struct {
+	name   controllername.ControllerNamer
+	client client.Client
+	lgr    logr.Logger
+}
+
+func (d *defaultNicReconciler) Start(ctx context.Context) (err error) {
+	start := time.Now()
+	d.lgr.Info("starting to reconcile default nginx ingress controller")
+	defer func() {
+		d.lgr.Info("finished reconciling default nginx ingress controller", "latencySec", time.Since(start).Seconds())
+		metrics.HandleControllerReconcileMetrics(d.name, ctrl.Result{}, err)
+	}()
+
+	nic := &approutingv1alpha1.NginxIngressController{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: approutingv1alpha1.GroupVersion.String(),
+			Kind:       "NginxIngressController",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: DefaultNicName,
+		},
+		Spec: approutingv1alpha1.NginxIngressControllerSpec{
+			ControllerNamePrefix: "nginx",
+			IngressClassName:     DefaultIcName,
+		},
+	}
+
+	d.lgr.Info("determining if default nginx ingress controller should be created")
+	shouldCreate, err := shouldCreateDefaultNic(d.client)
+	if err != nil {
+		d.lgr.Error(err, "checking if default nginx ingress controller should be created")
+		return fmt.Errorf("checking if default nginx ingress controller should be created: %w", err)
+	}
+
+	if !shouldCreate {
+		d.lgr.Info("default nginx ingress controller should not be created")
+		return nil
+	}
+
+	d.lgr.Info("upserting default nginx ingress controller")
+	if err := util.Upsert(ctx, d.client, nic); err != nil {
+		d.lgr.Error(err, "upserting default nginx ingress controller")
+		return fmt.Errorf("upserting default nginx ingress controller: %w", err)
+	}
+
+	return nil
+
+}
+
+func shouldCreateDefaultNic(cl client.Client) (bool, error) {
+	defaultIc := &netv1.IngressClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: DefaultIcName,
+		},
+	}
+
+	err := cl.Get(context.Background(), types.NamespacedName{Name: DefaultIcName}, defaultIc)
+	switch {
+	case err == nil: // default IngressClass exists, we must create default nic for upgrade story
+		return true, nil
+	case k8serrors.IsNotFound(err):
+		// default IngressClass does not exist, we don't need to create default nic. We aren't upgrading from older App Routing versions for the first time.
+		// this is either a new user or an existing user that deleted their default nic
+		return false, nil
+	case err != nil:
+		return false, fmt.Errorf("getting default ingress class: %w", err)
+	}
+
+	return false, nil
+}
+
+func getDefaultIngressClassControllerClass(cl client.Client) (string, error) {
+	defaultNicCc := "webapprouting.kubernetes.azure.com/nginx"
+
+	defaultIc := &netv1.IngressClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: DefaultIcName,
+		},
+	}
+	err := cl.Get(context.Background(), types.NamespacedName{Name: DefaultIcName}, defaultIc)
+	if err == nil {
+		defaultNicCc = defaultIc.Spec.Controller
+	}
+	if err != nil && !k8serrors.IsNotFound(err) {
+		return "", fmt.Errorf("getting default ingress class: %w", err)
+	}
+
+	return defaultNicCc, nil
+}
+
+// IsDefaultNic returns true if the given NginxIngressController is the default one
+func IsDefaultNic(nic *approutingv1alpha1.NginxIngressController) bool {
+	if nic == nil {
+		return false
+	}
+
+	return nic.Name == DefaultNicName && nic.Spec.IngressClassName == DefaultIcName
+}