add healthz deep subpathes for all controllers

controllers/apps/deployment_controller.go

@@ -16,15 +16,17 @@ package apps
 import (
 	"context"
 
-	"github.com/aws/amazon-vpc-resource-controller-k8s/controllers/core"
+	controllers "github.com/aws/amazon-vpc-resource-controller-k8s/controllers/core"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config"
+	rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager"
 
 	"github.com/go-logr/logr"
 	appV1 "k8s.io/api/apps/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 )
 
 type DeploymentReconciler struct {
@@ -63,7 +65,13 @@ func (r *DeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	return ctrl.Result{}, nil
 }
 
-func (r *DeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error {
+func (r *DeploymentReconciler) SetupWithManager(mgr ctrl.Manager, healthzHandler *rcHealthz.HealthzHandler) error {
+	// add health check on subpath for deployment controller
+	// TODO: this is a simple controller and unlikely hit blocking issue but we can revisit this after subpaths are released for a while
+	healthzHandler.AddControllersHealthCheckers(
+		map[string]healthz.Checker{"health-deploy-controller": rcHealthz.SimplePing("deployment controller", r.Log)},
+	)
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&appV1.Deployment{}).
 		Complete(r)

controllers/core/configmap_controller.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config"
+	rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager"
 	"github.com/go-logr/logr"
@@ -27,6 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 )
 
 // ConfigMapReconciler reconciles a ConfigMap object
@@ -38,6 +40,7 @@ type ConfigMapReconciler struct {
 	K8sAPI                k8s.K8sWrapper
 	Condition             condition.Conditions
 	curWinIPAMEnabledCond bool
+	Context               context.Context
 }
 
 //+kubebuilder:rbac:groups=core,resources=configmaps,namespace=kube-system,resourceNames=amazon-vpc-cni,verbs=get;list;watch
@@ -84,7 +87,12 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 }
 
 // SetupWithManager sets up the controller with the Manager.
-func (r *ConfigMapReconciler) SetupWithManager(mgr ctrl.Manager) error {
+func (r *ConfigMapReconciler) SetupWithManager(mgr ctrl.Manager, healthzHandler *rcHealthz.HealthzHandler) error {
+	// add health check on subpath for CM controller
+	healthzHandler.AddControllersHealthCheckers(
+		map[string]healthz.Checker{"health-cm-controller": r.check()},
+	)
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&corev1.ConfigMap{}).
 		Complete(r)
@@ -110,3 +118,9 @@ func UpdateNodesOnConfigMapChanges(k8sAPI k8s.K8sWrapper, nodeManager manager.Ma
 	}
 	return nil
 }
+
+func (r *ConfigMapReconciler) check() healthz.Checker {
+	r.Log.Info("ConfigMap controller's healthz subpath was added")
+	// We can revisit this to use PingWithTimeout() instead if we have concerns on this controller.
+	return rcHealthz.SimplePing("configmap controller", r.Log)
+}

controllers/core/node_controller.go

@@ -16,17 +16,22 @@ package controllers
 import (
 	"context"
 	goErr "errors"
+	"net/http"
 
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition"
+	rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager"
+	"github.com/google/uuid"
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 )
 
 // MaxNodeConcurrentReconciles is the number of go routines that can invoke
@@ -43,7 +48,7 @@ type NodeReconciler struct {
 	Scheme     *runtime.Scheme
 	Manager    manager.Manager
 	Conditions condition.Conditions
-	// NodeEventCache *bigcache.BigCache
+	Context    context.Context
 }
 
 // +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch
@@ -68,6 +73,7 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 
 	if err := r.Client.Get(ctx, req.NamespacedName, node); err != nil {
 		if errors.IsNotFound(err) {
+			r.Log.V(1).Info("the requested node couldn't be found by k8s client", "Node", req.NamespacedName)
 			_, found := r.Manager.GetNode(req.Name)
 			// if cachedNode != nil && cachedNode.HasInstance() {
 			// 	// delete the not found node instance id from node event cache for housekeeping
@@ -80,7 +86,7 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 					logger.Error(err, "failed to delete node from manager")
 					return ctrl.Result{}, nil
 				}
-				logger.Info("deleted the node from manager")
+				logger.V(1).Info("deleted the node from manager")
 			}
 		}
 		return ctrl.Result{}, client.IgnoreNotFound(err)
@@ -98,17 +104,45 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 	return ctrl.Result{}, err
 }
 
-// func (r *NodeReconciler) deleteNodeFromNodeEventCache(nodeId string) {
-// 	if err := r.NodeEventCache.Delete(nodeId); err != nil {
-// 		r.Log.V(1).Info("node controller removing node from node event cache failed", "Error", err)
-// 	} else {
-// 		r.Log.V(1).Info("node controller removed the node from node event cache successfully", "InstanceId", nodeId)
-// 	}
-// }
+func (r *NodeReconciler) SetupWithManager(mgr ctrl.Manager, healthzHandler *rcHealthz.HealthzHandler) error {
+	// add health check on subpath for node controller
+	healthzHandler.AddControllersHealthCheckers(
+		map[string]healthz.Checker{"health-node-controller": r.Check()},
+	)
 
-func (r *NodeReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&corev1.Node{}).
 		WithOptions(controller.Options{MaxConcurrentReconciles: MaxNodeConcurrentReconciles}).
 		Complete(r)
 }
+
+func (r *NodeReconciler) Check() healthz.Checker {
+	r.Log.Info("Node controller's healthz subpath was added")
+	return func(req *http.Request) error {
+		// if the reconciler is not ready, using the simple ping to test
+		// this can test the referenced cached pod datastore
+		if !r.Conditions.GetPodDataStoreSyncStatus() {
+			r.Log.V(1).Info("***** node controller healthz enpoint tested Simple Ping *****")
+			return nil
+		}
+
+		err := rcHealthz.PingWithTimeout(func(c chan<- error) {
+			// when the reconciler is ready, testing the reconciler with a fake node request
+			pingRequest := &ctrl.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: corev1.NamespaceDefault,
+					Name:      uuid.New().String(),
+				},
+			}
+
+			// expecting to 'return ctrl.Result{}, client.IgnoreNotFound(err)'
+			// IgnoreNotFound returns nil on NotFound errors.
+			// this can test the pod cached datastore and node cached datastore
+			_, rErr := r.Reconcile(r.Context, *pingRequest)
+			r.Log.V(1).Info("***** node controller healthz endpoint tested Reconcile *****")
+			c <- rErr
+		}, r.Log)
+
+		return err
+	}
+}

controllers/core/pod_controller.go

@@ -15,20 +15,25 @@ package controllers
 
 import (
 	"context"
+	"net/http"
 	"time"
 
 	"github.com/aws/amazon-vpc-resource-controller-k8s/controllers/custom"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition"
+	rcHealthz "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/healthz"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s/pod"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/node/manager"
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/resource"
+	"github.com/google/uuid"
 
 	"github.com/go-logr/logr"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 )
 
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;update;patch
@@ -71,7 +76,7 @@ func (r *PodReconciler) Reconcile(request custom.Request) (ctrl.Result, error) {
 			return ctrl.Result{}, nil
 		}
 		if !exists {
-			r.Log.Info("pod doesn't exists in the cache anymore",
+			r.Log.V(1).Info("pod doesn't exists in the cache anymore",
 				"namespace name", request.NamespacedName.String())
 			return ctrl.Result{}, nil
 		}
@@ -174,9 +179,10 @@ func getAggregateResources(pod *v1.Pod) map[string]int64 {
 // list of runnable. After Manager acquire the lease the pod controller runnable
 // will be started and the Pod events will be sent to Reconcile function
 func (r *PodReconciler) SetupWithManager(ctx context.Context, manager ctrl.Manager,
-	clientSet *kubernetes.Clientset, pageLimit int, syncPeriod time.Duration) error {
+	clientSet *kubernetes.Clientset, pageLimit int, syncPeriod time.Duration, healthzHandler *rcHealthz.HealthzHandler) error {
 	r.Log.Info("The pod controller is using MaxConcurrentReconciles", "Routines", MaxPodConcurrentReconciles)
-	return custom.NewControllerManagedBy(ctx, manager).
+
+	customChecker, err := custom.NewControllerManagedBy(ctx, manager).
 		WithLogger(r.Log.WithName("custom pod controller")).
 		UsingDataStore(r.DataStore).
 		WithClientSet(clientSet).
@@ -188,4 +194,36 @@ func (r *PodReconciler) SetupWithManager(ctx context.Context, manager ctrl.Manag
 		ResyncPeriod:            syncPeriod,
 		MaxConcurrentReconciles: MaxPodConcurrentReconciles,
 	}).UsingConditions(r.Condition).Complete(r)
+
+	// add health check on subpath for pod and pod customized controllers
+	healthzHandler.AddControllersHealthCheckers(
+		map[string]healthz.Checker{
+			"health-pod-controller":        r.check(),
+			"health-custom-pod-controller": customChecker,
+		},
+	)
+
+	return err
+}
+
+func (r *PodReconciler) check() healthz.Checker {
+	r.Log.Info("Pod controller's healthz subpath was added")
+	// more meaningful ping
+	return func(req *http.Request) error {
+		err := rcHealthz.PingWithTimeout(func(c chan<- error) {
+			pingRequest := &custom.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: v1.NamespaceDefault,
+					Name:      uuid.New().String(),
+				},
+				DeletedObject: nil,
+			}
+			// calling reconcile will test pod cache
+			_, rErr := r.Reconcile(*pingRequest)
+			r.Log.V(1).Info("***** pod controller healthz endpoint tested reconcile *****")
+			c <- rErr
+		}, r.Log)
+
+		return err
+	}
 }

controllers/custom/builder.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/workqueue"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 )
 
@@ -89,21 +90,21 @@ func NewControllerManagedBy(ctx context.Context, mgr manager.Manager) *Builder {
 
 // Complete adds the controller to manager's Runnable. The Controller
 // runnable will start when the manager starts
-func (b *Builder) Complete(reconciler Reconciler) error {
+func (b *Builder) Complete(reconciler Reconciler) (healthz.Checker, error) {
 	// Loggr is no longer an interface
 	// The suggestion is using LogSink to do nil check now
 	if b.log.GetSink() == nil {
-		return fmt.Errorf("need to set the logger")
+		return nil, fmt.Errorf("need to set the logger")
 	}
 	if b.converter == nil {
-		return fmt.Errorf("converter not provided, " +
+		return nil, fmt.Errorf("converter not provided, " +
 			"must use high level controller if conversion not required")
 	}
 	if b.clientSet == nil {
-		return fmt.Errorf("need to set kubernetes clienset")
+		return nil, fmt.Errorf("need to set kubernetes clienset")
 	}
 	if b.dataStore == nil {
-		return fmt.Errorf("need datastore to start the controller")
+		return nil, fmt.Errorf("need datastore to start the controller")
 	}
 
 	b.SetDefaults()
@@ -172,17 +173,17 @@ func (b *Builder) Complete(reconciler Reconciler) error {
 		},
 	}
 
-	controller := &CustomController{
-		log:        b.log,
-		options:    b.options,
-		config:     config,
-		Do:         reconciler,
-		workQueue:  workQueue,
-		conditions: b.conditions,
-	}
+	controller := NewCustomController(
+		b.log,
+		b.options,
+		config,
+		reconciler,
+		workQueue,
+		b.conditions,
+	)
 
 	// Adds the controller to the manager's Runnable
-	return b.mgr.Add(controller)
+	return controller.checker, b.mgr.Add(controller)
 }
 
 // SetDefaults sets the default options for controller