pmem-csi-operator: delete obsolete objects of a deployment

Different operator releases might result in obsolete sub-resources that
were created for a deployment. Operator shall detect these obsolete
objects if any and delete them in the reconcile loop.

There is no direct API to detect all the objects owned by an object.
So, we keep hard-coded list of all types that operator could deal and we
use this list to query and fetch the pre-deployed objects and check if
that object is owned by the deployment.

FIXES intel#595

pkg/pmem-csi-operator/controller/deployment/controller_driver.go

@@ -7,6 +7,7 @@ SPDX-License-Identifier: Apache-2.0
 package deployment
 
 import (
+	"context"
 	"crypto/rsa"
 	"crypto/tls"
 	"fmt"
@@ -25,10 +26,14 @@ import (
 	rbacv1 "k8s.io/api/rbac/v1"
 	storagev1beta1 "k8s.io/api/storage/v1beta1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	apiruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/klog"
+	"k8s.io/kubectl/pkg/scheme"
 )
 
 const (
@@ -37,6 +42,26 @@ const (
 	nodeControllerPort    = 10001
 )
 
+// A list of all object types potentially created by the
+// operator. It's okay and desirable to list more than actually used
+// at the moment, to catch new objects.
+// NOTE: Add the new object types to this list.
+// But, This list should not shrink even if the later versions of
+// operator does not install the object listed here.
+var allObjectTypes = []schema.GroupVersionKind{
+	rbacv1.SchemeGroupVersion.WithKind("RoleList"),
+	rbacv1.SchemeGroupVersion.WithKind("ClusterRoleList"),
+	rbacv1.SchemeGroupVersion.WithKind("RoleBindingList"),
+	rbacv1.SchemeGroupVersion.WithKind("ClusterRoleBindingList"),
+	corev1.SchemeGroupVersion.WithKind("ServiceAccountList"),
+	corev1.SchemeGroupVersion.WithKind("SecretList"),
+	corev1.SchemeGroupVersion.WithKind("ServiceList"),
+	corev1.SchemeGroupVersion.WithKind("ConfigMapList"),
+	appsv1.SchemeGroupVersion.WithKind("DaemonSetList"),
+	appsv1.SchemeGroupVersion.WithKind("StatefulSetList"),
+	storagev1beta1.SchemeGroupVersion.WithKind("CSIDriverList"),
+}
+
 type PmemCSIDriver struct {
 	*api.Deployment
 	// operators namespace used for creating sub-resources
@@ -135,6 +160,15 @@ func (d *PmemCSIDriver) reconcileDeploymentChanges(r *ReconcileDeployment, chang
 			return true, err
 		}
 		objects = append(objects, objs...)
+
+		// If not found cache might be result of operator restart
+		// check if this deployment has any objects to be deleted
+		if !foundInCache {
+			if err := d.deleteObsoleteObjects(r, objs); err != nil {
+				klog.Infof("Failed to delete obsolete objects: %v", err)
+				return true, err
+			}
+		}
 	} else {
 		if updateSecrets {
 			objs, err := d.getSecrets()
@@ -201,16 +235,82 @@ func (d *PmemCSIDriver) reconcileDeploymentChanges(r *ReconcileDeployment, chang
 	return false, nil
 }
 
-func (d *PmemCSIDriver) deployObjects(r *ReconcileDeployment) error {
-	objects, err := d.getDeploymentObjects()
-	if err != nil {
-		return err
+func (d *PmemCSIDriver) deleteObsoleteObjects(r *ReconcileDeployment, newObjects []apiruntime.Object) error {
+	type NameGVK struct {
+		name      string
+		namespace string
+		gvk       schema.GroupVersionKind
 	}
-	for _, obj := range objects {
-		if err := r.Create(obj); err != nil {
+
+	foundList := []NameGVK{}
+	obsoleteList := []NameGVK{}
+
+	for _, gvk := range allObjectTypes {
+		list := &unstructured.UnstructuredList{}
+		list.SetGroupVersionKind(gvk)
+
+		if err := r.client.List(context.TODO(), list); err != nil {
 			return err
 		}
+
+		for _, obj := range list.Items {
+			for _, owner := range obj.GetOwnerReferences() {
+				if owner.UID == d.GetUID() {
+					foundList = append(foundList, NameGVK{
+						name:      obj.GetName(),
+						namespace: obj.GetNamespace(),
+						gvk:       obj.GetObjectKind().GroupVersionKind()})
+					break
+				}
+			}
+		}
 	}
+
+	// prepare meta objects, that would be used later for validating the name
+	objectMetas := []metav1.Object{}
+	for _, o := range newObjects {
+		metaObj, err := meta.Accessor(o)
+		if err != nil {
+			return err
+		}
+		objectMetas = append(objectMetas, metaObj)
+	}
+
+	// check if the found objects for this deployment are still part of newObjects
+	// if not add them to obsoleteList
+	for _, found := range foundList {
+		obsolete := true
+
+		for i := range newObjects {
+
+			if objectMetas[i].GetName() == found.name && newObjects[i].GetObjectKind().GroupVersionKind() == found.gvk {
+				obsolete = false
+				break
+			}
+		}
+		if obsolete {
+			obsoleteList = append(obsoleteList, found)
+		}
+	}
+
+	// delete all obsolete objects
+	for _, o := range obsoleteList {
+		obj, err := scheme.Scheme.New(o.gvk)
+		if err != nil {
+			return err
+		}
+		metaObj, err := meta.Accessor(obj)
+		if err != nil {
+			return err
+		}
+		metaObj.SetName(o.name)
+		metaObj.SetNamespace(o.namespace)
+		klog.Infof("Deleteing obsolete object '%s' of type '%T", metaObj.GetName(), obj)
+		if err := r.Delete(obj); err != nil && !errors.IsNotFound(err) {
+			return err
+		}
+	}
+
 	return nil
 }
 

pkg/pmem-csi-operator/controller/deployment/deployment_controller.go

@@ -159,16 +159,21 @@ func (r *ReconcileDeployment) Reconcile(request reconcile.Request) (reconcile.Re
 		return reconcile.Result{Requeue: false}, nil
 	}
 
+	patch := client.MergeFrom(deployment.DeepCopy())
 	d := &PmemCSIDriver{deployment, r.namespace, r.k8sVersion}
 
-	// update status
+	// update status and deployment
 	defer func() {
 		klog.Infof("Updating deployment status....")
 		d.Deployment.Status.LastUpdated = metav1.Now()
 		if statusErr := r.client.Status().Update(context.TODO(), d.Deployment); statusErr != nil {
 			klog.Warningf("failed to update status %q for deployment %q: %v",
 				d.Deployment.Status.Phase, d.Name, statusErr)
 		}
+
+		if err := r.client.Patch(context.TODO(), d.Deployment, patch); err != nil {
+			klog.Warningf("Failed update deployment %q: %s", d.GetName(), err)
+		}
 	}()
 
 	requeue, err = d.Reconcile(r)