Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor unit test #49

Merged
merged 7 commits into from
Feb 24, 2022
Merged

refactor unit test #49

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c22406f
refactor unit test
yifeijin Feb 23, 2022
48900e8
address comments
yifeijin Feb 23, 2022
db353c6
gofmt
yifeijin Feb 23, 2022
4524802
address comments
yifeijin Feb 24, 2022
66de62f
Merge from main branch
yifeijin Feb 24, 2022
19b378b
fix testing
yifeijin Feb 24, 2022
bd24711
update event logging
yifeijin Feb 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 84 additions & 76 deletions controllers/csm_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"sync/atomic"
"time"

k8sClient "github.com/dell/csm-operator/k8s"
"github.com/dell/csm-operator/pkg/drivers"
"github.com/dell/csm-operator/pkg/modules"
"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -164,21 +163,21 @@ func (r *ContainerStorageModuleReconciler) Reconcile(ctx context.Context, req ct

if csm.IsBeingDeleted() {
r.Log.Info(fmt.Sprintf("HandleFinalizer for %v", req.NamespacedName))
if err := r.removeFinalizer(csm); err != nil {
r.EventRecorder.Event(csm, corev1.EventTypeWarning, "Deleting finalizer", fmt.Sprintf("Failed to delete finalizer: %s", err))
yifeijin marked this conversation as resolved.
Show resolved Hide resolved
return ctrl.Result{}, fmt.Errorf("error when handling finalizer: %v", err)
}
r.EventRecorder.Event(csm, corev1.EventTypeNormal, "Deleted", "Object finalizer is deleted")

// check for force cleanup
if csm.Spec.Driver.ForceRemoveDriver {
// remove all resource deployed from CR by operator
if err := r.removeDriver(ctx, *csm, *operatorConfig, log); err != nil {
r.EventRecorder.Event(csm, corev1.EventTypeWarning, "Removing Driver", fmt.Sprintf("Failed to remove driver: %s", err))
if err := r.removeDriver(ctx, *csm, *operatorConfig); err != nil {
r.EventRecorder.Event(csm, corev1.EventTypeWarning, "Removed", fmt.Sprintf("Failed to remove driver: %s", err))
return ctrl.Result{}, fmt.Errorf("error when deleteing driver: %v", err)
}
}

if err := r.removeFinalizer(csm); err != nil {
r.EventRecorder.Event(csm, corev1.EventTypeWarning, "Deleted", fmt.Sprintf("Failed to delete finalizer: %s", err))
return ctrl.Result{}, fmt.Errorf("error when handling finalizer: %v", err)
}
r.EventRecorder.Event(csm, corev1.EventTypeNormal, "Deleted", "Object finalizer is deleted")

return ctrl.Result{}, nil
}
Expand All @@ -187,7 +186,7 @@ func (r *ContainerStorageModuleReconciler) Reconcile(ctx context.Context, req ct
if !csm.HasFinalizer(CSMFinalizerName) {
r.Log.Info(fmt.Sprintf("AddFinalizer for %v", req.NamespacedName))
if err := r.addFinalizer(csm); err != nil {
r.EventRecorder.Event(csm, corev1.EventTypeWarning, "Adding finalizer", fmt.Sprintf("Failed to add finalizer: %s", err))
r.EventRecorder.Event(csm, corev1.EventTypeWarning, "Added", fmt.Sprintf("Failed to add finalizer: %s", err))
return ctrl.Result{}, fmt.Errorf("error when adding finalizer: %v", err)
}
r.EventRecorder.Event(csm, corev1.EventTypeNormal, "Added", "Object finalizer is added")
Expand All @@ -197,7 +196,7 @@ func (r *ContainerStorageModuleReconciler) Reconcile(ctx context.Context, req ct
oldStatus := csm.GetCSMStatus()

// Before doing anything else, check for config version and apply annotation if not set
isUpdated, err := checkAndApplyConfigVersionAnnotations(csm, false)
isUpdated, err := checkAndApplyConfigVersionAnnotations(ctx, csm)
if err != nil {
r.EventRecorder.Eventf(csm, corev1.EventTypeWarning, "Updated", "Failed add annotation during install: %s", err.Error())
return utils.HandleValidationError(ctx, csm, r, err)
Expand All @@ -213,7 +212,7 @@ func (r *ContainerStorageModuleReconciler) Reconcile(ctx context.Context, req ct
// perfrom prechecks
err = r.PreChecks(ctx, csm, *operatorConfig)
if err != nil {
r.EventRecorder.Event(csm, corev1.EventTypeWarning, "Running Pre-Check", fmt.Sprintf("Failed Prechecks: %s", err))
r.EventRecorder.Event(csm, corev1.EventTypeWarning, "Updated", fmt.Sprintf("Failed Prechecks: %s", err))
return utils.HandleValidationError(ctx, csm, r, err)
}

Expand Down Expand Up @@ -261,7 +260,7 @@ func (r *ContainerStorageModuleReconciler) handleDeploymentUpdate(oldObj interfa
key := name + "-" + fmt.Sprintf("%d", r.GetUpdateCount())
ctx, log := logger.GetNewContextWithLogger(key)
if name == "" {
r.Log.Info("ignore deployment not labeled for csm", "name", d.Name)
log.Infow("ignore deployment not labeled for csm", "name", d.Name)
return
}

Expand Down Expand Up @@ -334,7 +333,7 @@ func (r *ContainerStorageModuleReconciler) handlePodsUpdate(oldObj interface{},
log.Infow("pod status ", "state", err.Error())
r.EventRecorder.Eventf(csm, "Warning", "Updated", "%s Pod error details %s", stamp, err.Error())
} else {
r.EventRecorder.Eventf(csm, "Normal", "Complete", "%s Driver pods running OK", stamp)
r.EventRecorder.Eventf(csm, "Normal", "Completed", "%s Driver pods running OK", stamp)
}

}
Expand Down Expand Up @@ -390,31 +389,27 @@ func (r *ContainerStorageModuleReconciler) handleDaemonsetUpdate(oldObj interfac
}

// ContentWatch - watch updates on deployment and deamonset
func (r *ContainerStorageModuleReconciler) ContentWatch() error {
func (r *ContainerStorageModuleReconciler) ContentWatch() {

clientset, err := k8sClient.GetClientSetWrapper()
if err != nil {
r.Log.Error(err, err.Error())
}
sharedInformerFactory := sinformer.NewSharedInformerFactory(r.K8sClient, time.Duration(time.Hour))

sharedInformerFactory := sinformer.NewSharedInformerFactory(clientset, time.Duration(time.Hour))
daemonsetInformer := sharedInformerFactory.Apps().V1().DaemonSets().Informer()
deploymentInformer := sharedInformerFactory.Apps().V1().Deployments().Informer()
daemonsetInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
UpdateFunc: r.handleDaemonsetUpdate,
})

deploymentInformer := sharedInformerFactory.Apps().V1().Deployments().Informer()
deploymentInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
UpdateFunc: r.handleDeploymentUpdate,
})

podsInformer := sharedInformerFactory.Core().V1().Pods().Informer()
podsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
UpdateFunc: r.handlePodsUpdate,
})

stop := make(chan struct{})
sharedInformerFactory.Start(stop)

return nil
}

// SetupWithManager sets up the controller with the Manager.
Expand Down Expand Up @@ -449,7 +444,7 @@ func (r *ContainerStorageModuleReconciler) SyncCSM(ctx context.Context, cr csmv1
log := logger.GetLogger(ctx)

// Get Driver resources
driverConfig, err := r.getDriverConfig(ctx, cr, operatorConfig, log)
driverConfig, err := r.getDriverConfig(ctx, cr, operatorConfig)
if err != nil {
return err
}
Expand Down Expand Up @@ -487,71 +482,65 @@ func (r *ContainerStorageModuleReconciler) SyncCSM(ctx context.Context, cr csmv1
}

// Create/Update ServiceAccount
err = serviceaccount.SyncServiceAccount(ctx, &node.Rbac.ServiceAccount, r.Client)
if err != nil {
if err = serviceaccount.SyncServiceAccount(ctx, &node.Rbac.ServiceAccount, r.Client); err != nil {
return err
}
err = serviceaccount.SyncServiceAccount(ctx, &controller.Rbac.ServiceAccount, r.Client)
if err != nil {

if err = serviceaccount.SyncServiceAccount(ctx, &controller.Rbac.ServiceAccount, r.Client); err != nil {
return err
}

// Create/Update ClusterRoles
_, err = rbac.SyncClusterRole(ctx, &node.Rbac.ClusterRole, r.Client)
if err != nil {
if _, err = rbac.SyncClusterRole(ctx, &node.Rbac.ClusterRole, r.Client); err != nil {
return err
}
_, err = rbac.SyncClusterRole(ctx, &controller.Rbac.ClusterRole, r.Client)
if err != nil {

if _, err = rbac.SyncClusterRole(ctx, &controller.Rbac.ClusterRole, r.Client); err != nil {
return err
}

// Create/Update ClusterRoleBinding
err = rbac.SyncClusterRoleBindings(ctx, &node.Rbac.ClusterRoleBinding, r.Client)
if err != nil {
if err = rbac.SyncClusterRoleBindings(ctx, &node.Rbac.ClusterRoleBinding, r.Client); err != nil {
return err
}
err = rbac.SyncClusterRoleBindings(ctx, &controller.Rbac.ClusterRoleBinding, r.Client)
if err != nil {

if err = rbac.SyncClusterRoleBindings(ctx, &controller.Rbac.ClusterRoleBinding, r.Client); err != nil {
return err
}

// Create/Update CSIDriver
err = csidriver.SyncCSIDriver(ctx, driver, r.Client)
if err != nil {
if err = csidriver.SyncCSIDriver(ctx, driver, r.Client); err != nil {
return err
}

// Create/Update ConfigMap
err = configmap.SyncConfigMap(ctx, configMap, r.Client)
if err != nil {
if err = configmap.SyncConfigMap(ctx, configMap, r.Client); err != nil {
return err
}

// Create/Update Deployment
err = deployment.SyncDeployment(ctx, &controller.Deployment, r.K8sClient, cr.Name)
if err != nil {
if err = deployment.SyncDeployment(ctx, &controller.Deployment, r.K8sClient, cr.Name); err != nil {
return err
}

// Create/Update DeamonSet

err = daemonset.SyncDaemonset(ctx, &node.DaemonSetApplyConfig, r.K8sClient, cr.Name)
if err != nil {
if err = daemonset.SyncDaemonset(ctx, &node.DaemonSetApplyConfig, r.K8sClient, cr.Name); err != nil {
return err
}
return nil

return nil
}

func (r *ContainerStorageModuleReconciler) getDriverConfig(ctx context.Context, cr csmv1.ContainerStorageModule, operatorConfig utils.OperatorConfig,
log *zap.SugaredLogger) (*DriverConfig, error) {
func (r *ContainerStorageModuleReconciler) getDriverConfig(ctx context.Context,
cr csmv1.ContainerStorageModule,
operatorConfig utils.OperatorConfig) (*DriverConfig, error) {
var (
err error
driver *storagev1.CSIDriver
configMap *corev1.ConfigMap
node *utils.NodeYAML
controller *utils.ControllerYAML
log = logger.GetLogger(ctx)
)

// Get Driver resources
Expand Down Expand Up @@ -592,17 +581,23 @@ func (r *ContainerStorageModuleReconciler) getDriverConfig(ctx context.Context,

}

func (r *ContainerStorageModuleReconciler) removeDriver(ctx context.Context, instance csmv1.ContainerStorageModule, operatorConfig utils.OperatorConfig,
reqLogger *zap.SugaredLogger) error {
func (r *ContainerStorageModuleReconciler) removeDriver(ctx context.Context, instance csmv1.ContainerStorageModule, operatorConfig utils.OperatorConfig) error {
log := logger.GetLogger(ctx)

deleteObj := func(obj client.Object) error {
err := r.GetClient().Get(ctx, t1.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()}, obj)
kind := obj.GetObjectKind().GroupVersionKind().Kind
name := obj.GetName()

err := r.GetClient().Get(ctx, t1.NamespacedName{Name: name, Namespace: obj.GetNamespace()}, obj)

if err != nil && k8serror.IsNotFound(err) {
log.Infow("Object not found to delete", "Name:", name, "Kind:", kind)
return nil
} else if err != nil {
reqLogger.Info("Unknown error.", "Error", err.Error())
log.Errorw("Unknown error to find object in deleteObj", "Error", err.Error(), "Name:", name, "Kind:", kind)
return err
} else {
reqLogger.Info("Deleting object", "Name:", obj.GetName(), "Kind:", obj.GetObjectKind().GroupVersionKind().Kind)
log.Infow("Deleting object", "Name:", name, "Kind:", kind)
err = r.GetClient().Delete(ctx, obj)
if err != nil {
return err
Expand All @@ -612,64 +607,74 @@ func (r *ContainerStorageModuleReconciler) removeDriver(ctx context.Context, ins
}

// Get Driver resources
driverConfig, err := r.getDriverConfig(ctx, instance, operatorConfig, reqLogger)
driverConfig, err := r.getDriverConfig(ctx, instance, operatorConfig)
if err != nil {
log.Error("error in getDriverConfig")
return err
}

err = deleteObj(&driverConfig.Node.Rbac.ServiceAccount)
if err != nil {
if err = deleteObj(&driverConfig.Node.Rbac.ServiceAccount); err != nil {
log.Errorw("error delete node service account", "Error", err.Error())
return err
}
err = deleteObj(&driverConfig.Controller.Rbac.ServiceAccount)
if err != nil {

if err = deleteObj(&driverConfig.Controller.Rbac.ServiceAccount); err != nil {
log.Errorw("error delete controller service account", "Error", err.Error())
return err
}

err = deleteObj(&driverConfig.Node.Rbac.ClusterRole)
if err != nil {
if err = deleteObj(&driverConfig.Node.Rbac.ClusterRole); err != nil {
log.Errorw("error delete node cluster role", "Error", err.Error())
return err
}
err = deleteObj(&driverConfig.Controller.Rbac.ClusterRole)
if err != nil {

if err = deleteObj(&driverConfig.Controller.Rbac.ClusterRole); err != nil {
log.Errorw("error delete controller cluster role", "Error", err.Error())
return err
}

err = deleteObj(&driverConfig.Node.Rbac.ClusterRoleBinding)
if err != nil {
if err = deleteObj(&driverConfig.Node.Rbac.ClusterRoleBinding); err != nil {
log.Errorw("error delete controller cluster role", "Error", err.Error())
return err
}
err = deleteObj(&driverConfig.Controller.Rbac.ClusterRoleBinding)
if err != nil {

if err = deleteObj(&driverConfig.Controller.Rbac.ClusterRoleBinding); err != nil {
log.Errorw("error delete controller cluster role binding", "Error", err.Error())
return err
}

err = deleteObj(driverConfig.ConfigMap)
if err != nil {
if err = deleteObj(driverConfig.ConfigMap); err != nil {
log.Errorw("error delete configmap", "Error", err.Error())
return err
}

err = deleteObj(driverConfig.Driver)
if err != nil {
if err = deleteObj(driverConfig.Driver); err != nil {
log.Errorw("error delete csi driver", "Error", err.Error())
return err
}

daemonsetKey := client.ObjectKey{
Namespace: *driverConfig.Node.DaemonSetApplyConfig.Namespace,
Name: *driverConfig.Node.DaemonSetApplyConfig.Name,
}

daemonsetObj := &appsv1.DaemonSet{}
if err = r.Get(ctx, daemonsetKey, daemonsetObj); err == nil {
r.Delete(ctx, daemonsetObj)
if err = r.Delete(ctx, daemonsetObj); err != nil {
log.Errorw("error delete daemonset", "Error", err.Error())
}
}

deploymentKey := client.ObjectKey{
Namespace: *driverConfig.Controller.Deployment.Namespace,
Name: *driverConfig.Controller.Deployment.Name,
}

deploymentObj := &appsv1.Deployment{}
if err = r.Get(ctx, deploymentKey, deploymentObj); err == nil {
r.Delete(ctx, deploymentObj)
if err = r.Delete(ctx, deploymentObj); err != nil {
log.Errorw("error delete deployment", "Error", err.Error())
}
}

return nil
Expand Down Expand Up @@ -719,7 +724,10 @@ func (r *ContainerStorageModuleReconciler) PreChecks(ctx context.Context, cr *cs

}

func checkAndApplyConfigVersionAnnotations(instance *csmv1.ContainerStorageModule, update bool) (bool, error) {
func checkAndApplyConfigVersionAnnotations(ctx context.Context, instance *csmv1.ContainerStorageModule) (bool, error) {

log := logger.GetLogger(ctx)

if instance.Spec.Driver.ConfigVersion == "" {
// fail immediately
return false, fmt.Errorf("mandatory argument: ConfigVersion missing")
Expand All @@ -736,15 +744,15 @@ func checkAndApplyConfigVersionAnnotations(instance *csmv1.ContainerStorageModul
annotations[configVersionKey] = instance.Spec.Driver.ConfigVersion
isUpdated = true
instance.SetAnnotations(annotations)
fmt.Println(fmt.Sprintf("Installing CSI Driver %s with config Version %s. Updating Annotations with Config Version",
instance.GetName(), instance.Spec.Driver.ConfigVersion))
log.Infof("Installing CSI Driver %s with config Version %s. Updating Annotations with Config Version",
instance.GetName(), instance.Spec.Driver.ConfigVersion)
} else {
if configVersion != instance.Spec.Driver.ConfigVersion {
annotations[configVersionKey] = instance.Spec.Driver.ConfigVersion
isUpdated = true
instance.SetAnnotations(annotations)
fmt.Println(fmt.Sprintf("Config Version changed from %s to %s. Updating Annotations",
configVersion, instance.Spec.Driver.ConfigVersion))
log.Infof("Config Version changed from %s to %s. Updating Annotations",
configVersion, instance.Spec.Driver.ConfigVersion)
}
}
return isUpdated, nil
Expand Down
Loading