Update terminatingpod sanity check (#85)

* update terminatingpod sanity check The operator will now disassociate pods that are stuck in terminating state for over a minute Signed-off-by: Anand Sanmukhani <[email protected]> * add SetPodLabels function to Fakecontrol Signed-off-by: Anand Sanmukhani <[email protected]> Signed-off-by: Anand Sanmukhani <[email protected]> Signed-off-by: Anand Sanmukhani <[email protected]> Signed-off-by: Anand Sanmukhani <[email protected]>
IBM · Jan 13, 2023 · dad46da · dad46da
1 parent bbdc9f0
commit dad46da
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 1 deletion.
diff --git a/pkg/controller/pod/control.go b/pkg/controller/pod/control.go
@@ -31,6 +31,8 @@ type RedisClusterControlInterface interface {
 	DeletePod(redisCluster *rapi.RedisCluster, podName string) error
 	// DeletePodNow used to delete now (force) a pod from its name
 	DeletePodNow(redisCluster *rapi.RedisCluster, podName string) error
+	// SetPodLabels used to set a new map of labels to pod
+	SetPodLabels(pod kapiv1.Pod, podLabels map[string]string) error
 }
 
 var _ RedisClusterControlInterface = &RedisClusterControl{}
@@ -76,6 +78,16 @@ func (p *RedisClusterControl) CreatePod(redisCluster *rapi.RedisCluster) (*kapiv
 	return pod, nil
 }
 
+// SetPodLabels used to set a new map of labels to pod
+func (p *RedisClusterControl) SetPodLabels(pod kapiv1.Pod, podLabels map[string]string) error {
+
+	pod.SetLabels(podLabels)
+	if err := p.KubeClient.Update(context.Background(), &pod); err != nil {
+		return err
+	}
+	return nil
+}
+
 // CreatePodOnNode used to create a Pod on the given node
 func (p *RedisClusterControl) CreatePodOnNode(redisCluster *rapi.RedisCluster, nodeName string) (*kapiv1.Pod, error) {
 	pod, err := initPod(redisCluster)

diff --git a/pkg/controller/sanitycheck/process.go b/pkg/controller/sanitycheck/process.go
@@ -35,7 +35,7 @@ func RunSanityChecks(ctx context.Context, admin redis.AdminInterface, config *co
 	}
 
 	// delete pods that are stuck in terminating state
-	if actionDone, err = FixTerminatingPods(cluster, podControl, 5*time.Minute, dryRun); err != nil {
+	if actionDone, err = FixTerminatingPods(cluster, podControl, 1*time.Minute, dryRun); err != nil {
 		return actionDone, err
 	} else if actionDone {
 		glog.V(2).Infof("FixTerminatingPods executed an action on the cluster (dryRun: %v)", dryRun)

diff --git a/pkg/controller/sanitycheck/terminatingpod.go b/pkg/controller/sanitycheck/terminatingpod.go
@@ -31,11 +31,22 @@ func FixTerminatingPods(cluster *rapi.RedisCluster, podControl pod.RedisClusterC
 			// ignore pod without deletion timestamp
 			continue
 		}
+		podLabels := p.GetLabels()
+		if podLabels["redis-operator.k8s.io/marked-for-termination"] == "true" {
+			continue
+		}
 		maxTime := p.DeletionTimestamp.Add(maxDuration) // adding MaxDuration for configuration
 		if maxTime.Before(now) {
 			actionDone = true
 			// it means that this pod should already been deleted since a wild
 			if !dryRun {
+				podLabels["redis-operator.k8s.io/marked-for-termination"] = "true"
+
+				// delete the "cluster-name" label so operator disassociates pods with the redis cluster
+				delete(podLabels, "redis-operator.k8s.io/cluster-name")
+				if err := podControl.SetPodLabels(p, podLabels); err != nil {
+					errs = append(errs, err)
+				}
 				if err := podControl.DeletePod(cluster, p.Name); err != nil {
 					errs = append(errs, err)
 				}

diff --git a/pkg/controller/sanitycheck/untrustednodes_test.go b/pkg/controller/sanitycheck/untrustednodes_test.go
@@ -224,6 +224,11 @@ func (f *Fakecontrol) DeletePodNow(redisCluster *rapi.RedisCluster, podName stri
 	return nil
 }
 
+// SetPodLabels used to set labels for a pod
+func (f *Fakecontrol) SetPodLabels(pod kapiv1.Pod, labels map[string]string) error {
+	return nil
+}
+
 func newPod(name, vmName, ip string) kapiv1.Pod {
 	return kapiv1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: kapiv1.PodSpec{NodeName: vmName}, Status: kapiv1.PodStatus{PodIP: ip}}
 }