validate loop

kubernetes · Dec 10, 2019 · 2b954ec · 2b954ec
1 parent dec416f
commit 2b954ec
Showing 1 changed file with 14 additions and 1 deletion.
diff --git a/pkg/instancegroups/instancegroups.go b/pkg/instancegroups/instancegroups.go
@@ -195,7 +195,7 @@ func (r *RollingUpdateInstanceGroup) RollingUpdate(rollingUpdateData *RollingUpd
 		} else if featureflag.DrainAndValidateRollingUpdate.Enabled() {
 			klog.Info("Validating the cluster.")
 
-			if err = r.validateClusterWithDuration(rollingUpdateData, validationTimeout); err != nil {
+			if err = r.validateClusterWithDurationLoop(rollingUpdateData, validationTimeout); err != nil {
 
 				if rollingUpdateData.FailOnValidate {
 					klog.Errorf("Cluster did not validate within %s", validationTimeout)
@@ -248,6 +248,19 @@ func (r *RollingUpdateInstanceGroup) validateClusterWithDuration(rollingUpdateDa
 	}
 }
 
+// validateClusterWithDurationLoop validates cluster 3 times to make sure cluster does not flap
+func (r *RollingUpdateInstanceGroup) validateClusterWithDurationLoop(rollingUpdateData *RollingUpdateCluster, duration time.Duration) error {
+	var err error
+	for i := 1; i <= 3; i++ {
+		err = r.validateClusterWithDuration(rollingUpdateData, duration)
+		if err != nil {
+			return err
+		}
+		time.Sleep(5 * time.Second)
+	}
+	return nil
+}
+
 func (r *RollingUpdateInstanceGroup) tryValidateCluster(rollingUpdateData *RollingUpdateCluster, duration time.Duration, tickDuration time.Duration) bool {
 	result, err := rollingUpdateData.ClusterValidator.Validate()