From 5c927446154a44ec98fa5cf37a4bf454b60a8a17 Mon Sep 17 00:00:00 2001
From: Darren Shepherd <darren@rancher.com>
Date: Mon, 2 Apr 2018 04:26:22 -0700
Subject: [PATCH] Try three times to remove finalizer

---
 lifecycle/object.go | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/lifecycle/object.go b/lifecycle/object.go
index dae128e5c..725bd7d60 100644
--- a/lifecycle/object.go
+++ b/lifecycle/object.go
@@ -1,6 +1,7 @@
 package lifecycle
 
 import (
+	"fmt"
 	"reflect"
 
 	"github.com/rancher/norman/clientbase"
@@ -92,15 +93,10 @@ func (o *objectLifecycleAdapter) finalize(metadata metav1.Object, obj runtime.Ob
 		copyObj = newObj
 	}
 
-	if err := removeFinalizer(o.constructFinalizerKey(), copyObj); err != nil {
-		return false, err
-	}
-
-	_, err := o.objectClient.Update(metadata.GetName(), copyObj)
-	return false, err
+	return false, o.removeFinalizer(o.constructFinalizerKey(), copyObj)
 }
 
-func removeFinalizer(name string, obj runtime.Object) error {
+func (o *objectLifecycleAdapter) removeFinalizer(name string, obj runtime.Object) error {
 	metadata, err := meta.Accessor(obj)
 	if err != nil {
 		return err
@@ -115,7 +111,26 @@ func removeFinalizer(name string, obj runtime.Object) error {
 	}
 	metadata.SetFinalizers(finalizers)
 
-	return nil
+	for i := 0; i < 3; i++ {
+		_, err := o.objectClient.Update(metadata.GetName(), obj)
+		if err == nil {
+			return nil
+		}
+
+		obj, err = o.objectClient.GetNamespaced(metadata.GetNamespace(), metadata.GetName(), metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		metadata, err := meta.Accessor(obj)
+		if err != nil {
+			return err
+		}
+
+		metadata.SetFinalizers(finalizers)
+	}
+
+	return fmt.Errorf("failed to remove finalizer on %s:%s", metadata.GetNamespace(), metadata.GetName())
 }
 
 func (o *objectLifecycleAdapter) createKey() string {