Merge pull request #439 from zhengxiexie/retry_clean_main

Add retry to cleanup NSX resources(#386)

cmd_clean/main.go

@@ -104,6 +104,12 @@ func main() {
 	} else {
 		err = clean.Clean(cf, nil)
 	}
+	// the error roughly are:
+	// 1. failed to validate config
+	// 2. failed to get nsx client
+	// 3. failed to initialize cleanup service
+	// 4. failed to clean up specific resource
+	err = clean.Clean(cf, nil)
 	if err != nil {
 		log.Error(err, "failed to clean nsx resources")
 		os.Exit(1)

pkg/clean/clean.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"net/http"
 
+	"k8s.io/client-go/util/retry"
+
 	"github.com/vmware-tanzu/nsx-operator/pkg/config"
 	commonctl "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common"
 	"github.com/vmware-tanzu/nsx-operator/pkg/logger"
@@ -32,13 +34,24 @@ func Clean(cf *config.NSXOperatorConfig, client *http.Client) error {
 		return fmt.Errorf("failed to validate config: %w", err)
 	}
 	if cleanupService, err := InitializeCleanupService(cf, client); err != nil {
-		return fmt.Errorf("failed to initialize cleanup service: %w", err)
+		return err // failed to get nsx client
 	} else if cleanupService.err != nil {
 		return fmt.Errorf("failed to initialize cleanup service: %w", cleanupService.err)
 	} else {
 		for _, clean := range cleanupService.cleans {
-			if err := clean.Cleanup(); err != nil {
-				return fmt.Errorf("failed to clean up: %w", err)
+			if err := retry.OnError(retry.DefaultRetry, func(err error) bool {
+				if err != nil {
+					log.Info("retrying to clean up NSX resources", "error", err)
+					return true
+				}
+				return false
+			}, func() error {
+				if err := clean.Cleanup(); err != nil {
+					return fmt.Errorf("failed to clean up specific resource: %w", err)
+				}
+				return nil
+			}); err != nil {
+				return err
 			}
 		}
 	}