Reload bootstrap kubeconfig if cert mgr failed to load valid certs

When user recreate whole cluster certs, multus thick plugin's
previous cert is no longer valid. In such case, we need to prevent
to use cert manager's old certs and restart it from bootstrap
kubeconfig. This fix reloads client config from bootstrap
kubeconfig if cert mgr's cert is failed to load pod.

pkg/k8sclient/kubeconfig.go

@@ -22,10 +22,12 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"strings"
 	"time"
 
 	certificatesv1 "k8s.io/api/certificates/v1"
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -87,8 +89,28 @@ func PerNodeK8sClient(nodeName, bootstrapKubeconfigFile string, certDuration tim
 	// https://github.com/kubernetes/kubernetes/blob/068ee321bc7bfe1c2cefb87fb4d9e5deea84fbc8/cmd/kubelet/app/server.go#L953-L963
 	newClientsetFn := func(current *tls.Certificate) (kubernetes.Interface, error) {
 		cfg := bootstrapKubeconfig
-		if current != nil {
-			cfg = config
+
+		// validate the kubeconfig
+		tempClient, err := kubernetes.NewForConfig(cfg)
+		if err != nil {
+			logging.Errorf("failed to read kubeconfig from cert manager: %v", err)
+		} else {
+			_, err := tempClient.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{})
+			// tls unknown authority error is unrecoverable error with retry
+			if err != nil {
+				if strings.Contains(err.Error(), "x509: certificate signed by unknown authority") {
+					logging.Verbosef("cert mgr gets invalid config. rebuild from bootstrap kubeconfig")
+					// reload and use bootstrapKubeconfig again
+					newBootstrapKubeconfig, _ := clientcmd.BuildConfigFromFlags("", bootstrapKubeconfigFile)
+					cfg = newBootstrapKubeconfig
+				} else {
+					logging.Errorf("failed to list pods with new certs: %v", err)
+				}
+			}
+
+			if current != nil {
+				cfg = config
+			}
 		}
 		return kubernetes.NewForConfig(cfg)
 	}