clusterapi: refresh kubeconfig bearer tokens for management and workl…

…oad kubeconfigs dynamically

cluster-autoscaler/cloudprovider/clusterapi/README.md

@@ -165,6 +165,10 @@ cluster-autoscaler --cloud-provider=clusterapi \
                    --kubeconfig=/mnt/workload.kubeconfig
 ```
 
+### A note on kubeconfig credentials
+
+When the cluster autoscaler is configured to read kubeconfig(s) from a path, by passing a `--kubeconfig` or `--cloud-config` option, the autoscaler will re-read the contents of the kubeconfig file after it detects the file was modified by it's modtime. This was added to support EKS clusters with the Cluster API Provider for AWS, where CAPA would provide the cluster with a kubernetes secret resource populated with a short-lived EKS kubeconfig credential, but may be useful for other Cluster API providers as well. In the past, it was not possible to use the Cluster API's kubeconfig credential directly as a secret volume, because when the short-lived contents were refreshed by the CAPA controller, the cluster autoscaler would not re-read the secret mount. There were various work arounds, for example manually creating a longer-lived kubeconfig from a service account. This is no longer necessary, cluster autoscaler will automatically refresh the kubeconfig credentials as needed.
+
 ## Enabling Autoscaling
 
 To enable the automatic scaling of components in your cluster-api managed

cluster-autoscaler/cloudprovider/clusterapi/clusterapi_provider.go

@@ -17,6 +17,7 @@ limitations under the License.
 package clusterapi
 
 import (
+	"net/http"
 	"reflect"
 
 	corev1 "k8s.io/api/core/v1"
@@ -158,13 +159,22 @@ func BuildClusterAPI(opts config.AutoscalingOptions, do cloudprovider.NodeGroupD
 	if err != nil {
 		klog.Fatalf("cannot build management cluster config: %v", err)
 	}
+	if managementConfig.BearerToken != "" && !opts.ClusterAPICloudConfigAuthoritative {
+		managementConfig.Wrap(func(rt http.RoundTripper) http.RoundTripper {
+			return config.NewDynamicKubeConfigRoundTripper(managementKubeconfig, rt)
+		})
+	}
 
 	workloadKubeconfig := opts.KubeConfigPath
-
 	workloadConfig, err := clientcmd.BuildConfigFromFlags("", workloadKubeconfig)
 	if err != nil {
 		klog.Fatalf("cannot build workload cluster config: %v", err)
 	}
+	if workloadConfig.BearerToken != "" && workloadKubeconfig != "" {
+		workloadConfig.Wrap(func(rt http.RoundTripper) http.RoundTripper {
+			return config.NewDynamicKubeConfigRoundTripper(workloadKubeconfig, rt)
+		})
+	}
 
 	// Grab a dynamic interface that we can create informers from
 	managementClient, err := dynamic.NewForConfig(managementConfig)

cluster-autoscaler/config/kubernetes.go

@@ -19,9 +19,13 @@ package config
 import (
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"net/url"
+	"os"
 	"strconv"
+	"time"
 
+	utilnet "k8s.io/apimachinery/pkg/util/net"
 	kube_rest "k8s.io/client-go/rest"
 	kube_client_cmd "k8s.io/client-go/tools/clientcmd"
 )
@@ -113,3 +117,39 @@ func GetKubeClientConfig(uri *url.URL) (*kube_rest.Config, error) {
 
 	return kubeConfig, nil
 }
+
+// NewDynamicKubeConfigRoundTripper wraps the http.RoundTripper to provide the latest bearer token from the kube config on disk
+func NewDynamicKubeConfigRoundTripper(kubeconfigPath string, rt http.RoundTripper) http.RoundTripper {
+	return &bearerAuthRoundTripper{
+		kubeconfigPath: kubeconfigPath,
+		rt:             rt,
+	}
+}
+
+type bearerAuthRoundTripper struct {
+	kubeconfigPath string
+	rt             http.RoundTripper
+
+	lastModified time.Time
+	bearerToken  string
+}
+
+func (rt *bearerAuthRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	stat, err := os.Stat(rt.kubeconfigPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to stat kubeconfig path: %w", err)
+	}
+	if stat.ModTime().After(rt.lastModified) {
+		kubeConfig, err := kube_client_cmd.BuildConfigFromFlags("", rt.kubeconfigPath)
+		if err != nil {
+			return nil, fmt.Errorf("cannot build kube cluster config: %w", err)
+		}
+		rt.bearerToken = kubeConfig.BearerToken
+		rt.lastModified = stat.ModTime()
+	}
+	req = utilnet.CloneRequest(req)
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", rt.bearerToken))
+	return rt.rt.RoundTrip(req)
+}
+
+var _ http.RoundTripper = &bearerAuthRoundTripper{}

cluster-autoscaler/main.go

@@ -373,11 +373,14 @@ func getKubeConfig() *rest.Config {
 	if *kubeConfigFile != "" {
 		klog.V(1).Infof("Using kubeconfig file: %s", *kubeConfigFile)
 		// use the current context in kubeconfig
-		config, err := clientcmd.BuildConfigFromFlags("", *kubeConfigFile)
+		kubeConfig, err := clientcmd.BuildConfigFromFlags("", *kubeConfigFile)
 		if err != nil {
 			klog.Fatalf("Failed to build config: %v", err)
 		}
-		return config
+		kubeConfig.Wrap(func(rt http.RoundTripper) http.RoundTripper {
+			return config.NewDynamicKubeConfigRoundTripper(*kubeConfigFile, rt)
+		})
+		return kubeConfig
 	}
 	url, err := url.Parse(*kubernetes)
 	if err != nil {