Merge pull request #6531 from BigDarkClown/kube-env

GCE: Implement kube-env caching

cluster-autoscaler/cloudprovider/gce/cache.go

@@ -68,6 +68,7 @@ type GceCache struct {
 	migBaseNameCache          map[GceRef]string
 	instanceTemplateNameCache map[GceRef]string
 	instanceTemplatesCache    map[GceRef]*gce.InstanceTemplate
+	kubeEnvCache              map[GceRef]KubeEnv
 }
 
 // NewGceCache creates empty GceCache.
@@ -84,6 +85,7 @@ func NewGceCache() *GceCache {
 		migBaseNameCache:          map[GceRef]string{},
 		instanceTemplateNameCache: map[GceRef]string{},
 		instanceTemplatesCache:    map[GceRef]*gce.InstanceTemplate{},
+		kubeEnvCache:              map[GceRef]KubeEnv{},
 	}
 }
 
@@ -409,6 +411,46 @@ func (gc *GceCache) InvalidateAllMigInstanceTemplates() {
 	gc.instanceTemplatesCache = map[GceRef]*gce.InstanceTemplate{}
 }
 
+// GetMigKubeEnv returns the cached KubeEnv for a mig GceRef
+func (gc *GceCache) GetMigKubeEnv(ref GceRef) (KubeEnv, bool) {
+	gc.cacheMutex.Lock()
+	defer gc.cacheMutex.Unlock()
+
+	kubeEnv, found := gc.kubeEnvCache[ref]
+	if found {
+		klog.V(5).Infof("Kube-env cache hit for %s", ref)
+	}
+	return kubeEnv, found
+}
+
+// SetMigKubeEnv sets KubeEnv for a mig GceRef
+func (gc *GceCache) SetMigKubeEnv(ref GceRef, kubeEnv KubeEnv) {
+	gc.cacheMutex.Lock()
+	defer gc.cacheMutex.Unlock()
+
+	gc.kubeEnvCache[ref] = kubeEnv
+}
+
+// InvalidateMigKubeEnv clears the kube-env cache for a mig GceRef
+func (gc *GceCache) InvalidateMigKubeEnv(ref GceRef) {
+	gc.cacheMutex.Lock()
+	defer gc.cacheMutex.Unlock()
+
+	if _, found := gc.kubeEnvCache[ref]; found {
+		klog.V(5).Infof("Kube-env cache invalidated for %s", ref)
+		delete(gc.kubeEnvCache, ref)
+	}
+}
+
+// InvalidateAllMigKubeEnvs clears the kube-env cache
+func (gc *GceCache) InvalidateAllMigKubeEnvs() {
+	gc.cacheMutex.Lock()
+	defer gc.cacheMutex.Unlock()
+
+	klog.V(5).Infof("Kube-env cache invalidated")
+	gc.kubeEnvCache = map[GceRef]KubeEnv{}
+}
+
 // GetMachine retrieves machine type from cache under lock.
 func (gc *GceCache) GetMachine(machineTypeName string, zone string) (MachineType, bool) {
 	gc.cacheMutex.Lock()

cluster-autoscaler/cloudprovider/gce/gce_manager.go

@@ -345,12 +345,12 @@ func (m *gceManagerImpl) refreshAutoscalingOptions() {
 			klog.Warningf("Failed to extract autoscaling options from %q metadata: instance template is incomplete", template.Name)
 			continue
 		}
-		kubeEnvValue, err := getKubeEnvValueFromTemplateMetadata(template)
+		kubeEnv, err := m.migInfoProvider.GetMigKubeEnv(mig.GceRef())
 		if err != nil {
 			klog.Warningf("Failed to extract autoscaling options from %q instance template's metadata: can't get KubeEnv: %v", template.Name, err)
 			continue
 		}
-		options, err := extractAutoscalingOptionsFromKubeEnv(kubeEnvValue)
+		options, err := extractAutoscalingOptionsFromKubeEnv(kubeEnv)
 		if err != nil {
 			klog.Warningf("Failed to extract autoscaling options from %q instance template's metadata: %v", template.Name, err)
 			continue
@@ -591,15 +591,19 @@ func (m *gceManagerImpl) GetMigTemplateNode(mig Mig) (*apiv1.Node, error) {
 	if err != nil {
 		return nil, err
 	}
+	kubeEnv, err := m.migInfoProvider.GetMigKubeEnv(mig.GceRef())
+	if err != nil {
+		return nil, err
+	}
 	machineType, err := m.migInfoProvider.GetMigMachineType(mig.GceRef())
 	if err != nil {
 		return nil, err
 	}
-	migOsInfo, err := m.templates.MigOsInfo(mig.Id(), template)
+	migOsInfo, err := m.templates.MigOsInfo(mig.Id(), kubeEnv)
 	if err != nil {
 		return nil, err
 	}
-	return m.templates.BuildNodeFromTemplate(mig, migOsInfo, template, machineType.CPU, machineType.Memory, nil, m.reserved)
+	return m.templates.BuildNodeFromTemplate(mig, migOsInfo, template, kubeEnv, machineType.CPU, machineType.Memory, nil, m.reserved)
 }
 
 // parseMIGAutoDiscoverySpecs returns any provided NodeGroupAutoDiscoverySpecs

cluster-autoscaler/cloudprovider/gce/gce_manager_test.go

@@ -346,6 +346,7 @@ func newTestGceManager(t *testing.T, testServerURL string, regional bool) *gceMa
 		migTargetSizeCache:        map[GceRef]int64{},
 		instanceTemplateNameCache: map[GceRef]string{},
 		instanceTemplatesCache:    map[GceRef]*gce.InstanceTemplate{},
+		kubeEnvCache:              map[GceRef]KubeEnv{},
 		migBaseNameCache:          map[GceRef]string{},
 	}
 	migLister := NewMigLister(cache)

cluster-autoscaler/cloudprovider/gce/kube_env.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"errors"
+	"fmt"
+
+	gce "google.golang.org/api/compute/v1"
+	"sigs.k8s.io/yaml"
+)
+
+const (
+	kubeEnvKey = "kube-env"
+)
+
+// KubeEnv stores kube-env information from InstanceTemplate
+type KubeEnv struct {
+	templateName string
+	env          map[string]string
+}
+
+// ExtractKubeEnv extracts kube-env from InstanceTemplate
+func ExtractKubeEnv(template *gce.InstanceTemplate) (KubeEnv, error) {
+	if template == nil {
+		return KubeEnv{}, errors.New("instance template is nil")
+	}
+	if template.Properties == nil || template.Properties.Metadata == nil {
+		return KubeEnv{}, fmt.Errorf("instance template %s has no metadata", template.Name)
+	}
+	for _, item := range template.Properties.Metadata.Items {
+		if item.Key == kubeEnvKey {
+			if item.Value == nil {
+				return KubeEnv{}, fmt.Errorf("no kube-env content in metadata")
+			}
+			return ParseKubeEnv(template.Name, *item.Value)
+		}
+	}
+	return KubeEnv{templateName: template.Name}, nil
+}
+
+// ParseKubeEnv parses kube-env from its string representation
+func ParseKubeEnv(templateName, kubeEnvValue string) (KubeEnv, error) {
+	env := make(map[string]string)
+	err := yaml.Unmarshal([]byte(kubeEnvValue), &env)
+	if err != nil {
+		return KubeEnv{}, fmt.Errorf("error unmarshalling kubeEnv: %v", err)
+	}
+	return KubeEnv{templateName: templateName, env: env}, nil
+}
+
+// Var extracts variable from KubeEnv
+func (ke KubeEnv) Var(name string) (string, bool) {
+	if ke.env == nil {
+		return "", false
+	}
+	val, found := ke.env[name]
+	return val, found
+}