Merge pull request #6607 from vbhargav875/oci-node-ephemeral-storage

Support scale up of nodepool with zero nodes with ephemeral storage r…

cluster-autoscaler/cloudprovider/oci/common/oci_shape.go

@@ -19,7 +19,7 @@ import (
 
 // ShapeGetter returns the oci shape attributes for the pool.
 type ShapeGetter interface {
-	GetNodePoolShape(*oke.NodePool) (*Shape, error)
+	GetNodePoolShape(*oke.NodePool, int64) (*Shape, error)
 	GetInstancePoolShape(pool *core.InstancePool) (*Shape, error)
 	Refresh()
 }
@@ -51,10 +51,11 @@ func (cc ShapeClientImpl) ListShapes(ctx context.Context, req core.ListShapesReq
 // Shape includes the resource attributes of a given shape which should be used
 // for constructing node templates.
 type Shape struct {
-	Name          string
-	CPU           float32
-	GPU           int
-	MemoryInBytes float32
+	Name                    string
+	CPU                     float32
+	GPU                     int
+	MemoryInBytes           float32
+	EphemeralStorageInBytes float32
 }
 
 // CreateShapeGetter creates a new oci shape getter.
@@ -78,14 +79,15 @@ func (osf *shapeGetterImpl) Refresh() {
 }
 
 // GetNodePoolShape gets the shape by querying the node pool's configuration
-func (osf *shapeGetterImpl) GetNodePoolShape(np *oke.NodePool) (*Shape, error) {
+func (osf *shapeGetterImpl) GetNodePoolShape(np *oke.NodePool, ephemeralStorage int64) (*Shape, error) {
 	shapeName := *np.NodeShape
 	if np.NodeShapeConfig != nil {
 		return &Shape{
 			CPU: *np.NodeShapeConfig.Ocpus * 2,
 			// num_bytes * kilo * mega * giga
-			MemoryInBytes: *np.NodeShapeConfig.MemoryInGBs * 1024 * 1024 * 1024,
-			GPU:           0,
+			MemoryInBytes:           *np.NodeShapeConfig.MemoryInGBs * 1024 * 1024 * 1024,
+			GPU:                     0,
+			EphemeralStorageInBytes: float32(ephemeralStorage),
 		}, nil
 	}
 
@@ -113,9 +115,10 @@ func (osf *shapeGetterImpl) GetNodePoolShape(np *oke.NodePool) (*Shape, error) {
 	// Update the cache based on latest results
 	for _, s := range resp.Items {
 		osf.cache[*s.Shape] = &Shape{
-			CPU:           getFloat32(s.Ocpus) * 2, // convert ocpu to vcpu
-			GPU:           getInt(s.Gpus),
-			MemoryInBytes: getFloat32(s.MemoryInGBs) * 1024 * 1024 * 1024,
+			CPU:                     getFloat32(s.Ocpus) * 2, // convert ocpu to vcpu
+			GPU:                     getInt(s.Gpus),
+			MemoryInBytes:           getFloat32(s.MemoryInGBs) * 1024 * 1024 * 1024,
+			EphemeralStorageInBytes: float32(ephemeralStorage),
 		}
 	}
 

cluster-autoscaler/cloudprovider/oci/common/oci_shape_test.go

@@ -95,9 +95,10 @@ func TestNodePoolGetShape(t *testing.T) {
 		"basic shape": {
 			shape: "VM.Standard1.2",
 			expected: &Shape{
-				CPU:           4,
-				MemoryInBytes: 16 * 1024 * 1024 * 1024,
-				GPU:           0,
+				CPU:                     4,
+				MemoryInBytes:           16 * 1024 * 1024 * 1024,
+				GPU:                     0,
+				EphemeralStorageInBytes: -1,
 			},
 		},
 		"flex shape": {
@@ -107,9 +108,10 @@ func TestNodePoolGetShape(t *testing.T) {
 				MemoryInGBs: common.Float32(64),
 			},
 			expected: &Shape{
-				CPU:           8,
-				MemoryInBytes: 4 * 16 * 1024 * 1024 * 1024,
-				GPU:           0,
+				CPU:                     8,
+				MemoryInBytes:           4 * 16 * 1024 * 1024 * 1024,
+				GPU:                     0,
+				EphemeralStorageInBytes: -1,
 			},
 		},
 	}
@@ -118,7 +120,7 @@ func TestNodePoolGetShape(t *testing.T) {
 		shapeGetter := CreateShapeGetter(shapeClient)
 
 		t.Run(name, func(t *testing.T) {
-			shape, err := shapeGetter.GetNodePoolShape(&oke.NodePool{NodeShape: &tc.shape, NodeShapeConfig: tc.shapeConfig})
+			shape, err := shapeGetter.GetNodePoolShape(&oke.NodePool{NodeShape: &tc.shape, NodeShapeConfig: tc.shapeConfig}, -1)
 			if err != nil {
 				t.Fatal(err)
 			}

cluster-autoscaler/cloudprovider/oci/common/oci_tag.go

@@ -0,0 +1,27 @@
+/*
+Copyright 2021-2024 Oracle and/or its affiliates.
+*/
+
+package common
+
+import (
+	oke "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/oci/vendor-internal/github.com/oracle/oci-go-sdk/v65/containerengine"
+)
+
+// TagsGetter returns the oci tags for the pool.
+type TagsGetter interface {
+	GetNodePoolFreeformTags(*oke.NodePool) (map[string]string, error)
+}
+
+// TagsGetterImpl is the implementation to fetch the oci tags for the pool.
+type TagsGetterImpl struct{}
+
+// CreateTagsGetter creates a new oci tags getter.
+func CreateTagsGetter() TagsGetter {
+	return &TagsGetterImpl{}
+}
+
+// GetNodePoolFreeformTags returns the FreeformTags for the nodepool
+func (tgi *TagsGetterImpl) GetNodePoolFreeformTags(np *oke.NodePool) (map[string]string, error) {
+	return np.FreeformTags, nil
+}

cluster-autoscaler/cloudprovider/oci/nodepools/consts/annotations.go

@@ -25,4 +25,7 @@ const (
 	// ToBeDeletedByClusterAutoscaler is the taint used to ensure that after a node has been called to be deleted
 	// no more pods will schedule onto it
 	ToBeDeletedByClusterAutoscaler = "ignore-taint.cluster-autoscaler.kubernetes.io/oke-impending-node-termination"
+
+	// EphemeralStorageSize is the freeform tag key that would be used to determine the ephemeral-storage size of the node
+	EphemeralStorageSize = "cluster-autoscaler/node-ephemeral-storage"
 )

cluster-autoscaler/cloudprovider/oci/nodepools/oci_manager.go

@@ -136,6 +136,7 @@ func CreateNodePoolManager(cloudConfigPath string, discoveryOpts cloudprovider.N
 
 	//ociShapeGetter := ocicommon.CreateShapeGetter(computeClient)
 	ociShapeGetter := ocicommon.CreateShapeGetter(ocicommon.ShapeClientImpl{ComputeMgmtClient: computeMgmtClient, ComputeClient: computeClient})
+	ociTagsGetter := ocicommon.CreateTagsGetter()
 
 	registeredTaintsGetter := CreateRegisteredTaintsGetter()
 
@@ -145,6 +146,7 @@ func CreateNodePoolManager(cloudConfigPath string, discoveryOpts cloudprovider.N
 		computeClient:          &computeClient,
 		staticNodePools:        map[string]NodePool{},
 		ociShapeGetter:         ociShapeGetter,
+		ociTagsGetter:          ociTagsGetter,
 		registeredTaintsGetter: registeredTaintsGetter,
 		nodePoolCache:          newNodePoolCache(&okeClient),
 	}
@@ -210,6 +212,7 @@ type ociManagerImpl struct {
 	okeClient              okeClient
 	computeClient          *core.ComputeClient
 	ociShapeGetter         ocicommon.ShapeGetter
+	ociTagsGetter          ocicommon.TagsGetter
 	registeredTaintsGetter RegisteredTaintsGetter
 	staticNodePools        map[string]NodePool
 
@@ -521,7 +524,15 @@ func (m *ociManagerImpl) buildNodeFromTemplate(nodePool *oke.NodePool) (*apiv1.N
 		Capacity: apiv1.ResourceList{},
 	}
 
-	shape, err := m.ociShapeGetter.GetNodePoolShape(nodePool)
+	freeformTags, err := m.ociTagsGetter.GetNodePoolFreeformTags(nodePool)
+	if err != nil {
+		return nil, err
+	}
+	ephemeralStorage, err := getEphemeralResourceRequestsInBytes(freeformTags)
+	if err != nil {
+		klog.Error(err)
+	}
+	shape, err := m.ociShapeGetter.GetNodePoolShape(nodePool, ephemeralStorage)
 	if err != nil {
 		return nil, err
 	}
@@ -542,11 +553,16 @@ func (m *ociManagerImpl) buildNodeFromTemplate(nodePool *oke.NodePool) (*apiv1.N
 		})
 	}
 
+	if err != nil {
+		return nil, err
+	}
 	node.Status.Capacity[apiv1.ResourcePods] = *resource.NewQuantity(110, resource.DecimalSI)
-
 	node.Status.Capacity[apiv1.ResourceCPU] = *resource.NewQuantity(int64(shape.CPU), resource.DecimalSI)
 	node.Status.Capacity[apiv1.ResourceMemory] = *resource.NewQuantity(int64(shape.MemoryInBytes), resource.DecimalSI)
 	node.Status.Capacity[ipconsts.ResourceGPU] = *resource.NewQuantity(int64(shape.GPU), resource.DecimalSI)
+	if ephemeralStorage != -1 {
+		node.Status.Capacity[apiv1.ResourceEphemeralStorage] = *resource.NewQuantity(ephemeralStorage, resource.DecimalSI)
+	}
 
 	node.Status.Allocatable = node.Status.Capacity
 
@@ -634,6 +650,23 @@ func addTaintToSpec(node *apiv1.Node, taintKey string, effect apiv1.TaintEffect)
 	return true
 }
 
+func getEphemeralResourceRequestsInBytes(tags map[string]string) (int64, error) {
+	for key, value := range tags {
+		if key == npconsts.EphemeralStorageSize {
+			klog.V(4).Infof("ephemeral-storage size set with value : %v", value)
+			value = strings.ReplaceAll(value, " ", "")
+			resourceSize, err := resource.ParseQuantity(value)
+			if err != nil {
+				return -1, err
+			}
+			klog.V(4).Infof("ephemeral-storage size = %v (%v)", resourceSize.Value(), resourceSize.Format)
+			return resourceSize.Value(), nil
+		}
+	}
+	klog.V(4).Infof("ephemeral-storage size not set as part of the nodepool's freeform tags")
+	return -1, nil
+}
+
 // IsConflict checks if the error is a conflict
 func IsConflict(err error) bool {
 	return ReasonForError(err) == metav1.StatusReasonConflict