Implemented support for kubernetes.io/os (#932)

aws · Dec 7, 2021 · effd8d2 · effd8d2 · tuananh · Dec 8, 2021
1 parent e21b84c
commit effd8d2
Show file tree

Hide file tree

Showing 8 changed files with 55 additions and 19 deletions.
diff --git a/pkg/apis/provisioning/v1alpha5/register.go b/pkg/apis/provisioning/v1alpha5/register.go
@@ -59,6 +59,7 @@ var (
 		v1.LabelTopologyZone,
 		v1.LabelInstanceTypeStable,
 		v1.LabelArchStable,
+		v1.LabelOSStable,
 		LabelCapacityType,
 		v1.LabelHostname, // Used internally for hostname topology spread
 	)

diff --git a/pkg/apis/provisioning/v1alpha5/requirements.go b/pkg/apis/provisioning/v1alpha5/requirements.go
@@ -36,6 +36,10 @@ func (r Requirements) Architectures() sets.String {
 	return r.Requirement(v1.LabelArchStable)
 }
 
+func (r Requirements) OperatingSystems() sets.String {
+	return r.Requirement(v1.LabelOSStable)
+}
+
 func (r Requirements) CapacityTypes() sets.String {
 	return r.Requirement(LabelCapacityType)
 }

diff --git a/pkg/cloudprovider/aws/instancetype.go b/pkg/cloudprovider/aws/instancetype.go
@@ -24,6 +24,7 @@ import (
 	"github.com/aws/karpenter/pkg/utils/resources"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/sets"
 )
 
 // EC2VMAvailableMemoryFactor assumes the EC2 VM will consume <7.25% of the memory of a given machine
@@ -42,6 +43,10 @@ func (i *InstanceType) Offerings() []cloudprovider.Offering {
 	return i.AvailableOfferings
 }
 
+func (i *InstanceType) OperatingSystems() sets.String {
+	return sets.NewString("linux")
+}
+
 func (i *InstanceType) Architecture() string {
 	for _, architecture := range i.ProcessorInfo.SupportedArchitectures {
 		if value, ok := v1alpha1.AWSToKubeArchitectures[aws.StringValue(architecture)]; ok {

diff --git a/pkg/cloudprovider/fake/instancetype.go b/pkg/cloudprovider/fake/instancetype.go
@@ -19,6 +19,7 @@ import (
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/sets"
 )
 
 func NewInstanceType(options InstanceTypeOptions) *InstanceType {
@@ -33,6 +34,9 @@ func NewInstanceType(options InstanceTypeOptions) *InstanceType {
 	if len(options.architecture) == 0 {
 		options.architecture = "amd64"
 	}
+	if len(options.operatingSystems) == 0 {
+		options.operatingSystems = sets.NewString("linux", "windows", "mac")
+	}
 	if options.cpu.IsZero() {
 		options.cpu = resource.MustParse("4")
 	}
@@ -44,29 +48,31 @@ func NewInstanceType(options InstanceTypeOptions) *InstanceType {
 	}
 	return &InstanceType{
 		InstanceTypeOptions: InstanceTypeOptions{
-			name:         options.name,
-			offerings:    options.offerings,
-			architecture: options.architecture,
-			cpu:          options.cpu,
-			memory:       options.memory,
-			pods:         options.pods,
-			nvidiaGPUs:   options.nvidiaGPUs,
-			amdGPUs:      options.amdGPUs,
-			awsNeurons:   options.awsNeurons,
+			name:             options.name,
+			offerings:        options.offerings,
+			architecture:     options.architecture,
+			operatingSystems: options.operatingSystems,
+			cpu:              options.cpu,
+			memory:           options.memory,
+			pods:             options.pods,
+			nvidiaGPUs:       options.nvidiaGPUs,
+			amdGPUs:          options.amdGPUs,
+			awsNeurons:       options.awsNeurons,
 		},
 	}
 }
 
 type InstanceTypeOptions struct {
-	name         string
-	offerings    []cloudprovider.Offering
-	architecture string
-	cpu          resource.Quantity
-	memory       resource.Quantity
-	pods         resource.Quantity
-	nvidiaGPUs   resource.Quantity
-	amdGPUs      resource.Quantity
-	awsNeurons   resource.Quantity
+	name             string
+	offerings        []cloudprovider.Offering
+	architecture     string
+	operatingSystems sets.String
+	cpu              resource.Quantity
+	memory           resource.Quantity
+	pods             resource.Quantity
+	nvidiaGPUs       resource.Quantity
+	amdGPUs          resource.Quantity
+	awsNeurons       resource.Quantity
 }
 
 type InstanceType struct {
@@ -85,6 +91,10 @@ func (i *InstanceType) Architecture() string {
 	return i.architecture
 }
 
+func (i *InstanceType) OperatingSystems() sets.String {
+	return i.operatingSystems
+}
+
 func (i *InstanceType) CPU() *resource.Quantity {
 	return &i.cpu
 }

diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go
@@ -20,6 +20,7 @@ import (
 	"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/kubernetes"
 	"knative.dev/pkg/apis"
 )
@@ -55,6 +56,7 @@ type InstanceType interface {
 	// Note that though this is an array it is expected that all the Offerings are unique from one another
 	Offerings() []Offering
 	Architecture() string
+	OperatingSystems() sets.String
 	CPU() *resource.Quantity
 	Memory() *resource.Quantity
 	Pods() *resource.Quantity

diff --git a/pkg/controllers/provisioning/binpacking/packable.go b/pkg/controllers/provisioning/binpacking/packable.go
@@ -53,6 +53,7 @@ func PackablesFor(ctx context.Context, instanceTypes []cloudprovider.InstanceTyp
 			packable.validateZones(constraints),
 			packable.validateInstanceType(constraints),
 			packable.validateArchitecture(constraints),
+			packable.validateOperatingSystems(constraints),
 			packable.validateCapacityTypes(constraints),
 			// Although this will remove instances that have GPUs when
 			// not required, removal of instance types that *lack*
@@ -153,7 +154,7 @@ func (p *Packable) reservePod(pod *v1.Pod) bool {
 
 func (p *Packable) validateInstanceType(constraints *v1alpha5.Constraints) error {
 	if !constraints.Requirements.InstanceTypes().Has(p.Name()) {
-		return fmt.Errorf("instance type %s is not in %v", p.Name(), constraints.Requirements.InstanceTypes().List())
+		return fmt.Errorf("instance type %s not in %v", p.Name(), constraints.Requirements.InstanceTypes().List())
 	}
 	return nil
 }
@@ -165,6 +166,13 @@ func (p *Packable) validateArchitecture(constraints *v1alpha5.Constraints) error
 	return nil
 }
 
+func (p *Packable) validateOperatingSystems(constraints *v1alpha5.Constraints) error {
+	if constraints.Requirements.OperatingSystems().Intersection(p.OperatingSystems()).Len() == 0 {
+		return fmt.Errorf("operating systems %s not in %v", p.OperatingSystems(), constraints.Requirements.OperatingSystems().List())
+	}
+	return nil
+}
+
 func (p *Packable) validateZones(constraints *v1alpha5.Constraints) error {
 	zones := sets.String{}
 	for _, offering := range p.Offerings() {

diff --git a/pkg/controllers/provisioning/controller.go b/pkg/controllers/provisioning/controller.go
@@ -140,6 +140,7 @@ func requirements(instanceTypes []cloudprovider.InstanceType) (requirements v1al
 		v1.LabelInstanceTypeStable: sets.NewString(),
 		v1.LabelTopologyZone:       sets.NewString(),
 		v1.LabelArchStable:         sets.NewString(),
+		v1.LabelOSStable:           sets.NewString(),
 		v1alpha5.LabelCapacityType: sets.NewString(),
 	}
 	for _, instanceType := range instanceTypes {
@@ -149,6 +150,7 @@ func requirements(instanceTypes []cloudprovider.InstanceType) (requirements v1al
 		}
 		supported[v1.LabelInstanceTypeStable].Insert(instanceType.Name())
 		supported[v1.LabelArchStable].Insert(instanceType.Architecture())
+		supported[v1.LabelOSStable].Insert(instanceType.OperatingSystems().List()...)
 	}
 	for key, values := range supported {
 		requirements = append(requirements, v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: values.UnsortedList()})

diff --git a/pkg/controllers/provisioning/suite_test.go b/pkg/controllers/provisioning/suite_test.go
@@ -104,6 +104,8 @@ var _ = Describe("Provisioning", func() {
 				test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}}),
 				// Constrained by architecture
 				test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}),
+				// Constrained by operatingSystem
+				test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: "linux"}}),
 			}
 			unschedulable := []*v1.Pod{
 				// Ignored, matches another provisioner
@@ -114,6 +116,8 @@ var _ = Describe("Provisioning", func() {
 				test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "unknown"}}),
 				// Ignored, invalid architecture
 				test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "unknown"}}),
+				// Ignored, invalid operating system
+				test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: "unknown"}}),
 				// Ignored, invalid capacity type
 				test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.LabelCapacityType: "unknown"}}),
 				// Ignored, label selector does not match