diff --git a/pkg/apis/provisioning/v1alpha5/register.go b/pkg/apis/provisioning/v1alpha5/register.go index 18fa368a193a..fd4981a9e370 100644 --- a/pkg/apis/provisioning/v1alpha5/register.go +++ b/pkg/apis/provisioning/v1alpha5/register.go @@ -59,6 +59,7 @@ var ( v1.LabelTopologyZone, v1.LabelInstanceTypeStable, v1.LabelArchStable, + v1.LabelOSStable, LabelCapacityType, v1.LabelHostname, // Used internally for hostname topology spread ) diff --git a/pkg/apis/provisioning/v1alpha5/requirements.go b/pkg/apis/provisioning/v1alpha5/requirements.go index d8884dd18a08..7c48b47c86ad 100644 --- a/pkg/apis/provisioning/v1alpha5/requirements.go +++ b/pkg/apis/provisioning/v1alpha5/requirements.go @@ -36,6 +36,10 @@ func (r Requirements) Architectures() sets.String { return r.Requirement(v1.LabelArchStable) } +func (r Requirements) OperatingSystems() sets.String { + return r.Requirement(v1.LabelOSStable) +} + func (r Requirements) CapacityTypes() sets.String { return r.Requirement(LabelCapacityType) } diff --git a/pkg/cloudprovider/aws/instancetype.go b/pkg/cloudprovider/aws/instancetype.go index 1c7c2cfb381c..e824cedcea72 100644 --- a/pkg/cloudprovider/aws/instancetype.go +++ b/pkg/cloudprovider/aws/instancetype.go @@ -24,6 +24,7 @@ import ( "github.com/aws/karpenter/pkg/utils/resources" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/sets" ) // EC2VMAvailableMemoryFactor assumes the EC2 VM will consume <7.25% of the memory of a given machine @@ -42,6 +43,10 @@ func (i *InstanceType) Offerings() []cloudprovider.Offering { return i.AvailableOfferings } +func (i *InstanceType) OperatingSystems() sets.String { + return sets.NewString("linux") +} + func (i *InstanceType) Architecture() string { for _, architecture := range i.ProcessorInfo.SupportedArchitectures { if value, ok := v1alpha1.AWSToKubeArchitectures[aws.StringValue(architecture)]; ok { diff --git a/pkg/cloudprovider/fake/instancetype.go b/pkg/cloudprovider/fake/instancetype.go index 640eedb1e3e4..2050ed1a1e5e 100644 --- a/pkg/cloudprovider/fake/instancetype.go +++ b/pkg/cloudprovider/fake/instancetype.go @@ -19,6 +19,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/sets" ) func NewInstanceType(options InstanceTypeOptions) *InstanceType { @@ -33,6 +34,9 @@ func NewInstanceType(options InstanceTypeOptions) *InstanceType { if len(options.architecture) == 0 { options.architecture = "amd64" } + if len(options.operatingSystems) == 0 { + options.operatingSystems = sets.NewString("linux", "windows", "mac") + } if options.cpu.IsZero() { options.cpu = resource.MustParse("4") } @@ -44,29 +48,31 @@ func NewInstanceType(options InstanceTypeOptions) *InstanceType { } return &InstanceType{ InstanceTypeOptions: InstanceTypeOptions{ - name: options.name, - offerings: options.offerings, - architecture: options.architecture, - cpu: options.cpu, - memory: options.memory, - pods: options.pods, - nvidiaGPUs: options.nvidiaGPUs, - amdGPUs: options.amdGPUs, - awsNeurons: options.awsNeurons, + name: options.name, + offerings: options.offerings, + architecture: options.architecture, + operatingSystems: options.operatingSystems, + cpu: options.cpu, + memory: options.memory, + pods: options.pods, + nvidiaGPUs: options.nvidiaGPUs, + amdGPUs: options.amdGPUs, + awsNeurons: options.awsNeurons, }, } } type InstanceTypeOptions struct { - name string - offerings []cloudprovider.Offering - architecture string - cpu resource.Quantity - memory resource.Quantity - pods resource.Quantity - nvidiaGPUs resource.Quantity - amdGPUs resource.Quantity - awsNeurons resource.Quantity + name string + offerings []cloudprovider.Offering + architecture string + operatingSystems sets.String + cpu resource.Quantity + memory resource.Quantity + pods resource.Quantity + nvidiaGPUs resource.Quantity + amdGPUs resource.Quantity + awsNeurons resource.Quantity } type InstanceType struct { @@ -85,6 +91,10 @@ func (i *InstanceType) Architecture() string { return i.architecture } +func (i *InstanceType) OperatingSystems() sets.String { + return i.operatingSystems +} + func (i *InstanceType) CPU() *resource.Quantity { return &i.cpu } diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index 5c62a7ac5d4d..ff1615cb6dd6 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -20,6 +20,7 @@ import ( "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" "knative.dev/pkg/apis" ) @@ -55,6 +56,7 @@ type InstanceType interface { // Note that though this is an array it is expected that all the Offerings are unique from one another Offerings() []Offering Architecture() string + OperatingSystems() sets.String CPU() *resource.Quantity Memory() *resource.Quantity Pods() *resource.Quantity diff --git a/pkg/controllers/provisioning/binpacking/packable.go b/pkg/controllers/provisioning/binpacking/packable.go index 394aaf3da3a0..ff5bb89e541b 100644 --- a/pkg/controllers/provisioning/binpacking/packable.go +++ b/pkg/controllers/provisioning/binpacking/packable.go @@ -53,6 +53,7 @@ func PackablesFor(ctx context.Context, instanceTypes []cloudprovider.InstanceTyp packable.validateZones(constraints), packable.validateInstanceType(constraints), packable.validateArchitecture(constraints), + packable.validateOperatingSystems(constraints), packable.validateCapacityTypes(constraints), // Although this will remove instances that have GPUs when // not required, removal of instance types that *lack* @@ -153,7 +154,7 @@ func (p *Packable) reservePod(pod *v1.Pod) bool { func (p *Packable) validateInstanceType(constraints *v1alpha5.Constraints) error { if !constraints.Requirements.InstanceTypes().Has(p.Name()) { - return fmt.Errorf("instance type %s is not in %v", p.Name(), constraints.Requirements.InstanceTypes().List()) + return fmt.Errorf("instance type %s not in %v", p.Name(), constraints.Requirements.InstanceTypes().List()) } return nil } @@ -165,6 +166,13 @@ func (p *Packable) validateArchitecture(constraints *v1alpha5.Constraints) error return nil } +func (p *Packable) validateOperatingSystems(constraints *v1alpha5.Constraints) error { + if constraints.Requirements.OperatingSystems().Intersection(p.OperatingSystems()).Len() == 0 { + return fmt.Errorf("operating systems %s not in %v", p.OperatingSystems(), constraints.Requirements.OperatingSystems().List()) + } + return nil +} + func (p *Packable) validateZones(constraints *v1alpha5.Constraints) error { zones := sets.String{} for _, offering := range p.Offerings() { diff --git a/pkg/controllers/provisioning/controller.go b/pkg/controllers/provisioning/controller.go index 08657bfbf0fc..b8c1d3663fbc 100644 --- a/pkg/controllers/provisioning/controller.go +++ b/pkg/controllers/provisioning/controller.go @@ -140,6 +140,7 @@ func requirements(instanceTypes []cloudprovider.InstanceType) (requirements v1al v1.LabelInstanceTypeStable: sets.NewString(), v1.LabelTopologyZone: sets.NewString(), v1.LabelArchStable: sets.NewString(), + v1.LabelOSStable: sets.NewString(), v1alpha5.LabelCapacityType: sets.NewString(), } for _, instanceType := range instanceTypes { @@ -149,6 +150,7 @@ func requirements(instanceTypes []cloudprovider.InstanceType) (requirements v1al } supported[v1.LabelInstanceTypeStable].Insert(instanceType.Name()) supported[v1.LabelArchStable].Insert(instanceType.Architecture()) + supported[v1.LabelOSStable].Insert(instanceType.OperatingSystems().List()...) } for key, values := range supported { requirements = append(requirements, v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: values.UnsortedList()}) diff --git a/pkg/controllers/provisioning/suite_test.go b/pkg/controllers/provisioning/suite_test.go index c4b3df44408f..456d83a50b1f 100644 --- a/pkg/controllers/provisioning/suite_test.go +++ b/pkg/controllers/provisioning/suite_test.go @@ -104,6 +104,8 @@ var _ = Describe("Provisioning", func() { test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}}), // Constrained by architecture test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}), + // Constrained by operatingSystem + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: "linux"}}), } unschedulable := []*v1.Pod{ // Ignored, matches another provisioner @@ -114,6 +116,8 @@ var _ = Describe("Provisioning", func() { test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "unknown"}}), // Ignored, invalid architecture test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "unknown"}}), + // Ignored, invalid operating system + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: "unknown"}}), // Ignored, invalid capacity type test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.LabelCapacityType: "unknown"}}), // Ignored, label selector does not match