Skip to content

Commit

Permalink
Introducing cloudprovider API to vend provisioner specific Requirements
Browse files Browse the repository at this point in the history
  • Loading branch information
ellistarn committed Oct 28, 2021
1 parent dbfc914 commit 70a771e
Show file tree
Hide file tree
Showing 25 changed files with 403 additions and 479 deletions.
139 changes: 80 additions & 59 deletions pkg/apis/provisioning/v1alpha5/provisioner.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ package v1alpha5
import (
"sort"

"github.com/awslabs/karpenter/pkg/utils/functional"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/sets"
)

// ProvisionerSpec is the top level provisioner specification. Provisioners
Expand Down Expand Up @@ -95,105 +95,126 @@ type ProvisionerList struct {
}

// Zones for the constraints
func (c *Constraints) Zones() []string {
return c.Requirements.GetLabelValues(v1.LabelTopologyZone)
func (r Requirements) Zones() []string {
return r.GetLabelValues(v1.LabelTopologyZone)
}

// InstanceTypes for the constraints
func (c *Constraints) InstanceTypes() []string {
return c.Requirements.GetLabelValues(v1.LabelInstanceTypeStable)
func (r Requirements) InstanceTypes() []string {
return r.GetLabelValues(v1.LabelInstanceTypeStable)
}

// Architectures for the constraints
func (c *Constraints) Architectures() []string {
return c.Requirements.GetLabelValues(v1.LabelArchStable)
func (r Requirements) Architectures() []string {
return r.GetLabelValues(v1.LabelArchStable)
}

// OperatingSystems for the constraints
func (c *Constraints) OperatingSystems() []string {
return c.Requirements.GetLabelValues(v1.LabelOSStable)
func (r Requirements) OperatingSystems() []string {
return r.GetLabelValues(v1.LabelOSStable)
}

// Consolidate and copy the constraints
func (c *Constraints) Consolidate() *Constraints {
// Combine labels and requirements
combined := append(Requirements{}, c.Requirements...)
for key, value := range c.Labels {
combined = append(combined, v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}})
func (r Requirements) WithProvisioner(provisioner Provisioner) Requirements {
return r.
With(provisioner.Spec.Requirements).
WithLabels(provisioner.Spec.Labels).
WithLabels(map[string]string{ProvisionerNameLabelKey: provisioner.Name})
}

func (r Requirements) With(requirements Requirements) Requirements {
return append(r, requirements...)
}

func (r Requirements) WithLabels(labels map[string]string) Requirements {
for key, value := range labels {
r = append(r, v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}})
}
return r
}

func (r Requirements) WithPod(pod *v1.Pod) Requirements {
for key, value := range pod.Spec.NodeSelector {
r = append(r, v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}})
}
if pod.Spec.Affinity == nil || pod.Spec.Affinity.NodeAffinity == nil {
return r
}
// Simplify to a single OpIn per label
requirements := Requirements{}
for _, label := range combined.GetLabels() {
// Select heaviest preference and treat as a requirement. An outer loop will iteratively unconstrain them if unsatisfiable.
if preferred := pod.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution; len(preferred) > 0 {
sort.Slice(preferred, func(i int, j int) bool { return preferred[i].Weight > preferred[j].Weight })
r = append(r, preferred[0].Preference.MatchExpressions...)
}
// Select first requirement. An outer loop will iteratively remove OR requirements if unsatisfiable
if pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil &&
len(pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms) > 0 {
r = append(r, pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions...)
}
return r
}

func (r Requirements) Consolidate() (requirements Requirements) {
for _, label := range r.GetLabels() {
requirements = append(requirements, v1.NodeSelectorRequirement{
Key: label,
Operator: v1.NodeSelectorOpIn,
Values: combined.GetLabelValues(label),
Values: r.GetLabelValues(label),
})
}
return &Constraints{
Labels: c.Labels,
Taints: c.Taints,
Requirements: requirements,
Provider: c.Provider,
}
return requirements
}

// With adds additional requirements from the pods
func (r Requirements) With(pods ...*v1.Pod) Requirements {
for _, pod := range pods {
for key, value := range pod.Spec.NodeSelector {
r = append(r, v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}})
func (r Requirements) CustomLabels() map[string]string {
labels := map[string]string{}
for _, label := range r.GetLabels() {
if !WellKnownLabels.Has(label) {
if values := r.GetLabelValues(label); len(values) > 0 {
labels[label] = values[0]
}
}
if pod.Spec.Affinity == nil || pod.Spec.Affinity.NodeAffinity == nil {
continue
}
// Select heaviest preference and treat as a requirement. An outer loop will iteratively unconstrain them if unsatisfiable.
if preferred := pod.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution; len(preferred) > 0 {
sort.Slice(preferred, func(i int, j int) bool { return preferred[i].Weight > preferred[j].Weight })
r = append(r, preferred[0].Preference.MatchExpressions...)
}
// Select first requirement. An outer loop will iteratively remove OR requirements if unsatisfiable
if pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil &&
len(pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms) > 0 {
r = append(r, pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions...)
}
return labels
}

func (r Requirements) WellKnown() (requirements Requirements) {
for _, requirement := range r {
if WellKnownLabels.Has(requirement.Key) {
requirements = append(requirements, requirement)
}
}
return r
return requirements
}

// GetLabels returns unique set of the label keys from the requirements
func (r Requirements) GetLabels() []string {
keys := map[string]bool{}
keys := sets.NewString()
for _, requirement := range r {
keys[requirement.Key] = true
}
result := []string{}
for key := range keys {
result = append(result, key)
keys.Insert(requirement.Key)
}
return result
return keys.List()
}

// GetLabelValues for the provided key constrained by the requirements
func (r Requirements) GetLabelValues(label string) []string {
var result []string
if known, ok := WellKnownLabels[label]; ok {
result = known
}
var result sets.String
// OpIn
for _, requirement := range r {
if requirement.Key == label && requirement.Operator == v1.NodeSelectorOpIn {
result = functional.IntersectStringSlice(result, requirement.Values)
if result == nil {
result = sets.NewString(requirement.Values...)
} else {
result = result.Intersection(sets.NewString(requirement.Values...))
}
}
}
// OpNotIn
for _, requirement := range r {
if requirement.Key == label && requirement.Operator == v1.NodeSelectorOpNotIn {
result = functional.StringSliceWithout(result, requirement.Values...)
result = result.Difference(sets.NewString(requirement.Values...))
}
}
if len(result) == 0 {
result = []string{}
// Unconstrained
if result == nil {
return nil
}
return result
return result.List()
}
9 changes: 0 additions & 9 deletions pkg/apis/provisioning/v1alpha5/provisioner_validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,6 @@ func (c *Constraints) Validate(ctx context.Context) (errs *apis.FieldError) {
c.validateLabels(),
c.validateTaints(),
c.validateRequirements(),
c.Consolidate().Requirements.Validate(),
ValidateHook(ctx, c),
)
}
Expand All @@ -95,9 +94,6 @@ func (c *Constraints) validateLabels() (errs *apis.FieldError) {
for _, err := range validation.IsValidLabelValue(value) {
errs = errs.Also(apis.ErrInvalidValue(fmt.Sprintf("%s, %s", value, err), fmt.Sprintf("labels[%s]", key)))
}
if known, ok := WellKnownLabels[key]; ok && !functional.ContainsString(known, value) {
errs = errs.Also(apis.ErrInvalidValue(fmt.Sprintf("%s not in %s", value, known), fmt.Sprintf("labels[%s]", key)))
}
}
return errs
}
Expand Down Expand Up @@ -153,11 +149,6 @@ func validateRequirement(requirement v1.NodeSelectorRequirement) (errs *apis.Fie
for _, err := range validation.IsValidLabelValue(value) {
errs = errs.Also(apis.ErrInvalidArrayValue(fmt.Sprintf("%s, %s", value, err), "values", i))
}
if known, ok := WellKnownLabels[requirement.Key]; ok {
if !functional.ContainsString(known, value) {
errs = errs.Also(apis.ErrInvalidArrayValue(fmt.Sprintf("%s not in %s", value, known), "values", i))
}
}
}
if !functional.ContainsString(SupportedNodeSelectorOps, string(requirement.Operator)) {
errs = errs.Also(apis.ErrInvalidValue(fmt.Sprintf("%s not in %s", requirement.Operator, SupportedNodeSelectorOps), "operator"))
Expand Down
51 changes: 0 additions & 51 deletions pkg/apis/provisioning/v1alpha5/provisioner_validation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,24 +77,6 @@ var _ = Describe("Validation", func() {
Expect(provisioner.Validate(ctx)).ToNot(Succeed())
}
})
It("should succeed for well known label values", func() {
WellKnownLabels[v1.LabelTopologyZone] = []string{"test-1", "test1"}
WellKnownLabels[v1.LabelInstanceTypeStable] = []string{"test-1", "test1"}
WellKnownLabels[v1.LabelArchStable] = []string{"test-1", "test1"}
WellKnownLabels[v1.LabelOSStable] = []string{"test-1", "test1"}
for key, values := range WellKnownLabels {
for _, value := range values {
provisioner.Spec.Labels = map[string]string{key: value}
Expect(provisioner.Validate(ctx)).To(Succeed())
}
}
})
It("should fail for invalid well known label values", func() {
for key := range WellKnownLabels {
provisioner.Spec.Labels = map[string]string{key: "unknown"}
Expect(provisioner.Validate(ctx)).ToNot(Succeed())
}
})
})
Context("Taints", func() {
It("should succeed for valid taints", func() {
Expand Down Expand Up @@ -137,38 +119,5 @@ var _ = Describe("Validation", func() {
Expect(provisioner.Validate(ctx)).ToNot(Succeed())
}
})
It("should validate well known labels", func() {
WellKnownLabels[v1.LabelTopologyZone] = []string{"test"}
provisioner.Spec.Requirements = Requirements{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}}
Expect(provisioner.Validate(ctx)).To(Succeed())
provisioner.Spec.Labels = map[string]string{}
provisioner.Spec.Requirements = Requirements{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}}
Expect(provisioner.Validate(ctx)).To(Succeed())
provisioner.Spec.Labels = map[string]string{v1.LabelTopologyZone: "test"}
provisioner.Spec.Requirements = Requirements{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}}
Expect(provisioner.Validate(ctx)).To(Succeed())
provisioner.Spec.Labels = map[string]string{v1.LabelTopologyZone: "test"}
provisioner.Spec.Requirements = Requirements{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test"}}}
Expect(provisioner.Validate(ctx)).ToNot(Succeed())
provisioner.Spec.Labels = map[string]string{v1.LabelTopologyZone: "test"}
provisioner.Spec.Requirements = Requirements{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}
Expect(provisioner.Validate(ctx)).ToNot(Succeed())
})
It("should validate custom labels", func() {
provisioner.Spec.Requirements = Requirements{{Key: "test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}}
Expect(provisioner.Validate(ctx)).To(Succeed())
provisioner.Spec.Labels = map[string]string{}
provisioner.Spec.Requirements = Requirements{{Key: "test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}}
Expect(provisioner.Validate(ctx)).To(Succeed())
provisioner.Spec.Labels = map[string]string{"test": "test"}
provisioner.Spec.Requirements = Requirements{{Key: "test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}}
Expect(provisioner.Validate(ctx)).To(Succeed())
provisioner.Spec.Labels = map[string]string{"test": "test"}
provisioner.Spec.Requirements = Requirements{{Key: "test", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test"}}}
Expect(provisioner.Validate(ctx)).ToNot(Succeed())
provisioner.Spec.Labels = map[string]string{"test": "test"}
provisioner.Spec.Requirements = Requirements{{Key: "test", Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}
Expect(provisioner.Validate(ctx)).ToNot(Succeed())
})
})
})
15 changes: 8 additions & 7 deletions pkg/apis/provisioning/v1alpha5/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"knative.dev/pkg/apis"
)

Expand All @@ -45,13 +46,13 @@ var (
EmptinessTimestampAnnotationKey,
v1.LabelHostname,
}
// WellKnownLabels supported by karpenter and their allowable values
WellKnownLabels = map[string][]string{
v1.LabelTopologyZone: {},
v1.LabelInstanceTypeStable: {},
v1.LabelArchStable: {},
v1.LabelOSStable: {},
}
// WellKnownLabels supported by karpenter
WellKnownLabels = sets.NewString(
v1.LabelTopologyZone,
v1.LabelInstanceTypeStable,
v1.LabelArchStable,
v1.LabelOSStable,
)
DefaultHook = func(ctx context.Context, constraints *Constraints) {}
ValidateHook = func(ctx context.Context, constraints *Constraints) *apis.FieldError { return nil }
)
Expand Down
10 changes: 8 additions & 2 deletions pkg/cloudprovider/aws/apis/v1alpha1/provider_defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@ func (c *Constraints) Default(ctx context.Context) {
}

func (c *Constraints) defaultCapacityTypes() {
if functional.ContainsString(c.Consolidate().Requirements.GetLabels(), CapacityTypeLabel) {
if _, ok := c.Labels[CapacityTypeLabel]; ok {
return
}
if functional.ContainsString(c.Requirements.GetLabels(), CapacityTypeLabel) {
return
}
c.Requirements = append(c.Requirements, v1.NodeSelectorRequirement{
Expand All @@ -45,7 +48,10 @@ func (c *Constraints) defaultCapacityTypes() {
}

func (c *Constraints) defaultArchitecture() {
if functional.ContainsString(c.Consolidate().Requirements.GetLabels(), v1.LabelArchStable) {
if _, ok := c.Labels[v1.LabelArchStable]; ok {
return
}
if functional.ContainsString(c.Requirements.GetLabels(), v1.LabelArchStable) {
return
}
c.Requirements = append(c.Requirements, v1.NodeSelectorRequirement{
Expand Down
2 changes: 1 addition & 1 deletion pkg/cloudprovider/aws/apis/v1alpha1/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,5 @@ var (
func init() {
Scheme.AddKnownTypes(schema.GroupVersion{Group: v1alpha5.ExtensionsGroup, Version: "v1alpha1"}, &AWS{})
v1alpha5.RestrictedLabels = append(v1alpha5.RestrictedLabels, AWSLabelPrefix)
v1alpha5.WellKnownLabels[CapacityTypeLabel] = []string{CapacityTypeSpot, CapacityTypeOnDemand}
v1alpha5.WellKnownLabels.Insert(CapacityTypeLabel)
}
Loading

0 comments on commit 70a771e

Please sign in to comment.