[WIP] Add support for node readiness taints

API.md

@@ -88,6 +88,25 @@ overriden by NodeSelectors at the pod level.</p>
 <tbody>
 <tr>
 <td>
+<code>readinessTaints</code><br/>
+<em>
+<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#taint-v1-core">
+[]Kubernetes core/v1.Taint
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Readiness taints will be applied to every node launched by the Provisioner.
+Those taints will be ignored by the provisioner for scheduling purposes
+and it will provision nodes for pods that do not have matching tolerations.
+Readiness taints are useful for setups which have cluster add-ons such as CNI
+plugins, which use some specific taints on all new nodes to make sure Pods
+are only scheduled onto that nodes once the plugin has prepared them.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>taints</code><br/>
 <em>
 <a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#taint-v1-core">
@@ -427,5 +446,5 @@ its target, and indicates whether or not those conditions are met.</p>
 <hr/>
 <p><em>
 Generated with <code>gen-crd-api-reference-docs</code>
-on git commit <code>7eab29b</code>.
+on git commit <code>cb274e9</code>.
 </em></p>

charts/karpenter/templates/karpenter.sh_provisioners.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.6.1
   creationTimestamp: null
   name: provisioners.karpenter.sh
 spec:
@@ -89,6 +89,39 @@ spec:
                 description: OperatingSystem constrains the underlying node operating
                   system
                 type: string
+              readinessTaints:
+                description: Readiness taints will be applied to every node launched
+                  by the Provisioner. Those taints will be ignored by the provisioner
+                  for scheduling purposes and it will provision nodes for pods that
+                  do not have matching tolerations. Readiness taints are useful for
+                  setups which have cluster add-ons such as CNI plugins, which use
+                  some specific taints on all new nodes to make sure Pods are only
+                  scheduled onto that nodes once the plugin has prepared them.
+                items:
+                  description: The node this Taint is attached to has the "effect"
+                    on any pod that does not tolerate the Taint.
+                  properties:
+                    effect:
+                      description: Required. The effect of the taint on pods that
+                        do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule
+                        and NoExecute.
+                      type: string
+                    key:
+                      description: Required. The taint key to be applied to a node.
+                      type: string
+                    timeAdded:
+                      description: TimeAdded represents the time at which the taint
+                        was added. It is only written for NoExecute taints.
+                      format: date-time
+                      type: string
+                    value:
+                      description: The taint value corresponding to the taint key.
+                      type: string
+                  required:
+                  - effect
+                  - key
+                  type: object
+                type: array
               taints:
                 description: Taints will be applied to every node launched by the
                   Provisioner. If specified, the provisioner will not provision nodes

pkg/apis/provisioning/v1alpha3/provisioner.go

@@ -75,6 +75,14 @@ type Cluster struct {
 // Constraints are applied to all nodes created by the provisioner. They can be
 // overriden by NodeSelectors at the pod level.
 type Constraints struct {
+	// Readiness taints will be applied to every node launched by the Provisioner.
+	// Those taints will be ignored by the provisioner for scheduling purposes
+	// and it will provision nodes for pods that do not have matching tolerations.
+	// Readiness taints are useful for setups which have cluster add-ons such as CNI
+	// plugins, which use some specific taints on all new nodes to make sure Pods
+	// are only scheduled onto that nodes once the plugin has prepared them.
+	// +optional
+	ReadinessTaints []v1.Taint `json:"readinessTaints,omitempty"`
 	// Taints will be applied to every node launched by the Provisioner. If
 	// specified, the provisioner will not provision nodes for pods that do not
 	// have matching tolerations.
@@ -125,6 +133,7 @@ var (
 	// Reserved annotations
 	DoNotEvictPodAnnotationKey      = SchemeGroupVersion.Group + "/do-not-evict"
 	EmptinessTimestampAnnotationKey = SchemeGroupVersion.Group + "/emptiness-timestamp"
+	AsyncBindFailureAnnotationKey   = SchemeGroupVersion.Group + "/async-bind-failed-at"
 	// Finalizers
 	TerminationFinalizer = SchemeGroupVersion.Group + "/termination"
 	// Default provisioner
@@ -158,6 +167,7 @@ func (c *Constraints) WithLabel(key string, value string) *Constraints {
 
 func (c *Constraints) WithOverrides(pod *v1.Pod) *Constraints {
 	return &Constraints{
+		ReadinessTaints: c.ReadinessTaints,
 		Taints:          c.Taints,
 		Labels:          functional.UnionStringMaps(c.Labels, pod.Spec.NodeSelector),
 		Zones:           c.getZones(pod),

pkg/apis/provisioning/v1alpha3/provisioner_validation.go

@@ -122,6 +122,7 @@ func (c *Constraints) Validate(ctx context.Context) (errs *apis.FieldError) {
 	errs = errs.Also(
 		c.validateLabels(),
 		c.validateTaints(),
+		c.validateReadinessTaints(),
 		c.validateArchitecture(),
 		c.validateOperatingSystem(),
 		c.validateZones(),
@@ -146,18 +147,26 @@ func (c *Constraints) validateLabels() (errs *apis.FieldError) {
 }
 
 func (c *Constraints) validateTaints() (errs *apis.FieldError) {
-	for i, taint := range c.Taints {
+	return c.validateTaintsImpl(c.Taints, "taints")
+}
+
+func (c *Constraints) validateReadinessTaints() (errs *apis.FieldError) {
+	return c.validateTaintsImpl(c.ReadinessTaints, "readinessTaints")
+}
+
+func (c *Constraints) validateTaintsImpl(taints []v1.Taint, propertyName string) (errs *apis.FieldError) {
+	for i, taint := range taints {
 		// Validate Key
 		if len(taint.Key) == 0 {
-			errs = errs.Also(apis.ErrInvalidArrayValue(errs, "taints", i))
+			errs = errs.Also(apis.ErrInvalidArrayValue(errs, propertyName, i))
 		}
 		for _, err := range validation.IsQualifiedName(taint.Key) {
-			errs = errs.Also(apis.ErrInvalidArrayValue(err, "taints", i))
+			errs = errs.Also(apis.ErrInvalidArrayValue(err, propertyName, i))
 		}
 		// Validate Value
 		if len(taint.Value) != 0 {
 			for _, err := range validation.IsQualifiedName(taint.Value) {
-				errs = errs.Also(apis.ErrInvalidArrayValue(err, "taints", i))
+				errs = errs.Also(apis.ErrInvalidArrayValue(err, propertyName, i))
 			}
 		}
 		// Validate effect

pkg/apis/provisioning/v1alpha3/provisioner_validation_test.go

@@ -142,6 +142,33 @@ var _ = Describe("Validation", func() {
 			Expect(provisioner.Validate(ctx)).ToNot(Succeed())
 		})
 	})
+	Context("ReadinessTaints", func() {
+		It("should succeed for valid readinessTaints", func() {
+			provisioner.Spec.ReadinessTaints = []v1.Taint{
+				{Key: "a", Value: "b", Effect: v1.TaintEffectNoSchedule},
+				{Key: "c", Value: "d", Effect: v1.TaintEffectNoExecute},
+				{Key: "e", Value: "f", Effect: v1.TaintEffectPreferNoSchedule},
+				{Key: "key-only", Effect: v1.TaintEffectNoExecute},
+			}
+			Expect(provisioner.Validate(ctx)).To(Succeed())
+		})
+		It("should fail for invalid taint keys", func() {
+			provisioner.Spec.ReadinessTaints = []v1.Taint{{Key: "???"}}
+			Expect(provisioner.Validate(ctx)).ToNot(Succeed())
+		})
+		It("should fail for missing taint key", func() {
+			provisioner.Spec.ReadinessTaints = []v1.Taint{{Effect: v1.TaintEffectNoSchedule}}
+			Expect(provisioner.Validate(ctx)).ToNot(Succeed())
+		})
+		It("should fail for invalid taint value", func() {
+			provisioner.Spec.ReadinessTaints = []v1.Taint{{Key: "invalid-value", Effect: v1.TaintEffectNoSchedule, Value: "???"}}
+			Expect(provisioner.Validate(ctx)).ToNot(Succeed())
+		})
+		It("should fail for invalid taint effect", func() {
+			provisioner.Spec.ReadinessTaints = []v1.Taint{{Key: "invalid-effect", Effect: "???"}}
+			Expect(provisioner.Validate(ctx)).ToNot(Succeed())
+		})
+	})
 	Context("Zones", func() {
 		SupportedZones = append(SupportedZones, "test-zone-1")
 		It("should succeed if unspecified", func() {

pkg/cloudprovider/aws/launchtemplate.go

@@ -43,9 +43,11 @@ cluster-name = "{{if .Cluster.Name}}{{.Cluster.Name}}{{end}}"
 {{if .Constraints.Labels }}[settings.kubernetes.node-labels]{{ end }}
 {{ range $Key, $Value := .Constraints.Labels }}"{{ $Key }}" = "{{ $Value }}"
 {{ end }}
-{{if .Constraints.Taints }}[settings.kubernetes.node-taints]{{ end }}
+{{if or (.Constraints.Taints) (.Constraints.ReadinessTaints) }}[settings.kubernetes.node-taints]{{ end }}
 {{ range $Taint := .Constraints.Taints }}"{{ $Taint.Key }}" = "{{ $Taint.Value}}:{{ $Taint.Effect }}"
 {{ end }}
+{{ range $Taint := .Constraints.ReadinessTaints }}"{{ $Taint.Key }}" = "{{ $Taint.Value}}:{{ $Taint.Effect }}"
+{{ end }}
 `
 )
 

pkg/cloudprovider/fake/cloudprovider.go

@@ -23,6 +23,7 @@ import (
 	"github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha3"
 	"github.com/awslabs/karpenter/pkg/cloudprovider"
 	"github.com/awslabs/karpenter/pkg/utils/functional"
+	"github.com/awslabs/karpenter/pkg/utils/node"
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -45,14 +46,17 @@ func (c *CloudProvider) Create(ctx context.Context, provisioner *v1alpha3.Provis
 
 	err := make(chan error)
 	go func() {
+		taints := make([]v1.Taint, 0)
+		taints = append(taints, packing.Constraints.Taints...)
+		taints = node.UniqueTaints(taints, packing.Constraints.ReadinessTaints...)
 		err <- bind(&v1.Node{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:   name,
 				Labels: packing.Constraints.Labels,
 			},
 			Spec: v1.NodeSpec{
 				ProviderID: fmt.Sprintf("fake:///%s/%s", name, zone),
-				Taints:     packing.Constraints.Taints,
+				Taints:     taints,
 			},
 			Status: v1.NodeStatus{
 				NodeInfo: v1.NodeSystemInfo{