Relax PreferNoSchedule Taint

pkg/controllers/selection/preferences.go

@@ -49,26 +49,34 @@ func NewPreferences() *Preferences {
 // scheduled to that zone. Preferences are removed iteratively until only hard
 // constraints remain. Pods relaxation is reset (forgotten) after 5 minutes.
 func (p *Preferences) Relax(ctx context.Context, pod *v1.Pod) {
-	affinity, ok := p.cache.Get(string(pod.UID))
+	spec, ok := p.cache.Get(string(pod.UID))
 	// Add to cache if we've never seen it before
 	if !ok {
-		p.cache.SetDefault(string(pod.UID), pod.Spec.Affinity)
+		// Limit cached PodSpec to only required data
+		cachedSpec := v1.PodSpec{
+			Affinity:    pod.Spec.Affinity,
+			Tolerations: pod.Spec.Tolerations,
+		}
+		p.cache.SetDefault(string(pod.UID), cachedSpec)
 		return
 	}
 	// Attempt to relax the pod and update the cache
-	pod.Spec.Affinity = affinity.(*v1.Affinity)
+	cachedSpec := spec.(v1.PodSpec)
+	pod.Spec.Affinity = cachedSpec.Affinity
+	pod.Spec.Tolerations = cachedSpec.Tolerations
 	if relaxed := p.relax(ctx, pod); relaxed {
-		p.cache.SetDefault(string(pod.UID), pod.Spec.Affinity)
+		p.cache.SetDefault(string(pod.UID), pod.Spec)
 	}
 }
 
 func (p *Preferences) relax(ctx context.Context, pod *v1.Pod) bool {
 	for _, relaxFunc := range []func(*v1.Pod) *string{
 		func(pod *v1.Pod) *string { return p.removePreferredNodeAffinityTerm(pod) },
 		func(pod *v1.Pod) *string { return p.removeRequiredNodeAffinityTerm(pod) },
+		func(pod *v1.Pod) *string { return p.toleratePreferNoScheduleTaints(pod) },
 	} {
 		if reason := relaxFunc(pod); reason != nil {
-			logging.FromContext(ctx).Debugf("Relaxing soft constraints for %s/%s since it previously failed to schedule, removing: %s", pod.Namespace, pod.Name, ptr.StringValue(reason))
+			logging.FromContext(ctx).Debugf("Relaxing soft constraints for pod since it previously failed to schedule, %s", ptr.StringValue(reason))
 			return true
 		}
 	}
@@ -85,7 +93,7 @@ func (p *Preferences) removePreferredNodeAffinityTerm(pod *v1.Pod) *string {
 		// Sort descending by weight to remove heaviest preferences to try lighter ones
 		sort.SliceStable(terms, func(i, j int) bool { return terms[i].Weight > terms[j].Weight })
 		pod.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution = terms[1:]
-		return ptr.String(fmt.Sprintf("spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[0]=%s", pretty.Concise(terms[0])))
+		return ptr.String(fmt.Sprintf("removing: spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[0]=%s", pretty.Concise(terms[0])))
 	}
 	return nil
 }
@@ -101,7 +109,23 @@ func (p *Preferences) removeRequiredNodeAffinityTerm(pod *v1.Pod) *string {
 	// Remove the first term if there's more than one (terms are an OR semantic), Unlike preferred affinity, we cannot remove all terms
 	if len(terms) > 1 {
 		pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = terms[1:]
-		return ptr.String(fmt.Sprintf("spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution[0]=%s", pretty.Concise(terms[0])))
+		return ptr.String(fmt.Sprintf("removing: spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution[0]=%s", pretty.Concise(terms[0])))
 	}
 	return nil
 }
+
+func (p *Preferences) toleratePreferNoScheduleTaints(pod *v1.Pod) *string {
+	// Tolerate all Taints with PreferNoSchedule effect
+	toleration := v1.Toleration{
+		Operator: v1.TolerationOpExists,
+		Effect:   v1.TaintEffectPreferNoSchedule,
+	}
+	for _, t := range pod.Spec.Tolerations {
+		if t.MatchToleration(&toleration) {
+			return nil
+		}
+	}
+	tolerations := append(pod.Spec.Tolerations, toleration)
+	pod.Spec.Tolerations = tolerations
+	return ptr.String("adding: toleration for PreferNoSchedule taints")
+}

pkg/controllers/selection/suite_test.go

@@ -233,6 +233,35 @@ var _ = Describe("Preferential Fallback", func() {
 			node := ExpectScheduled(ctx, env.Client, pod)
 			Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2"))
 		})
+		It("should tolerate PreferNoSchedule taint only after trying to relax Affinity terms", func() {
+			provisioner.Spec.Taints = v1alpha5.Taints{{Key: "foo", Value: "bar", Effect: v1.TaintEffectPreferNoSchedule}}
+			pod := test.UnschedulablePod()
+			pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{
+				{
+					Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{
+						{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}},
+					}},
+				},
+				{
+					Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{
+						{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}},
+					}},
+				},
+			}}}
+			// Remove first term
+			pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0]
+			ExpectNotScheduled(ctx, env.Client, pod)
+			// Remove second term
+			pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0]
+			ExpectNotScheduled(ctx, env.Client, pod)
+			// Tolerate PreferNoSchedule Taint
+			pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0]
+			ExpectNotScheduled(ctx, env.Client, pod)
+			// Success
+			pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0]
+			node := ExpectScheduled(ctx, env.Client, pod)
+			Expect(node.Spec.Taints).To(ContainElement(v1.Taint{Key: "foo", Value: "bar", Effect: v1.TaintEffectPreferNoSchedule}))
+		})
 	})
 })
 
@@ -267,6 +296,15 @@ var _ = Describe("Multiple Provisioners", func() {
 		node := ExpectScheduled(ctx, env.Client, pod)
 		Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(provisioner2.Name))
 	})
+	It("should not match provisioners with PreferNoSchedule taint when other provisioners match", func() {
+		provisioner2 := provisioner.DeepCopy()
+		provisioner2.Name = "prefer-no-schedule"
+		provisioner2.Spec.Taints = v1alpha5.Taints{{Key: "foo", Value: "bar", Effect: v1.TaintEffectPreferNoSchedule}}
+		ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner2)
+		pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod())[0]
+		node := ExpectScheduled(ctx, env.Client, pod)
+		Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(provisioner.Name))
+	})
 })
 
 var _ = Describe("Pod Affinity and AntiAffinity", func() {