Skip to content

Commit

Permalink
fix getDaemonSet to preserve pod affinity in daemonset template pod s…
Browse files Browse the repository at this point in the history 
…pec. (aws#533)

Co-authored-by: shikai.ng <[email protected]>
Co-authored-by: Jonathan Innis <[email protected]>
  • Loading branch information
@shikai93 @jonathan-innis
3 people authored Sep 26, 2023
1 parent 7c33a18 commit 7bda4a7
Show file tree
Hide file tree
Showing 3 changed files with 160 additions and 0 deletions.
79 changes: 79 additions & 0 deletions pkg/controllers/provisioning/nodepool_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -553,6 +553,85 @@ var _ = Describe("Provisioning", func() {
Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4")))
Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi")))
})
It("should account for daemonset spec affinity", func() {
nodePool := test.NodePool(v1beta1.NodePool{
Spec: v1beta1.NodePoolSpec{
Template: v1beta1.NodeClaimTemplate{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"foo": "voo",
},
},
},
Limits: v1beta1.Limits(v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
}),
},
})
nodePoolDaemonset := test.NodePool(v1beta1.NodePool{
Spec: v1beta1.NodePoolSpec{
Template: v1beta1.NodeClaimTemplate{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"foo": "bar",
},
},
},
},
})
// Create a daemonset with large resource requests
daemonset := test.DaemonSet(
test.DaemonSetOptions{PodOptions: test.PodOptions{
NodeRequirements: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"bar"},
},
},
ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("4Gi")}},
}},
)
ExpectApplied(ctx, env.Client, nodePoolDaemonset, daemonset)
// Create the actual daemonSet pod with lower resource requests and expect to use the pod
daemonsetPod := test.UnschedulablePod(
test.PodOptions{
ObjectMeta: metav1.ObjectMeta{
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: "apps/v1",
Kind: "DaemonSet",
Name: daemonset.Name,
UID: daemonset.UID,
Controller: ptr.Bool(true),
BlockOwnerDeletion: ptr.Bool(true),
},
},
},
NodeRequirements: []v1.NodeSelectorRequirement{
{
Key: metav1.ObjectNameField,
Operator: v1.NodeSelectorOpIn,
Values: []string{"node-name"},
},
},
ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("4Gi")}},
})
ExpectApplied(ctx, env.Client, daemonsetPod)
ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, daemonsetPod)
ExpectReconcileSucceeded(ctx, daemonsetController, client.ObjectKeyFromObject(daemonset))

//Deploy pod
pod := test.UnschedulablePod(test.PodOptions{
ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}},
NodeSelector: map[string]string{
"foo": "voo",
},
})
ExpectApplied(ctx, env.Client, nodePool, pod)
ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod)
ExpectScheduled(ctx, env.Client, pod)
})
})
Context("Annotations", func() {
It("should annotate nodes", func() {
Expand Down
12 changes: 12 additions & 0 deletions pkg/controllers/provisioning/provisioner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,6 +417,18 @@ func (p *Provisioner) getDaemonSetPods(ctx context.Context) ([]*v1.Pod, error) {
if pod == nil {
pod = &v1.Pod{Spec: d.Spec.Template.Spec}
}
// Replacing retrieved pod affinity with daemonset pod template required node affinity since this is overridden
// by the daemonset controller during pod creation
// https://github.com/kubernetes/kubernetes/blob/c5cf0ac1889f55ab51749798bec684aed876709d/pkg/controller/daemon/util/daemonset_util.go#L176
if d.Spec.Template.Spec.Affinity != nil && d.Spec.Template.Spec.Affinity.NodeAffinity != nil && d.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
if pod.Spec.Affinity == nil {
pod.Spec.Affinity = &v1.Affinity{}
}
if pod.Spec.Affinity.NodeAffinity == nil {
pod.Spec.Affinity.NodeAffinity = &v1.NodeAffinity{}
}
pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = d.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution
}
return pod
}), nil
}
Expand Down
69 changes: 69 additions & 0 deletions pkg/controllers/provisioning/provisioner_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -537,6 +537,75 @@ var _ = Describe("Provisioning", func() {
Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4")))
Expect(*allocatable.Memory()).To(Equal(resource.MustParse("4Gi")))
})
It("should account for daemonset spec affinity", func() {
provisioner := test.Provisioner(test.ProvisionerOptions{
Labels: map[string]string{
"foo": "voo",
},
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
})

provisionerDaemonset := test.Provisioner(test.ProvisionerOptions{
Labels: map[string]string{
"foo": "bar",
},
})

// Create a daemonset with large resource requests
daemonset := test.DaemonSet(
test.DaemonSetOptions{PodOptions: test.PodOptions{
NodeRequirements: []v1.NodeSelectorRequirement{
{
Key: "foo",
Operator: v1.NodeSelectorOpIn,
Values: []string{"bar"},
},
},
ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("4Gi")}},
}},
)
ExpectApplied(ctx, env.Client, provisionerDaemonset, daemonset)
// Create the actual daemonSet pod with lower resource requests and expect to use the pod
daemonsetPod := test.UnschedulablePod(
test.PodOptions{
ObjectMeta: metav1.ObjectMeta{
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: "apps/v1",
Kind: "DaemonSet",
Name: daemonset.Name,
UID: daemonset.UID,
Controller: ptr.Bool(true),
BlockOwnerDeletion: ptr.Bool(true),
},
},
},
NodeRequirements: []v1.NodeSelectorRequirement{
{
Key: metav1.ObjectNameField,
Operator: v1.NodeSelectorOpIn,
Values: []string{"node-name"},
},
},
ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("4Gi")}},
})
ExpectApplied(ctx, env.Client, daemonsetPod)
ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, daemonsetPod)
ExpectReconcileSucceeded(ctx, daemonsetController, client.ObjectKeyFromObject(daemonset))

//Deploy pod
pod := test.UnschedulablePod(test.PodOptions{
ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}},
NodeSelector: map[string]string{
"foo": "voo",
},
})
ExpectApplied(ctx, env.Client, provisioner, pod)
ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod)
ExpectScheduled(ctx, env.Client, pod)
})
})
Context("Annotations", func() {
It("should annotate nodes", func() {
Expand Down

0 comments on commit 7bda4a7

Please sign in to comment.