From 4b4530cf2bf63274af54ae60235724b8dd3f4026 Mon Sep 17 00:00:00 2001 From: Ellis Tarn Date: Mon, 13 Dec 2021 13:12:06 -0800 Subject: [PATCH] Moved scheduling tests to scheduling package and renamed scheduling controller to selection --- cmd/controller/main.go | 8 +- pkg/cloudprovider/aws/suite_test.go | 46 +++--- .../scheduling/suite_test.go | 139 +++++++----------- pkg/controllers/provisioning/suite_test.go | 36 ++--- .../{scheduling => selection}/controller.go | 2 +- .../{scheduling => selection}/preferences.go | 2 +- pkg/controllers/selection/suite_test.go | 106 +++++++++++++ pkg/test/expectations/expectations.go | 8 +- pkg/utils/pod/scheduling.go | 4 + 9 files changed, 214 insertions(+), 137 deletions(-) rename pkg/controllers/{ => provisioning}/scheduling/suite_test.go (82%) rename pkg/controllers/{scheduling => selection}/controller.go (99%) rename pkg/controllers/{scheduling => selection}/preferences.go (99%) create mode 100644 pkg/controllers/selection/suite_test.go diff --git a/cmd/controller/main.go b/cmd/controller/main.go index a93f15ed5ddc..20daa20188e1 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -26,7 +26,7 @@ import ( "github.com/aws/karpenter/pkg/controllers/metrics" "github.com/aws/karpenter/pkg/controllers/node" "github.com/aws/karpenter/pkg/controllers/provisioning" - "github.com/aws/karpenter/pkg/controllers/scheduling" + "github.com/aws/karpenter/pkg/controllers/selection" "github.com/aws/karpenter/pkg/controllers/termination" "github.com/aws/karpenter/pkg/utils/injection" "github.com/aws/karpenter/pkg/utils/options" @@ -83,11 +83,11 @@ func main() { HealthProbeBindAddress: fmt.Sprintf(":%d", opts.HealthProbePort), }) - provisioners := provisioning.NewController(ctx, manager.GetClient(), clientSet.CoreV1(), cloudProvider) + provisioningController := provisioning.NewController(ctx, manager.GetClient(), clientSet.CoreV1(), cloudProvider) if err := manager.RegisterControllers(ctx, - provisioners, - scheduling.NewController(manager.GetClient(), provisioners), + provisioningController, + selection.NewController(manager.GetClient(), provisioningController), termination.NewController(ctx, manager.GetClient(), clientSet.CoreV1(), cloudProvider), node.NewController(manager.GetClient()), metrics.NewController(manager.GetClient(), cloudProvider), diff --git a/pkg/cloudprovider/aws/suite_test.go b/pkg/cloudprovider/aws/suite_test.go index b5747795c831..f0e7a21f80d2 100644 --- a/pkg/cloudprovider/aws/suite_test.go +++ b/pkg/cloudprovider/aws/suite_test.go @@ -26,7 +26,7 @@ import ( "github.com/aws/karpenter/pkg/cloudprovider/aws/fake" "github.com/aws/karpenter/pkg/cloudprovider/registry" "github.com/aws/karpenter/pkg/controllers/provisioning" - "github.com/aws/karpenter/pkg/controllers/scheduling" + "github.com/aws/karpenter/pkg/controllers/selection" "github.com/aws/karpenter/pkg/test" . "github.com/aws/karpenter/pkg/test/expectations" "github.com/aws/karpenter/pkg/utils/injection" @@ -54,7 +54,7 @@ var launchTemplateCache *cache.Cache var unavailableOfferingsCache *cache.Cache var fakeEC2API *fake.EC2API var provisioners *provisioning.Controller -var scheduler *scheduling.Controller +var selectionController *selection.Controller func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -91,7 +91,7 @@ var _ = BeforeSuite(func() { } registry.RegisterOrDie(ctx, cloudProvider) provisioners = provisioning.NewController(ctx, e.Client, clientSet.CoreV1(), cloudProvider) - scheduler = scheduling.NewController(e.Client, provisioners) + selectionController = selection.NewController(e.Client, provisioners) }) Expect(env.Start()).To(Succeed(), "Failed to start environment") @@ -123,7 +123,7 @@ var _ = Describe("Allocation", func() { Context("Reconciliation", func() { Context("Specialized Hardware", func() { It("should not launch AWS Pod ENI on a t3", func() { - for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod(test.PodOptions{ NodeSelector: map[string]string{ v1.LabelInstanceTypeStable: "t3.large", @@ -137,7 +137,7 @@ var _ = Describe("Allocation", func() { } }) It("should launch AWS Pod ENI on a compatible instance type", func() { - for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod(test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{resources.AWSPodENI: resource.MustParse("1")}, @@ -155,7 +155,7 @@ var _ = Describe("Allocation", func() { }) It("should launch instances for Nvidia GPU resource requests", func() { nodeNames := sets.NewString() - for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod(test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{resources.NvidiaGPU: resource.MustParse("1")}, @@ -184,7 +184,7 @@ var _ = Describe("Allocation", func() { }) It("should launch instances for AWS Neuron resource requests", func() { nodeNames := sets.NewString() - for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod(test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{resources.AWSNeuron: resource.MustParse("1")}, @@ -216,7 +216,7 @@ var _ = Describe("Allocation", func() { Context("Insufficient Capacity Error Cache", func() { It("should launch instances of different type on second reconciliation attempt with Insufficient Capacity Error Cache fallback", func() { fakeEC2API.InsufficientCapacityPools = []fake.CapacityPool{{CapacityType: v1alpha1.CapacityTypeOnDemand, InstanceType: "inf1.6xlarge", Zone: "test-zone-1a"}} - pods := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + pods := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod(test.PodOptions{ NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}, ResourceRequirements: v1.ResourceRequirements{ @@ -237,7 +237,7 @@ var _ = Describe("Allocation", func() { ExpectNotScheduled(ctx, env.Client, pod) } nodeNames := sets.NewString() - for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, pods...) { + for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pods...) { node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "inf1.2xlarge")) nodeNames.Insert(node.Name) @@ -259,11 +259,11 @@ var _ = Describe("Allocation", func() { }}, }, }}} - pod = ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, pod)[0] + pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0] // it should've tried to pack them in test-zone-1a on a p3.8xlarge then hit insufficient capacity, the next attempt will try test-zone-1b ExpectNotScheduled(ctx, env.Client, pod) - pod = ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, pod)[0] + pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0] node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(SatisfyAll( HaveKeyWithValue(v1.LabelInstanceTypeStable, "p3.8xlarge"), @@ -271,7 +271,7 @@ var _ = Describe("Allocation", func() { }) It("should launch instances on later reconciliation attempt with Insufficient Capacity Error Cache expiry", func() { fakeEC2API.InsufficientCapacityPools = []fake.CapacityPool{{CapacityType: v1alpha1.CapacityTypeOnDemand, InstanceType: "inf1.6xlarge", Zone: "test-zone-1a"}} - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod(test.PodOptions{ NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "inf1.6xlarge"}, ResourceRequirements: v1.ResourceRequirements{ @@ -284,7 +284,7 @@ var _ = Describe("Allocation", func() { // capacity shortage is over - expire the item from the cache and try again fakeEC2API.InsufficientCapacityPools = []fake.CapacityPool{} unavailableOfferingsCache.Delete(UnavailableOfferingsCacheKey(v1alpha1.CapacityTypeOnDemand, "inf1.6xlarge", "test-zone-1a")) - pod = ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, pod)[0] + pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0] node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "inf1.6xlarge")) }) @@ -296,23 +296,23 @@ var _ = Describe("Allocation", func() { {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"m5.large"}}, } // Spot Unavailable - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod())[0] + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod())[0] ExpectNotScheduled(ctx, env.Client, pod) // Fallback to OD - pod = ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod())[0] + pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod())[0] node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.LabelCapacityType, v1alpha1.CapacityTypeOnDemand)) }) }) Context("CapacityType", func() { It("should default to on demand", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod())[0] + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod())[0] node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.LabelCapacityType, v1alpha1.CapacityTypeOnDemand)) }) It("should launch spot capacity if flexible to both spot and on demand", func() { provisioner.Spec.Requirements = v1alpha5.Requirements{{Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha1.CapacityTypeSpot, v1alpha1.CapacityTypeOnDemand}}} - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod())[0] + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod())[0] node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.LabelCapacityType, v1alpha1.CapacityTypeSpot)) }) @@ -338,7 +338,7 @@ var _ = Describe("Allocation", func() { Effect: "NoSchedule", } - pod1 := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + pod1 := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod(test.PodOptions{ Tolerations: []v1.Toleration{t1, t2, t3}, }), @@ -347,7 +347,7 @@ var _ = Describe("Allocation", func() { Expect(fakeEC2API.CalledWithCreateFleetInput.Cardinality()).To(Equal(1)) name1 := fakeEC2API.CalledWithCreateFleetInput.Pop().(*ec2.CreateFleetInput).LaunchTemplateConfigs[0].LaunchTemplateSpecification.LaunchTemplateName - pod2 := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + pod2 := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod(test.PodOptions{ Tolerations: []v1.Toleration{t2, t3, t1}, }), @@ -360,7 +360,7 @@ var _ = Describe("Allocation", func() { }) It("should default to a generated launch template", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod())[0] + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod())[0] ExpectScheduled(ctx, env.Client, pod) Expect(fakeEC2API.CalledWithCreateFleetInput.Cardinality()).To(Equal(1)) input := fakeEC2API.CalledWithCreateFleetInput.Pop().(*ec2.CreateFleetInput) @@ -370,7 +370,7 @@ var _ = Describe("Allocation", func() { }) It("should allow a launch template to be specified", func() { provider.LaunchTemplate = aws.String("test-launch-template") - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0] + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0] ExpectScheduled(ctx, env.Client, pod) Expect(fakeEC2API.CalledWithCreateFleetInput.Cardinality()).To(Equal(1)) input := fakeEC2API.CalledWithCreateFleetInput.Pop().(*ec2.CreateFleetInput) @@ -382,7 +382,7 @@ var _ = Describe("Allocation", func() { }) Context("Subnets", func() { It("should default to the cluster's subnets", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0] + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0] ExpectScheduled(ctx, env.Client, pod) Expect(fakeEC2API.CalledWithCreateFleetInput.Cardinality()).To(Equal(1)) input := fakeEC2API.CalledWithCreateFleetInput.Pop().(*ec2.CreateFleetInput) @@ -396,7 +396,7 @@ var _ = Describe("Allocation", func() { }) Context("Security Groups", func() { It("should default to the clusters security groups", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0] + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0] ExpectScheduled(ctx, env.Client, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput) diff --git a/pkg/controllers/scheduling/suite_test.go b/pkg/controllers/provisioning/scheduling/suite_test.go similarity index 82% rename from pkg/controllers/scheduling/suite_test.go rename to pkg/controllers/provisioning/scheduling/suite_test.go index c176a8751a90..d46462208451 100644 --- a/pkg/controllers/scheduling/suite_test.go +++ b/pkg/controllers/provisioning/scheduling/suite_test.go @@ -22,7 +22,7 @@ import ( "github.com/aws/karpenter/pkg/cloudprovider/fake" "github.com/aws/karpenter/pkg/cloudprovider/registry" "github.com/aws/karpenter/pkg/controllers/provisioning" - "github.com/aws/karpenter/pkg/controllers/scheduling" + "github.com/aws/karpenter/pkg/controllers/selection" "github.com/aws/karpenter/pkg/test" "sigs.k8s.io/controller-runtime/pkg/client" @@ -39,7 +39,7 @@ import ( var ctx context.Context var provisioner *v1alpha5.Provisioner var provisioners *provisioning.Controller -var scheduler *scheduling.Controller +var selectionController *selection.Controller var env *test.Environment func TestAPIs(t *testing.T) { @@ -53,7 +53,7 @@ var _ = BeforeSuite(func() { cloudProvider := &fake.CloudProvider{} registry.RegisterOrDie(ctx, cloudProvider) provisioners = provisioning.NewController(ctx, e.Client, corev1.NewForConfigOrDie(e.Config), cloudProvider) - scheduler = scheduling.NewController(e.Client, provisioners) + selectionController = selection.NewController(e.Client, provisioners) }) Expect(env.Start()).To(Succeed(), "Failed to start environment") }) @@ -78,20 +78,20 @@ var _ = Describe("Combined Constraints", func() { Context("Custom Labels", func() { It("should schedule unconstrained pods that don't have matching node selectors", func() { provisioner.Spec.Labels = map[string]string{"test-key": "test-value"} - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod())[0] + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod())[0] node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) }) It("should not schedule pods that have conflicting node selectors", func() { provisioner.Spec.Labels = map[string]string{"test-key": "test-value"} - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{NodeSelector: map[string]string{"test-key": "different-value"}}, ))[0] ExpectNotScheduled(ctx, env.Client, pod) }) It("should schedule pods that have matching requirements", func() { provisioner.Spec.Labels = map[string]string{"test-key": "test-value"} - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}, }}, @@ -101,7 +101,7 @@ var _ = Describe("Combined Constraints", func() { }) It("should not schedule pods that have conflicting requirements", func() { provisioner.Spec.Labels = map[string]string{"test-key": "test-value"} - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, }}, @@ -110,7 +110,7 @@ var _ = Describe("Combined Constraints", func() { }) It("should schedule pods that have matching preferences", func() { provisioner.Spec.Labels = map[string]string{"test-key": "test-value"} - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{NodePreferences: []v1.NodeSelectorRequirement{ {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value", "test-value"}}, }}, @@ -120,7 +120,7 @@ var _ = Describe("Combined Constraints", func() { }) It("should not schedule pods with have conflicting preferences", func() { provisioner.Spec.Labels = map[string]string{"test-key": "test-value"} - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{NodePreferences: []v1.NodeSelectorRequirement{ {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, }}, @@ -131,13 +131,13 @@ var _ = Describe("Combined Constraints", func() { Context("Well Known Labels", func() { It("should use provisioner constraints", func() { provisioner.Spec.Requirements = v1alpha5.Requirements{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}} - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod())[0] + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod())[0] node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) }) It("should use node selectors", func() { provisioner.Spec.Requirements = v1alpha5.Requirements{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, ))[0] node := ExpectScheduled(ctx, env.Client, pod) @@ -145,20 +145,20 @@ var _ = Describe("Combined Constraints", func() { }) It("should not schedule the pod if nodeselector unknown", func() { provisioner.Spec.Requirements = v1alpha5.Requirements{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "unknown"}}, ))[0] ExpectNotScheduled(ctx, env.Client, pod) }) It("should not schedule if node selector outside of provisioner constraints", func() { provisioner.Spec.Requirements = v1alpha5.Requirements{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, ))[0] ExpectNotScheduled(ctx, env.Client, pod) }) It("should schedule compatible requirements with Operator=In", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, }}, @@ -167,7 +167,7 @@ var _ = Describe("Combined Constraints", func() { Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) }) It("should not schedule incompatible preferences and requirements with Operator=In", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}, }}, @@ -175,7 +175,7 @@ var _ = Describe("Combined Constraints", func() { ExpectNotScheduled(ctx, env.Client, pod) }) It("should schedule compatible requirements with Operator=NotIn", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "unknown"}}, }}, @@ -184,7 +184,7 @@ var _ = Describe("Combined Constraints", func() { Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) }) It("should not schedule incompatible preferences and requirements with Operator=NotIn", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{ NodeRequirements: []v1.NodeSelectorRequirement{ {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}, @@ -193,7 +193,7 @@ var _ = Describe("Combined Constraints", func() { ExpectNotScheduled(ctx, env.Client, pod) }) It("should schedule compatible preferences and requirements with Operator=In", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{ NodeRequirements: []v1.NodeSelectorRequirement{ {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, @@ -205,7 +205,7 @@ var _ = Describe("Combined Constraints", func() { Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) }) It("should not schedule incompatible preferences and requirements with Operator=In", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{ NodeRequirements: []v1.NodeSelectorRequirement{ {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, @@ -217,7 +217,7 @@ var _ = Describe("Combined Constraints", func() { }) It("should schedule compatible preferences and requirements with Operator=NotIn", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{ NodeRequirements: []v1.NodeSelectorRequirement{ {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, @@ -229,7 +229,7 @@ var _ = Describe("Combined Constraints", func() { Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) }) It("should not schedule incompatible preferences and requirements with Operator=NotIn", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{ NodeRequirements: []v1.NodeSelectorRequirement{ {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, @@ -240,7 +240,7 @@ var _ = Describe("Combined Constraints", func() { ExpectNotScheduled(ctx, env.Client, pod) }) It("should schedule compatible node selectors, preferences and requirements", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{ NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}, NodeRequirements: []v1.NodeSelectorRequirement{ @@ -253,7 +253,7 @@ var _ = Describe("Combined Constraints", func() { Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) }) It("should not schedule incompatible node selectors, preferences and requirements", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{ NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}, NodeRequirements: []v1.NodeSelectorRequirement{ @@ -265,7 +265,7 @@ var _ = Describe("Combined Constraints", func() { ExpectNotScheduled(ctx, env.Client, pod) }) It("should combine multidimensional node selectors, preferences and requirements", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{ NodeSelector: map[string]string{ v1.LabelTopologyZone: "test-zone-3", @@ -286,7 +286,7 @@ var _ = Describe("Combined Constraints", func() { Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "arm-instance-type")) }) It("should not combine incompatible multidimensional node selectors, preferences and requirements", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{ NodeSelector: map[string]string{ v1.LabelTopologyZone: "test-zone-3", @@ -321,11 +321,11 @@ var _ = Describe("Preferential Fallback", func() { }}, }}}} // Don't relax - pod = ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, pod)[0] + pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0] ExpectNotScheduled(ctx, env.Client, pod) // Don't relax - pod = ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, pod)[0] + pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0] ExpectNotScheduled(ctx, env.Client, pod) }) It("should relax multiple terms", func() { @@ -345,13 +345,13 @@ var _ = Describe("Preferential Fallback", func() { }}, }}}} // Remove first term - pod = ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, pod)[0] + pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0] ExpectNotScheduled(ctx, env.Client, pod) // Remove second term - pod = ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, pod)[0] + pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0] ExpectNotScheduled(ctx, env.Client, pod) // Success - pod = ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, pod)[0] + pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0] node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-1")) }) @@ -372,13 +372,13 @@ var _ = Describe("Preferential Fallback", func() { }, }}} // Remove first term - pod = ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, pod)[0] + pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0] ExpectNotScheduled(ctx, env.Client, pod) // Remove second term - pod = ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, pod)[0] + pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0] ExpectNotScheduled(ctx, env.Client, pod) // Success - pod = ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, pod)[0] + pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0] ExpectScheduled(ctx, env.Client, pod) }) It("should relax to use lighter weights", func() { @@ -402,10 +402,10 @@ var _ = Describe("Preferential Fallback", func() { }, }}} // Remove heaviest term - pod = ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, pod)[0] + pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0] ExpectNotScheduled(ctx, env.Client, pod) // Success - pod = ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, pod)[0] + pod = ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, pod)[0] node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) }) @@ -416,7 +416,7 @@ var _ = Describe("Topology", func() { labels := map[string]string{"test": "test"} It("should ignore unknown topology keys", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{Labels: labels, TopologySpreadConstraints: []v1.TopologySpreadConstraint{{ TopologyKey: "unknown", WhenUnsatisfiable: v1.DoNotSchedule, @@ -435,7 +435,7 @@ var _ = Describe("Topology", func() { LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, MaxSkew: 1, }} - ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod(test.PodOptions{Labels: labels, TopologySpreadConstraints: topology}), test.UnschedulablePod(test.PodOptions{Labels: labels, TopologySpreadConstraints: topology}), test.UnschedulablePod(test.PodOptions{Labels: labels, TopologySpreadConstraints: topology}), @@ -451,7 +451,7 @@ var _ = Describe("Topology", func() { LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, MaxSkew: 1, }} - ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod(test.PodOptions{Labels: labels, TopologySpreadConstraints: topology}), test.UnschedulablePod(test.PodOptions{Labels: labels, TopologySpreadConstraints: topology}), test.UnschedulablePod(test.PodOptions{Labels: labels, TopologySpreadConstraints: topology}), @@ -470,7 +470,7 @@ var _ = Describe("Topology", func() { LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, MaxSkew: 1, }} - ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.Pod(test.PodOptions{Labels: labels}), // ignored, pending test.Pod(test.PodOptions{}), // ignored, missing labels test.Pod(test.PodOptions{Labels: labels, NodeName: firstNode.Name}), @@ -494,7 +494,7 @@ var _ = Describe("Topology", func() { LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, MaxSkew: 1, }} - ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod(test.PodOptions{Labels: labels, TopologySpreadConstraints: topology}), test.UnschedulablePod(test.PodOptions{Labels: labels, TopologySpreadConstraints: topology}), test.UnschedulablePod(test.PodOptions{Labels: labels, TopologySpreadConstraints: topology}), @@ -509,7 +509,7 @@ var _ = Describe("Topology", func() { LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, MaxSkew: 4, }} - ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod(test.PodOptions{Labels: labels, TopologySpreadConstraints: topology}), test.UnschedulablePod(test.PodOptions{Labels: labels, TopologySpreadConstraints: topology}), test.UnschedulablePod(test.PodOptions{Labels: labels, TopologySpreadConstraints: topology}), @@ -532,25 +532,25 @@ var _ = Describe("Topology", func() { LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, MaxSkew: 3, }} - ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, MakePods(2, test.PodOptions{Labels: labels, TopologySpreadConstraints: topology})..., ) ExpectSkew(env.Client, v1.LabelTopologyZone).To(ConsistOf(1, 1)) ExpectSkew(env.Client, v1.LabelHostname).ToNot(ContainElements(BeNumerically(">", 3))) - ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, MakePods(3, test.PodOptions{Labels: labels, TopologySpreadConstraints: topology})..., ) ExpectSkew(env.Client, v1.LabelTopologyZone).To(ConsistOf(2, 2, 1)) ExpectSkew(env.Client, v1.LabelHostname).ToNot(ContainElements(BeNumerically(">", 3))) - ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, MakePods(5, test.PodOptions{Labels: labels, TopologySpreadConstraints: topology})..., ) ExpectSkew(env.Client, v1.LabelTopologyZone).To(ConsistOf(4, 3, 3)) ExpectSkew(env.Client, v1.LabelHostname).ToNot(ContainElements(BeNumerically(">", 3))) - ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, MakePods(11, test.PodOptions{Labels: labels, TopologySpreadConstraints: topology})..., ) ExpectSkew(env.Client, v1.LabelTopologyZone).To(ConsistOf(7, 7, 7)) @@ -567,7 +567,7 @@ var _ = Describe("Topology", func() { LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, MaxSkew: 1, }} - ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, append( MakePods(5, test.PodOptions{ Labels: labels, @@ -590,7 +590,7 @@ var _ = Describe("Topology", func() { LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, MaxSkew: 1, }} - ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, append( + ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, append( MakePods(6, test.PodOptions{ Labels: labels, TopologySpreadConstraints: topology, @@ -607,7 +607,7 @@ var _ = Describe("Topology", func() { })..., )...) ExpectSkew(env.Client, v1.LabelTopologyZone).To(ConsistOf(4, 3)) - ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, MakePods(5, test.PodOptions{ Labels: labels, TopologySpreadConstraints: topology, @@ -621,7 +621,7 @@ var _ = Describe("Topology", func() { var _ = Describe("Taints", func() { It("should taint nodes with provisioner taints", func() { provisioner.Spec.Taints = []v1.Taint{{Key: "test", Value: "bar", Effect: v1.TaintEffectNoSchedule}} - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{Tolerations: []v1.Toleration{{Effect: v1.TaintEffectNoSchedule, Operator: v1.TolerationOpExists}}}, ))[0] node := ExpectScheduled(ctx, env.Client, pod) @@ -629,7 +629,7 @@ var _ = Describe("Taints", func() { }) It("should schedule pods that tolerate provisioner constraints", func() { provisioner.Spec.Taints = []v1.Taint{{Key: "test-key", Value: "test-value", Effect: v1.TaintEffectNoSchedule}} - for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, // Tolerates with OpExists test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "test-key", Operator: v1.TolerationOpExists, Effect: v1.TaintEffectNoSchedule}}}), // Tolerates with OpEqual @@ -637,7 +637,7 @@ var _ = Describe("Taints", func() { ) { ExpectScheduled(ctx, env.Client, pod) } - for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, // Missing toleration test.UnschedulablePod(), // key mismatch with OpExists @@ -649,7 +649,7 @@ var _ = Describe("Taints", func() { } }) It("should not generate taints for OpExists", func() { - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "test-key", Operator: v1.TolerationOpExists, Effect: v1.TaintEffectNoExecute}}}), )[0] node := ExpectScheduled(ctx, env.Client, pod) @@ -657,7 +657,7 @@ var _ = Describe("Taints", func() { }) It("should generate taints for pod tolerations", func() { Skip("until taint generation is reimplmented") - pods := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + pods := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, // Matching pods schedule together on a node with a matching taint test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{ {Key: "test-key", Operator: v1.TolerationOpEqual, Value: "test-value", Effect: v1.TaintEffectNoSchedule}}, @@ -702,39 +702,6 @@ var _ = Describe("Taints", func() { }) }) -var _ = Describe("Multiple Provisioners", func() { - It("should schedule to an explicitly selected provisioner", func() { - provisioner2 := provisioner.DeepCopy() - provisioner2.Name = "provisioner2" - ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner2) - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.ProvisionerNameLabelKey: provisioner2.Name}}), - )[0] - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(provisioner2.Name)) - }) - It("should schedule to a provisioner by labels", func() { - provisioner2 := provisioner.DeepCopy() - provisioner2.Name = "provisioner2" - provisioner2.Spec.Labels = map[string]string{"foo": "bar"} - provisioner.Spec.Labels = map[string]string{"foo": "baz"} - ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner2) - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{"foo": "bar"}}), - )[0] - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(provisioner2.Name)) - }) - It("should prioritize provisioners alphabetically if multiple match", func() { - provisioner2 := provisioner.DeepCopy() - provisioner2.Name = "aaaaaaaaa" - ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner2) - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod())[0] - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(provisioner2.Name)) - }) -}) - func MakePods(count int, options test.PodOptions) (pods []*v1.Pod) { for i := 0; i < count; i++ { pods = append(pods, test.UnschedulablePod(options)) diff --git a/pkg/controllers/provisioning/suite_test.go b/pkg/controllers/provisioning/suite_test.go index 456d83a50b1f..4f480b2b2918 100644 --- a/pkg/controllers/provisioning/suite_test.go +++ b/pkg/controllers/provisioning/suite_test.go @@ -22,7 +22,7 @@ import ( "github.com/aws/karpenter/pkg/cloudprovider/fake" "github.com/aws/karpenter/pkg/cloudprovider/registry" "github.com/aws/karpenter/pkg/controllers/provisioning" - "github.com/aws/karpenter/pkg/controllers/scheduling" + "github.com/aws/karpenter/pkg/controllers/selection" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/pkg/utils/resources" @@ -38,8 +38,8 @@ import ( ) var ctx context.Context -var provisioners *provisioning.Controller -var scheduler *scheduling.Controller +var provisioningController *provisioning.Controller +var selectionController *selection.Controller var env *test.Environment func TestAPIs(t *testing.T) { @@ -52,8 +52,8 @@ var _ = BeforeSuite(func() { env = test.NewEnvironment(ctx, func(e *test.Environment) { cloudProvider := &fake.CloudProvider{} registry.RegisterOrDie(ctx, cloudProvider) - provisioners = provisioning.NewController(ctx, e.Client, corev1.NewForConfigOrDie(e.Config), cloudProvider) - scheduler = scheduling.NewController(e.Client, provisioners) + provisioningController = provisioning.NewController(ctx, e.Client, corev1.NewForConfigOrDie(e.Config), cloudProvider) + selectionController = selection.NewController(e.Client, provisioningController) }) Expect(env.Start()).To(Succeed(), "Failed to start environment") }) @@ -81,12 +81,12 @@ var _ = Describe("Provisioning", func() { }) AfterEach(func() { - ExpectProvisioningCleanedUp(ctx, env.Client, provisioners) + ExpectProvisioningCleanedUp(ctx, env.Client, provisioningController) }) Context("Reconcilation", func() { It("should provision nodes", func() { - pods := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod()) + pods := ExpectProvisioned(ctx, env.Client, selectionController, provisioningController, provisioner, test.UnschedulablePod()) nodes := &v1.NodeList{} Expect(env.Client.List(ctx, nodes)).To(Succeed()) Expect(len(nodes.Items)).To(Equal(1)) @@ -123,15 +123,15 @@ var _ = Describe("Provisioning", func() { // Ignored, label selector does not match test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{"foo": "bar"}}), } - for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, schedulable...) { + for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioningController, provisioner, schedulable...) { ExpectScheduled(ctx, env.Client, pod) } - for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, unschedulable...) { + for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioningController, provisioner, unschedulable...) { ExpectNotScheduled(ctx, env.Client, pod) } }) It("should provision nodes for accelerators", func() { - for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioningController, provisioner, test.UnschedulablePod(test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{Limits: v1.ResourceList{resources.NvidiaGPU: resource.MustParse("1")}}, }), @@ -153,7 +153,7 @@ var _ = Describe("Provisioning", func() { }, } provisioner.Spec.Limits.Resources[v1.ResourceCPU] = resource.MustParse("20") - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod())[0] + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioningController, provisioner, test.UnschedulablePod())[0] ExpectNotScheduled(ctx, env.Client, pod) }) }) @@ -164,7 +164,7 @@ var _ = Describe("Provisioning", func() { ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }}, )) - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioningController, provisioner, test.UnschedulablePod( test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }, @@ -179,7 +179,7 @@ var _ = Describe("Provisioning", func() { ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}}, }}, )) - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod(test.PodOptions{}))[0] + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioningController, provisioner, test.UnschedulablePod(test.PodOptions{}))[0] ExpectNotScheduled(ctx, env.Client, pod) }) It("should ignore daemonsets without matching tolerations", func() { @@ -189,7 +189,7 @@ var _ = Describe("Provisioning", func() { ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }}, )) - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioningController, provisioner, test.UnschedulablePod( test.PodOptions{ Tolerations: []v1.Toleration{{Operator: v1.TolerationOperator(v1.NodeSelectorOpExists)}}, ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, @@ -206,7 +206,7 @@ var _ = Describe("Provisioning", func() { ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }}, )) - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioningController, provisioner, test.UnschedulablePod( test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }, @@ -222,7 +222,7 @@ var _ = Describe("Provisioning", func() { ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }}, )) - pod := ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod( + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioningController, provisioner, test.UnschedulablePod( test.PodOptions{ NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}}, ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, @@ -236,7 +236,7 @@ var _ = Describe("Provisioning", func() { Context("Labels", func() { It("should label nodes", func() { provisioner.Spec.Labels = map[string]string{"test-key": "test-value", "test-key-2": "test-value-2"} - for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod()) { + for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioningController, provisioner, test.UnschedulablePod()) { node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.ProvisionerNameLabelKey, provisioner.Name)) Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) @@ -249,7 +249,7 @@ var _ = Describe("Provisioning", func() { Context("Taints", func() { It("should apply unready taints", func() { ExpectCreated(ctx, env.Client, provisioner) - for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, test.UnschedulablePod()) { + for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioningController, provisioner, test.UnschedulablePod()) { node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Spec.Taints).To(ContainElement(v1.Taint{Key: v1alpha5.NotReadyTaintKey, Effect: v1.TaintEffectNoSchedule})) } diff --git a/pkg/controllers/scheduling/controller.go b/pkg/controllers/selection/controller.go similarity index 99% rename from pkg/controllers/scheduling/controller.go rename to pkg/controllers/selection/controller.go index aae40add24dc..3e40d223c153 100644 --- a/pkg/controllers/scheduling/controller.go +++ b/pkg/controllers/selection/controller.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package scheduling +package selection import ( "context" diff --git a/pkg/controllers/scheduling/preferences.go b/pkg/controllers/selection/preferences.go similarity index 99% rename from pkg/controllers/scheduling/preferences.go rename to pkg/controllers/selection/preferences.go index 639b266b6bb3..e141b9e58dca 100644 --- a/pkg/controllers/scheduling/preferences.go +++ b/pkg/controllers/selection/preferences.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package scheduling +package selection import ( "context" diff --git a/pkg/controllers/selection/suite_test.go b/pkg/controllers/selection/suite_test.go new file mode 100644 index 000000000000..1cb864896dc2 --- /dev/null +++ b/pkg/controllers/selection/suite_test.go @@ -0,0 +1,106 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package selection_test + +import ( + "context" + "testing" + + "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" + "github.com/aws/karpenter/pkg/cloudprovider/fake" + "github.com/aws/karpenter/pkg/cloudprovider/registry" + "github.com/aws/karpenter/pkg/controllers/provisioning" + "github.com/aws/karpenter/pkg/controllers/selection" + "github.com/aws/karpenter/pkg/test" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + + . "github.com/aws/karpenter/pkg/test/expectations" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + . "knative.dev/pkg/logging/testing" +) + +var ctx context.Context +var provisioner *v1alpha5.Provisioner +var provisioners *provisioning.Controller +var selectionController *selection.Controller +var env *test.Environment + +func TestAPIs(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "Controllers/Scheduling") +} + +var _ = BeforeSuite(func() { + env = test.NewEnvironment(ctx, func(e *test.Environment) { + cloudProvider := &fake.CloudProvider{} + registry.RegisterOrDie(ctx, cloudProvider) + provisioners = provisioning.NewController(ctx, e.Client, corev1.NewForConfigOrDie(e.Config), cloudProvider) + selectionController = selection.NewController(e.Client, provisioners) + }) + Expect(env.Start()).To(Succeed(), "Failed to start environment") +}) + +var _ = AfterSuite(func() { + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) + +var _ = BeforeEach(func() { + provisioner = &v1alpha5.Provisioner{ + ObjectMeta: metav1.ObjectMeta{Name: v1alpha5.DefaultProvisioner.Name}, + Spec: v1alpha5.ProvisionerSpec{}, + } + provisioner.SetDefaults(ctx) +}) + +var _ = AfterEach(func() { + ExpectProvisioningCleanedUp(ctx, env.Client, provisioners) +}) + +var _ = Describe("Multiple Provisioners", func() { + It("should schedule to an explicitly selected provisioner", func() { + provisioner2 := provisioner.DeepCopy() + provisioner2.Name = "provisioner2" + ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner2) + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.ProvisionerNameLabelKey: provisioner2.Name}}), + )[0] + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(provisioner2.Name)) + }) + It("should schedule to a provisioner by labels", func() { + provisioner2 := provisioner.DeepCopy() + provisioner2.Name = "provisioner2" + provisioner2.Spec.Labels = map[string]string{"foo": "bar"} + provisioner.Spec.Labels = map[string]string{"foo": "baz"} + ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner2) + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{"foo": "bar"}}), + )[0] + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(provisioner2.Name)) + }) + It("should prioritize provisioners alphabetically if multiple match", func() { + provisioner2 := provisioner.DeepCopy() + provisioner2.Name = "aaaaaaaaa" + ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner2) + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod())[0] + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(provisioner2.Name)) + }) +}) diff --git a/pkg/test/expectations/expectations.go b/pkg/test/expectations/expectations.go index 975df8b870ae..760740654202 100644 --- a/pkg/test/expectations/expectations.go +++ b/pkg/test/expectations/expectations.go @@ -33,7 +33,7 @@ import ( "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" "github.com/aws/karpenter/pkg/controllers/provisioning" - "github.com/aws/karpenter/pkg/controllers/scheduling" + "github.com/aws/karpenter/pkg/controllers/selection" ) const ( @@ -157,7 +157,7 @@ func ExpectProvisioningCleanedUp(ctx context.Context, c client.Client, controlle } } -func ExpectProvisioned(ctx context.Context, c client.Client, scheduler *scheduling.Controller, provisioners *provisioning.Controller, provisioner *v1alpha5.Provisioner, pods ...*v1.Pod) (result []*v1.Pod) { +func ExpectProvisioned(ctx context.Context, c client.Client, selectionController *selection.Controller, provisioningController *provisioning.Controller, provisioner *v1alpha5.Provisioner, pods ...*v1.Pod) (result []*v1.Pod) { // Persist objects ExpectApplied(ctx, c, provisioner) ExpectStatusUpdated(ctx, c, provisioner) @@ -165,12 +165,12 @@ func ExpectProvisioned(ctx context.Context, c client.Client, scheduler *scheduli ExpectCreatedWithStatus(ctx, c, pod) } // Wait for reconcile - ExpectReconcileSucceeded(ctx, provisioners, client.ObjectKeyFromObject(provisioner)) + ExpectReconcileSucceeded(ctx, provisioningController, client.ObjectKeyFromObject(provisioner)) wg := sync.WaitGroup{} for _, pod := range pods { wg.Add(1) go func(pod *v1.Pod) { - scheduler.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(pod)}) + selectionController.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(pod)}) wg.Done() }(pod) } diff --git a/pkg/utils/pod/scheduling.go b/pkg/utils/pod/scheduling.go index d3fcf6e563b4..3ada7fef0aba 100644 --- a/pkg/utils/pod/scheduling.go +++ b/pkg/utils/pod/scheduling.go @@ -28,6 +28,10 @@ func FailedToSchedule(pod *v1.Pod) bool { return false } +func IsScheduled(pod *v1.Pod) bool { + return pod.Spec.NodeName != "" +} + func HasFinished(pod *v1.Pod) bool { return pod.Status.Phase == v1.PodFailed || pod.Status.Phase == v1.PodSucceeded }