-
Notifications
You must be signed in to change notification settings - Fork 983
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implemented support for multiple provisioners
- Loading branch information
Showing
11 changed files
with
439 additions
and
22 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| /* | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
| package provisioning | ||
|
|
||
| import ( | ||
| "context" | ||
| "sync" | ||
| "time" | ||
|
|
||
| v1 "k8s.io/api/core/v1" | ||
| "k8s.io/apimachinery/pkg/api/errors" | ||
| corev1 "k8s.io/client-go/kubernetes/typed/core/v1" | ||
| "knative.dev/pkg/logging" | ||
| controllerruntime "sigs.k8s.io/controller-runtime" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
| "sigs.k8s.io/controller-runtime/pkg/controller" | ||
| "sigs.k8s.io/controller-runtime/pkg/manager" | ||
| "sigs.k8s.io/controller-runtime/pkg/reconcile" | ||
|
|
||
| "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha5" | ||
| "github.com/awslabs/karpenter/pkg/cloudprovider" | ||
| "github.com/awslabs/karpenter/pkg/controllers/allocation" | ||
| "github.com/awslabs/karpenter/pkg/controllers/allocation/binpacking" | ||
| "github.com/awslabs/karpenter/pkg/controllers/allocation/scheduling" | ||
| "github.com/awslabs/karpenter/pkg/utils/functional" | ||
| ) | ||
|
|
||
| // Controller for the resource | ||
| type Controller struct { | ||
| // TODO docs | ||
| ctx context.Context | ||
| provisioners *sync.Map | ||
| scheduler *scheduling.Scheduler | ||
| launcher *allocation.Launcher | ||
| kubeClient client.Client | ||
| cloudProvider cloudprovider.CloudProvider | ||
| } | ||
|
|
||
| // NewController is a constructor | ||
| func NewController(ctx context.Context, kubeClient client.Client, coreV1Client corev1.CoreV1Interface, cloudProvider cloudprovider.CloudProvider, provisioners *sync.Map) *Controller { | ||
| return &Controller{ | ||
| ctx: ctx, | ||
| provisioners: provisioners, | ||
| kubeClient: kubeClient, | ||
| cloudProvider: cloudProvider, | ||
| scheduler: scheduling.NewScheduler(kubeClient, cloudProvider), | ||
| launcher: &allocation.Launcher{KubeClient: kubeClient, CoreV1Client: coreV1Client, CloudProvider: cloudProvider, Packer: &binpacking.Packer{}}, | ||
| } | ||
| } | ||
|
|
||
| // Reconcile a control loop for the resource | ||
| func (c *Controller) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { | ||
| ctx = logging.WithLogger(ctx, logging.FromContext(ctx).Named("provisioning").With("provisioner", req.Name)) | ||
| provisioner := &v1alpha5.Provisioner{} | ||
| if err := c.kubeClient.Get(ctx, req.NamespacedName, provisioner); err != nil { | ||
| if errors.IsNotFound(err) { | ||
| c.DeleteProvisioner(ctx, req) | ||
| return reconcile.Result{}, nil | ||
| } | ||
| return reconcile.Result{}, err | ||
| } | ||
| instanceTypes, err := c.cloudProvider.GetInstanceTypes(ctx, &provisioner.Spec.Constraints) | ||
| if err != nil { | ||
| return reconcile.Result{}, err | ||
| } | ||
| provisioner.Spec.Labels = functional.UnionStringMaps( | ||
| provisioner.Spec.Labels, | ||
| map[string]string{v1alpha5.ProvisionerNameLabelKey: provisioner.Name}, | ||
| ) | ||
| provisioner.Spec.Requirements = provisioner.Spec.Requirements. | ||
| With(scheduling.GlobalRequirements(instanceTypes)). // TODO(etarn) move GlobalRequirements to this file | ||
| With(v1alpha5.LabelRequirements(provisioner.Spec.Labels)) | ||
|
|
||
| // Stop the existing provisioner if exists. This will drain the current | ||
| // workflow and replace it with an updated provisioner configuration. | ||
| // Requeue in order to discover any changes from GetInstanceTypes. | ||
| c.DeleteProvisioner(ctx, req) | ||
| c.CreateProvisioner(ctx, provisioner, instanceTypes) | ||
| return reconcile.Result{RequeueAfter: 5 * time.Minute}, nil | ||
| } | ||
|
|
||
| // DeleteProvisioner stops and removes a provisioner | ||
| func (c *Controller) DeleteProvisioner(ctx context.Context, req reconcile.Request) { | ||
| if p, ok := c.provisioners.LoadAndDelete(req.String()); ok { | ||
| p.(*Provisioner).stop(ctx) | ||
| } | ||
| } | ||
|
|
||
| func (c *Controller) CreateProvisioner(ctx context.Context, provisioner *v1alpha5.Provisioner, instanceTypes []cloudprovider.InstanceType) *Provisioner { | ||
| ctx, cancelFunc := context.WithCancel(ctx) | ||
| p := &Provisioner{ | ||
| Provisioner: provisioner, | ||
| instanceTypes: instanceTypes, | ||
| pods: make(chan *v1.Pod), | ||
| results: make(chan error), | ||
| scheduler: c.scheduler, | ||
| launcher: c.launcher, | ||
| cancelFunc: cancelFunc, | ||
| } | ||
| c.provisioners.Store(provisioner.Name, p) | ||
| go p.start(ctx) | ||
| return p | ||
| } | ||
|
|
||
| func (c *Controller) Register(ctx context.Context, m manager.Manager) error { | ||
| return controllerruntime. | ||
| NewControllerManagedBy(m). | ||
| Named("provisioning"). | ||
| For(&v1alpha5.Provisioner{}). | ||
| WithOptions(controller.Options{MaxConcurrentReconciles: 10}). | ||
| Complete(c) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| /* | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package provisioning | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "time" | ||
|
|
||
| "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha5" | ||
| "github.com/awslabs/karpenter/pkg/cloudprovider" | ||
| "github.com/awslabs/karpenter/pkg/controllers/allocation" | ||
| "github.com/awslabs/karpenter/pkg/controllers/allocation/scheduling" | ||
| v1 "k8s.io/api/core/v1" | ||
| "knative.dev/pkg/logging" | ||
| ) | ||
|
|
||
| var ( | ||
| MaxBatchWindow = time.Second * 10 | ||
| MinBatchWindow = time.Second * 1 | ||
| ) | ||
|
|
||
| // Provisioner is a stateful, threadsafe controller. Pods are enqueud and | ||
| // batched, capacity is launched, and pods are bound to the new capacity. | ||
| type Provisioner struct { | ||
| // State | ||
| *v1alpha5.Provisioner | ||
| instanceTypes []cloudprovider.InstanceType | ||
| pods chan *v1.Pod | ||
| results chan error | ||
| cancelFunc context.CancelFunc | ||
|
|
||
| // Dependencies | ||
| scheduler *scheduling.Scheduler | ||
| launcher *allocation.Launcher | ||
| } | ||
|
|
||
| // Start the provisioner's loop | ||
| func (p *Provisioner) start(ctx context.Context) { | ||
| logging.FromContext(ctx).Info("Starting provisioner") | ||
| for { | ||
| select { | ||
| case <-ctx.Done(): | ||
| return | ||
| default: | ||
| if err := p.provision(ctx); err != nil { | ||
| logging.FromContext(ctx).Errorf("Provisioning failed, %s", err.Error()) | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func (p *Provisioner) stop(ctx context.Context) { | ||
| logging.FromContext(ctx).Info("Stopping provisioner") | ||
| p.cancelFunc() | ||
| close(p.pods) | ||
| close(p.results) | ||
| } | ||
|
|
||
| func (p *Provisioner) provision(ctx context.Context) (err error) { | ||
| // Wait for a batch of pods | ||
| pods := p.Batch(ctx) | ||
| // Send results | ||
| defer func() { | ||
| for i := 0; i < len(pods); i++ { | ||
| p.results <- err | ||
| } | ||
| }() | ||
| // Separate pods by scheduling constraints | ||
| schedules, err := p.scheduler.Solve(ctx, p.Provisioner, p.instanceTypes, pods) | ||
| if err != nil { | ||
| return fmt.Errorf("solving scheduling constraints, %w", err) | ||
| } | ||
| // Launch capacity and bind pods | ||
| if err := p.launcher.Launch(ctx, schedules, p.instanceTypes); err != nil { | ||
| return fmt.Errorf("launching capacity, %w", err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (p *Provisioner) Enqueue(ctx context.Context, pod *v1.Pod) error { | ||
| p.pods <- pod | ||
| return <-p.results | ||
| } | ||
|
|
||
| func (p *Provisioner) Batch(ctx context.Context) (pods []*v1.Pod) { | ||
| logging.FromContext(ctx).Infof("Waiting for unschedulable pods") | ||
| pods = append(pods, <-p.pods) | ||
| timeout := time.NewTimer(MaxBatchWindow) | ||
| idle := time.NewTimer(MinBatchWindow) | ||
| start := time.Now() | ||
| defer func() { | ||
| logging.FromContext(ctx).Infof("Batched %d pods in %s", len(pods), time.Since(start)) | ||
| }() | ||
| for { | ||
| select { | ||
| case <-ctx.Done(): | ||
| return pods | ||
| case <-timeout.C: | ||
| return pods | ||
| case <-idle.C: | ||
| return pods | ||
| case pod := <-p.pods: | ||
| idle.Reset(MinBatchWindow) | ||
| pods = append(pods, pod) | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.