From a94ccefe0cf3529ae13bd0d0f1c10c5267e7d9f1 Mon Sep 17 00:00:00 2001 From: Prateek Gogia Date: Fri, 22 Jan 2021 17:36:41 -0600 Subject: [PATCH 1/2] Adding fleet API client --- pkg/cloudprovider/aws/ec2_fleet.go | 88 +++++++++++++++++++++ pkg/cloudprovider/aws/ec2_fleet_test.go | 101 ++++++++++++++++++++++++ pkg/cloudprovider/aws/factory.go | 9 +++ pkg/cloudprovider/types.go | 25 ++++++ 4 files changed, 223 insertions(+) create mode 100644 pkg/cloudprovider/aws/ec2_fleet.go create mode 100644 pkg/cloudprovider/aws/ec2_fleet_test.go diff --git a/pkg/cloudprovider/aws/ec2_fleet.go b/pkg/cloudprovider/aws/ec2_fleet.go new file mode 100644 index 000000000000..ac19a1a62009 --- /dev/null +++ b/pkg/cloudprovider/aws/ec2_fleet.go @@ -0,0 +1,88 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "context" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" +) + +type instanceConfig struct { + ec2Iface ec2iface.EC2API + templateConfig *ec2.FleetLaunchTemplateConfigRequest + capacitySpec *ec2.TargetCapacitySpecificationRequest + instanceID string +} + +func NewFleetRequest(templateID, templateVersion string, client ec2iface.EC2API) *instanceConfig { + + return &instanceConfig{ + ec2Iface: client, + templateConfig: &ec2.FleetLaunchTemplateConfigRequest{ + LaunchTemplateSpecification: &ec2.FleetLaunchTemplateSpecificationRequest{ + LaunchTemplateId: aws.String(templateID), + Version: aws.String(templateVersion), + }, + Overrides: []*ec2.FleetLaunchTemplateOverridesRequest{ + &ec2.FleetLaunchTemplateOverridesRequest{}, + }, + }, + capacitySpec: &ec2.TargetCapacitySpecificationRequest{ + DefaultTargetCapacityType: aws.String(ec2.DefaultTargetCapacityTypeOnDemand), + }, + } +} + +func (cfg *instanceConfig) SetAvailabilityZone(zone string) { + cfg.templateConfig.Overrides[0].AvailabilityZone = aws.String(zone) +} + +func (cfg *instanceConfig) SetSubnet(subnetID string) { + cfg.templateConfig.Overrides[0].SubnetId = aws.String(subnetID) +} + +func (cfg *instanceConfig) SetOnDemandCapacity(targetCap, totalCap int64) { + cfg.capacitySpec.OnDemandTargetCapacity = aws.Int64(targetCap) + cfg.capacitySpec.TotalTargetCapacity = aws.Int64(totalCap) +} + +func (cfg *instanceConfig) SetInstanceType(instanceType string) { + cfg.capacitySpec.DefaultTargetCapacityType = aws.String(instanceType) +} + +func (cfg *instanceConfig) Create(ctx context.Context) error { + return cfg.validateAndCreate(ctx) +} + +func (cfg *instanceConfig) validateAndCreate(ctx context.Context) error { + input := &ec2.CreateFleetInput{ + LaunchTemplateConfigs: []*ec2.FleetLaunchTemplateConfigRequest{cfg.templateConfig}, + TargetCapacitySpecification: cfg.capacitySpec, + Type: aws.String(ec2.FleetTypeInstant), + } + if err := input.Validate(); err != nil { + return err + } + output, err := cfg.ec2Iface.CreateFleetWithContext(ctx, input) + if err != nil { + return err + } + // TODO Get instanceID from the output + _ = output + return nil +} diff --git a/pkg/cloudprovider/aws/ec2_fleet_test.go b/pkg/cloudprovider/aws/ec2_fleet_test.go new file mode 100644 index 000000000000..f51809b08619 --- /dev/null +++ b/pkg/cloudprovider/aws/ec2_fleet_test.go @@ -0,0 +1,101 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/awslabs/karpenter/pkg/cloudprovider/aws/fake" +) + +var ( + launchTemplate = "test-templateID" + version = "10" + az = "us-east-2b" + subnetID = "subnet-12345dead789" +) + +var FleetCreateSampleOutput = ` +{ + FleetId: "fleet-44f501fd-d40a-ecd4-acba-032a99d6b167", + Instances: [{ + InstanceIds: ["i-0dfdacec11d0d5bb3"], + InstanceType: "t2.medium", + LaunchTemplateAndOverrides: { + LaunchTemplateSpecification: { + LaunchTemplateId: "lt-02f427483e1be00f5", + Version: "6" + }, + Overrides: { + AvailabilityZone: "us-east-2b", + SubnetId: "subnet-0f5bae1584d67d456" + } + }, + Lifecycle: "on-demand" + }] +} +` + +func Test_instanceConfig_Create(t *testing.T) { + type fields struct { + ec2Iface ec2iface.EC2API + templateConfig *ec2.FleetLaunchTemplateConfigRequest + capacitySpec *ec2.TargetCapacitySpecificationRequest + instanceID string + } + type args struct { + ctx context.Context + } + + ec2Iface := fake.EC2API{FleetOutput: &ec2.CreateFleetOutput{}, WantErr: nil} + cfg := NewFleetRequest(launchTemplate, version, ec2Iface) + + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "First basic test", + fields: fields{ + ec2Iface: ec2Iface, + templateConfig: cfg.templateConfig, + capacitySpec: cfg.capacitySpec, + }, + args: args{context.Background()}, + }, + } + cfg.SetAvailabilityZone(az) + cfg.SetInstanceType(ec2.DefaultTargetCapacityTypeOnDemand) + cfg.SetOnDemandCapacity(1, 1) + cfg.SetSubnet(subnetID) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &instanceConfig{ + ec2Iface: tt.fields.ec2Iface, + templateConfig: tt.fields.templateConfig, + capacitySpec: tt.fields.capacitySpec, + instanceID: tt.fields.instanceID, + } + if err := cfg.Create(tt.args.ctx); (err != nil) != tt.wantErr { + t.Errorf("instanceConfig.Create() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/cloudprovider/aws/factory.go b/pkg/cloudprovider/aws/factory.go index be815457f9b5..82be3a8a5fb3 100644 --- a/pkg/cloudprovider/aws/factory.go +++ b/pkg/cloudprovider/aws/factory.go @@ -20,6 +20,8 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/aws/aws-sdk-go/service/eks" "github.com/aws/aws-sdk-go/service/eks/eksiface" "github.com/aws/aws-sdk-go/service/sqs" @@ -35,6 +37,7 @@ type Factory struct { AutoscalingClient autoscalingiface.AutoScalingAPI SQSClient sqsiface.SQSAPI EKSClient eksiface.EKSAPI + EC2Client ec2iface.EC2API Client client.Client } @@ -44,6 +47,7 @@ func NewFactory(options cloudprovider.Options) *Factory { AutoscalingClient: autoscaling.New(sess), EKSClient: eks.New(sess), SQSClient: sqs.New(sess), + EC2Client: ec2.New(sess), Client: options.Client, } } @@ -68,6 +72,11 @@ func (f *Factory) QueueFor(spec *v1alpha1.QueueSpec) cloudprovider.Queue { } } +func (f *Factory) FleetClient() cloudprovider.Fleet { + // TODO add templateID, version + return NewFleetRequest("", "", f.EC2Client) +} + func withRegion(sess *session.Session) *session.Session { region, err := ec2metadata.New(sess).Region() log.PanicIfError(err, "failed to call the metadata server's region API") diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index 848bd394eb21..9d620c1f25a4 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -15,6 +15,8 @@ limitations under the License. package cloudprovider import ( + "context" + "github.com/awslabs/karpenter/pkg/apis/autoscaling/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -25,6 +27,8 @@ type Factory interface { NodeGroupFor(sng *v1alpha1.ScalableNodeGroupSpec) NodeGroup // QueueFor returns a queue for the provided spec QueueFor(queue *v1alpha1.QueueSpec) Queue + // FleetClient returns a client for the provider to create VM instances + FleetClient() Fleet } // Queue abstracts all provider specific behavior for Queues @@ -49,6 +53,27 @@ type NodeGroup interface { Stabilized() (bool, string, error) } +// Fleet represents a fleet request in the cloud provider, +// all the instances in the fleet will have the same properties +// number of instances can be controlled by setting the capacity +type Fleet interface { + // SetAvailabilityZone is the zone the instances will run in this fleet + SetAvailabilityZone(zone string) + + // SetSubnet is the subnet for the instances + SetSubnet(subnetID string) + + // SetOnDemandCapacity is the desired capacity for this fleet + SetOnDemandCapacity(targetCap, totalCap int64) + + // SetInstanceType configures the instanceType, + // it can be on-demand or spot + SetInstanceType(instanceType string) + + // Create will send the request to cloud provider to create the instant fleet request. + Create(context.Context) error +} + // Options are injected into cloud providers' factories type Options struct { Client client.Client From e740dfb6afe5761bd814ddec0fabaf5fe2cfa949 Mon Sep 17 00:00:00 2001 From: Prateek Gogia Date: Mon, 25 Jan 2021 16:40:20 -0600 Subject: [PATCH 2/2] Add capacity provisioner and refine the interface definitions --- .../{ec2_fleet.go => capacity_provisioner.go} | 48 +++++---- pkg/cloudprovider/aws/ec2_fleet_test.go | 101 ------------------ pkg/cloudprovider/aws/factory.go | 5 +- pkg/cloudprovider/fake/factory.go | 3 + pkg/cloudprovider/fake/provisioner.go | 28 +++++ pkg/cloudprovider/types.go | 52 +++++---- 6 files changed, 89 insertions(+), 148 deletions(-) rename pkg/cloudprovider/aws/{ec2_fleet.go => capacity_provisioner.go} (68%) delete mode 100644 pkg/cloudprovider/aws/ec2_fleet_test.go create mode 100644 pkg/cloudprovider/fake/provisioner.go diff --git a/pkg/cloudprovider/aws/ec2_fleet.go b/pkg/cloudprovider/aws/capacity_provisioner.go similarity index 68% rename from pkg/cloudprovider/aws/ec2_fleet.go rename to pkg/cloudprovider/aws/capacity_provisioner.go index ac19a1a62009..ca2fb9287a97 100644 --- a/pkg/cloudprovider/aws/ec2_fleet.go +++ b/pkg/cloudprovider/aws/capacity_provisioner.go @@ -20,8 +20,32 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/awslabs/karpenter/pkg/cloudprovider" ) +type CapacityProvisioner struct { + ec2Iface ec2iface.EC2API +} + +// NewCapacityProvisioner lets user provision nodes in AWS +func NewCapacityProvisioner(client ec2iface.EC2API) *CapacityProvisioner { + return &CapacityProvisioner{ec2Iface: client} +} + +// Provision accepts desired capacity and contraints for provisioning +func (cp *CapacityProvisioner) Provision(context.Context, *cloudprovider.CapacityConstraints) error { + // Convert contraints to the Node types and select the launch template + // TODO + + // Create the desired number of instances based on desired capacity + config := defaultInstanceConfig("", "", cp.ec2Iface) + _ = config + + // Set AvailabilityZone, subnet, capacity, on-demand or spot + // and validateAndCreate instances + return nil +} + type instanceConfig struct { ec2Iface ec2iface.EC2API templateConfig *ec2.FleetLaunchTemplateConfigRequest @@ -29,8 +53,7 @@ type instanceConfig struct { instanceID string } -func NewFleetRequest(templateID, templateVersion string, client ec2iface.EC2API) *instanceConfig { - +func defaultInstanceConfig(templateID, templateVersion string, client ec2iface.EC2API) *instanceConfig { return &instanceConfig{ ec2Iface: client, templateConfig: &ec2.FleetLaunchTemplateConfigRequest{ @@ -48,27 +71,6 @@ func NewFleetRequest(templateID, templateVersion string, client ec2iface.EC2API) } } -func (cfg *instanceConfig) SetAvailabilityZone(zone string) { - cfg.templateConfig.Overrides[0].AvailabilityZone = aws.String(zone) -} - -func (cfg *instanceConfig) SetSubnet(subnetID string) { - cfg.templateConfig.Overrides[0].SubnetId = aws.String(subnetID) -} - -func (cfg *instanceConfig) SetOnDemandCapacity(targetCap, totalCap int64) { - cfg.capacitySpec.OnDemandTargetCapacity = aws.Int64(targetCap) - cfg.capacitySpec.TotalTargetCapacity = aws.Int64(totalCap) -} - -func (cfg *instanceConfig) SetInstanceType(instanceType string) { - cfg.capacitySpec.DefaultTargetCapacityType = aws.String(instanceType) -} - -func (cfg *instanceConfig) Create(ctx context.Context) error { - return cfg.validateAndCreate(ctx) -} - func (cfg *instanceConfig) validateAndCreate(ctx context.Context) error { input := &ec2.CreateFleetInput{ LaunchTemplateConfigs: []*ec2.FleetLaunchTemplateConfigRequest{cfg.templateConfig}, diff --git a/pkg/cloudprovider/aws/ec2_fleet_test.go b/pkg/cloudprovider/aws/ec2_fleet_test.go deleted file mode 100644 index f51809b08619..000000000000 --- a/pkg/cloudprovider/aws/ec2_fleet_test.go +++ /dev/null @@ -1,101 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package aws - -import ( - "context" - "testing" - - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/ec2/ec2iface" - "github.com/awslabs/karpenter/pkg/cloudprovider/aws/fake" -) - -var ( - launchTemplate = "test-templateID" - version = "10" - az = "us-east-2b" - subnetID = "subnet-12345dead789" -) - -var FleetCreateSampleOutput = ` -{ - FleetId: "fleet-44f501fd-d40a-ecd4-acba-032a99d6b167", - Instances: [{ - InstanceIds: ["i-0dfdacec11d0d5bb3"], - InstanceType: "t2.medium", - LaunchTemplateAndOverrides: { - LaunchTemplateSpecification: { - LaunchTemplateId: "lt-02f427483e1be00f5", - Version: "6" - }, - Overrides: { - AvailabilityZone: "us-east-2b", - SubnetId: "subnet-0f5bae1584d67d456" - } - }, - Lifecycle: "on-demand" - }] -} -` - -func Test_instanceConfig_Create(t *testing.T) { - type fields struct { - ec2Iface ec2iface.EC2API - templateConfig *ec2.FleetLaunchTemplateConfigRequest - capacitySpec *ec2.TargetCapacitySpecificationRequest - instanceID string - } - type args struct { - ctx context.Context - } - - ec2Iface := fake.EC2API{FleetOutput: &ec2.CreateFleetOutput{}, WantErr: nil} - cfg := NewFleetRequest(launchTemplate, version, ec2Iface) - - tests := []struct { - name string - fields fields - args args - wantErr bool - }{ - { - name: "First basic test", - fields: fields{ - ec2Iface: ec2Iface, - templateConfig: cfg.templateConfig, - capacitySpec: cfg.capacitySpec, - }, - args: args{context.Background()}, - }, - } - cfg.SetAvailabilityZone(az) - cfg.SetInstanceType(ec2.DefaultTargetCapacityTypeOnDemand) - cfg.SetOnDemandCapacity(1, 1) - cfg.SetSubnet(subnetID) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg := &instanceConfig{ - ec2Iface: tt.fields.ec2Iface, - templateConfig: tt.fields.templateConfig, - capacitySpec: tt.fields.capacitySpec, - instanceID: tt.fields.instanceID, - } - if err := cfg.Create(tt.args.ctx); (err != nil) != tt.wantErr { - t.Errorf("instanceConfig.Create() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/pkg/cloudprovider/aws/factory.go b/pkg/cloudprovider/aws/factory.go index 82be3a8a5fb3..fae20511a376 100644 --- a/pkg/cloudprovider/aws/factory.go +++ b/pkg/cloudprovider/aws/factory.go @@ -72,9 +72,8 @@ func (f *Factory) QueueFor(spec *v1alpha1.QueueSpec) cloudprovider.Queue { } } -func (f *Factory) FleetClient() cloudprovider.Fleet { - // TODO add templateID, version - return NewFleetRequest("", "", f.EC2Client) +func (f *Factory) CapacityClient() cloudprovider.CapacityProvisioner { + return NewCapacityProvisioner(f.EC2Client) } func withRegion(sess *session.Session) *session.Session { diff --git a/pkg/cloudprovider/fake/factory.go b/pkg/cloudprovider/fake/factory.go index a2fe0b386140..2d839f542310 100644 --- a/pkg/cloudprovider/fake/factory.go +++ b/pkg/cloudprovider/fake/factory.go @@ -63,3 +63,6 @@ func (f *Factory) NodeGroupFor(sng *v1alpha1.ScalableNodeGroupSpec) cloudprovide func (f *Factory) QueueFor(spec *v1alpha1.QueueSpec) cloudprovider.Queue { return &Queue{Id: spec.ID, WantErr: f.WantErr} } +func (f *Factory) CapacityClient() cloudprovider.CapacityProvisioner { + return &Provisioner{} +} diff --git a/pkg/cloudprovider/fake/provisioner.go b/pkg/cloudprovider/fake/provisioner.go new file mode 100644 index 000000000000..ca38899207c4 --- /dev/null +++ b/pkg/cloudprovider/fake/provisioner.go @@ -0,0 +1,28 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + + "github.com/awslabs/karpenter/pkg/cloudprovider" +) + +type Provisioner struct { +} + +func (p *Provisioner) Provision(context.Context, *cloudprovider.CapacityConstraints) error { + return nil +} diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index 9d620c1f25a4..3189d1dd7133 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -18,6 +18,7 @@ import ( "context" "github.com/awslabs/karpenter/pkg/apis/autoscaling/v1alpha1" + v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -27,8 +28,8 @@ type Factory interface { NodeGroupFor(sng *v1alpha1.ScalableNodeGroupSpec) NodeGroup // QueueFor returns a queue for the provided spec QueueFor(queue *v1alpha1.QueueSpec) Queue - // FleetClient returns a client for the provider to create VM instances - FleetClient() Fleet + // CapacityClient returns a provisioner for the provider to create instances + CapacityClient() CapacityProvisioner } // Queue abstracts all provider specific behavior for Queues @@ -53,28 +54,37 @@ type NodeGroup interface { Stabilized() (bool, string, error) } -// Fleet represents a fleet request in the cloud provider, -// all the instances in the fleet will have the same properties -// number of instances can be controlled by setting the capacity -type Fleet interface { - // SetAvailabilityZone is the zone the instances will run in this fleet - SetAvailabilityZone(zone string) - - // SetSubnet is the subnet for the instances - SetSubnet(subnetID string) - - // SetOnDemandCapacity is the desired capacity for this fleet - SetOnDemandCapacity(targetCap, totalCap int64) - - // SetInstanceType configures the instanceType, - // it can be on-demand or spot - SetInstanceType(instanceType string) - - // Create will send the request to cloud provider to create the instant fleet request. - Create(context.Context) error +// CapacityProvisioner helps provision a desired capacity +// with a set of constraints in the cloud provider, +// number of instances and resource capacity can be controlled by +// setting the capacityConstraints +type CapacityProvisioner interface { + // Provision will send the request to cloud provider to provision the desired capacity. + Provision(context.Context, *CapacityConstraints) error } // Options are injected into cloud providers' factories type Options struct { Client client.Client } + +// Architecture for the provisioned capacity +type Architecture string + +const ( + Linux386 Architecture = "linux/386" + LinuxAMD64 Architecture = "linux/amd64" +) + +// CapacityConstraints lets the controller define the desired capacity, +// avalability zone, architecture for the desired nodes. +type CapacityConstraints struct { + // Zone constrains where a node can be created within a region + Zone *string + // Resources constrains the minimum capacity to provision (e.g. CPU, Memory) + Resources v1.ResourceList + // NodeOverhead constrains the per node overhead of system resources + NodeOverhead v1.ResourceList + // Architecture constrains the underlying hardware architecture. + Architecture *Architecture +}