Skip to content

Commit

Permalink
Add BlockDeviceMappings to the AWS cloudprovider (#1420)
Browse files Browse the repository at this point in the history
* Add support for specifying blockDeviceMappings in the provisioner

* cleanup

* a few fixes

* rename pkg ltresolver to amifamily

* naming changes
  • Loading branch information
bwagner5 authored Mar 10, 2022
1 parent 1e75c7c commit c1d08c6
Show file tree
Hide file tree
Showing 27 changed files with 1,200 additions and 503 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ require (
github.com/onsi/ginkgo v1.16.5
github.com/onsi/gomega v1.18.1
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pelletier/go-toml/v2 v2.0.0-beta.5
github.com/prometheus/client_golang v1.12.1
github.com/prometheus/client_model v0.2.0
go.uber.org/multierr v1.7.0
Expand Down
6 changes: 5 additions & 1 deletion go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -593,7 +593,10 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml/v2 v2.0.0-beta.5 h1:zRY68WYiDE9YZLflUWuyOny5YA+DwvyFzZBfb8E/9wk=
github.com/pelletier/go-toml/v2 v2.0.0-beta.5/go.mod h1:ke6xncR3W76Ba8xnVxkrZG0js6Rd2BsQEAYrfgJ6eQA=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
Expand Down Expand Up @@ -687,8 +690,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942 h1:t0lM6y/M5IiUZyvbBTcngso8SZEZICH7is9B6g/obVU=
github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
Expand Down
139 changes: 0 additions & 139 deletions pkg/cloudprovider/aws/ami.go

This file was deleted.

68 changes: 68 additions & 0 deletions pkg/cloudprovider/aws/amifamily/al2.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package amifamily

import (
"fmt"

"github.com/aws/aws-sdk-go/aws"
core "k8s.io/api/core/v1"

"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
"github.com/aws/karpenter/pkg/cloudprovider"
"github.com/aws/karpenter/pkg/cloudprovider/aws/amifamily/bootstrap"
"github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1"
)

type AL2 struct {
*Options
}

// SSMAlias returns the AMI Alias to query SSM
func (a AL2) SSMAlias(version string, instanceType cloudprovider.InstanceType) string {
amiSuffix := ""
if !instanceType.NvidiaGPUs().IsZero() || !instanceType.AWSNeurons().IsZero() {
amiSuffix = "-gpu"
} else if instanceType.Architecture() == v1alpha5.ArchitectureArm64 {
amiSuffix = fmt.Sprintf("-%s", instanceType.Architecture())
}
return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix)
}

// UserData returns the exact same string for equivalent input,
// even if elements of those inputs are in differing orders,
// guaranteeing it won't cause spurious hash differences.
// AL2 userdata also works on Ubuntu
func (a AL2) UserData(kubeletConfig v1alpha5.KubeletConfiguration, taints []core.Taint, labels map[string]string, caBundle *string) bootstrap.Bootstrapper {
return bootstrap.EKS{
Options: bootstrap.Options{
ClusterName: a.Options.ClusterName,
ClusterEndpoint: a.Options.ClusterEndpoint,
AWSENILimitedPodDensity: a.Options.AWSENILimitedPodDensity,
KubeletConfig: kubeletConfig,
Taints: taints,
Labels: labels,
CABundle: caBundle,
},
}
}

// DefaultBlockDeviceMappings returns the default block device mappings for the AMI Family
func (a AL2) DefaultBlockDeviceMappings() []*v1alpha1.BlockDeviceMapping {
return []*v1alpha1.BlockDeviceMapping{{
DeviceName: aws.String("/dev/xvda"),
EBS: &defaultEBS,
}}
}
48 changes: 48 additions & 0 deletions pkg/cloudprovider/aws/amifamily/ami.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package amifamily

import (
"context"
"fmt"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ssm"
"github.com/aws/aws-sdk-go/service/ssm/ssmiface"
"github.com/patrickmn/go-cache"
"knative.dev/pkg/logging"

"github.com/aws/karpenter/pkg/cloudprovider"
)

type AMIProvider struct {
cache *cache.Cache
ssm ssmiface.SSMAPI
}

// Get returns a set of AMIIDs and corresponding instance types. AMI may vary due to architecture, accelerator, etc
func (p *AMIProvider) Get(ctx context.Context, instanceType cloudprovider.InstanceType, ssmQuery string) (string, error) {
if id, ok := p.cache.Get(ssmQuery); ok {
return id.(string), nil
}
output, err := p.ssm.GetParameterWithContext(ctx, &ssm.GetParameterInput{Name: aws.String(ssmQuery)})
if err != nil {
return "", fmt.Errorf("getting ssm parameter, %w", err)
}
ami := aws.StringValue(output.Parameter.Value)
p.cache.SetDefault(ssmQuery, ami)
logging.FromContext(ctx).Debugf("Discovered %s for query %s", ami, ssmQuery)
return ami, nil
}
40 changes: 40 additions & 0 deletions pkg/cloudprovider/aws/amifamily/bootstrap/bootstrap.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package bootstrap

import (
core "k8s.io/api/core/v1"

"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
)

// Options is the node bootstrapping parameters passed from Karpenter to the provisioning node
type Options struct {
ClusterName string
ClusterEndpoint string
KubeletConfig v1alpha5.KubeletConfiguration
Taints []core.Taint `hash:"set"`
Labels map[string]string `hash:"set"`
CABundle *string
AWSENILimitedPodDensity bool
}

// Bootstrapper can be implemented to generate a bootstrap script
// that uses the params from the Bootstrap type for a specific
// bootstrapping method.
// Examples are the Bottlerocket config and the eks-bootstrap script
type Bootstrapper interface {
Script() string
}
Loading

0 comments on commit c1d08c6

Please sign in to comment.