change to block device mappings

pkg/cloudprovider/aws/apis/v1alpha1/provider.go

@@ -78,27 +78,9 @@ type AWS struct {
 	// required.
 	// +optional
 	MetadataOptions *MetadataOptions `json:"metadataOptions,omitempty"`
-	// RootVolumeOptions to be applied to the root block device volume of provisioned nodes.
-	// Refer to the EBS documentation for configuring proper volume options:
-	// (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html)
-	// +optional
-	RootVolumeOptions *RootVolumeOptions `json:"rootVolumeOptions,omitempty"`
-}
-
-// RootVolumeOptions are applied to the root block device volume of provisioned nodes.
-type RootVolumeOptions struct {
-	// Size adjusts the block device volume of the instance provisioned.
-	// +optional
-	Size *resource.Quantity `json:"size,omitempty"`
-
-	// Type adjusts the volume type (gp2, gp3, io1, io2, sc1, st1, standard)
-	// If io1 or io2 is specified, then IOPS must also be specified.
-	// +optional
-	Type *string `json:"type,omitempty"`
-
-	// IOPS adjusts the I/O operations per second of the volume
-	// +optional
-	IOPS *int64 `json:"iops,omitempty"`
+	// BlockDeviceMappings to be applied to provisioned nodes.
+	// +optionals
+	BlockDeviceMappings []*BlockDeviceMapping `json:"blockDeviceMappings,omitempty"`
 }
 
 // MetadataOptions contains parameters for specifying the exposure of the
@@ -145,6 +127,69 @@ type MetadataOptions struct {
 	HTTPTokens *string `json:"httpTokens,omitempty"`
 }
 
+type BlockDeviceMapping struct {
+	// The device name (for example, /dev/sdh or xvdh).
+	DeviceName *string `json:"deviceName,omitempty"`
+	// EBS contains parameters used to automatically set up EBS volumes when an instance is launched.
+	EBS *BlockDevice `json:"ebs,omitempty"`
+}
+
+type BlockDevice struct {
+	// DeleteOnTermination indicates whether the EBS volume is deleted on instance termination.
+	DeleteOnTermination *bool `json:"deleteOnTermination,omitempty"`
+
+	// Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only
+	// be attached to instances that support Amazon EBS encryption. If you are creating
+	// a volume from a snapshot, you can't specify an encryption value.
+	Encrypted *bool `json:"encrypted,omitempty"`
+
+	// IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes,
+	// this represents the number of IOPS that are provisioned for the volume. For
+	// gp2 volumes, this represents the baseline performance of the volume and the
+	// rate at which the volume accumulates I/O credits for bursting.
+	//
+	// The following are the supported values for each volume type:
+	//
+	//    * gp3: 3,000-16,000 IOPS
+	//
+	//    * io1: 100-64,000 IOPS
+	//
+	//    * io2: 100-64,000 IOPS
+	//
+	// For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built
+	// on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances).
+	// Other instance families guarantee performance up to 32,000 IOPS.
+	//
+	// This parameter is supported for io1, io2, and gp3 volumes only. This parameter
+	// is not supported for gp2, st1, sc1, or standard volumes.
+	IOPS *int64 `json:"iops,omitempty"`
+
+	// KMSKeyID (ARN) of the symmetric Key Management Service (KMS) CMK used for encryption.
+	KMSKeyID *string `json:"kmsKeyID,omitempty"`
+
+	// Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s.
+	// Valid Range: Minimum value of 125. Maximum value of 1000.
+	Throughput *int64 `json:"throughput,omitempty"`
+
+	// VolumeSize in GiBs. You must specify either a snapshot ID or
+	// a volume size. The following are the supported volumes sizes for each volume
+	// type:
+	//
+	//    * gp2 and gp3: 1-16,384
+	//
+	//    * io1 and io2: 4-16,384
+	//
+	//    * st1 and sc1: 125-16,384
+	//
+	//    * standard: 1-1,024
+	VolumeSize *resource.Quantity `json:"volumeSize,omitempty"`
+
+	// VolumeType of the block device.
+	// For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html)
+	// in the Amazon Elastic Compute Cloud User Guide.
+	VolumeType *string `json:"volumeType,omitempty"`
+}
+
 func Deserialize(constraints *v1alpha5.Constraints) (*Constraints, error) {
 	if constraints.Provider == nil {
 		return nil, fmt.Errorf("invariant violated: spec.provider is not defined. Is the defaulting webhook installed?")

pkg/cloudprovider/aws/apis/v1alpha1/provider_defaults.go

@@ -17,6 +17,8 @@ package v1alpha1
 import (
 	"context"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -27,7 +29,7 @@ func (c *Constraints) Default(ctx context.Context) {
 	c.defaultArchitecture()
 	c.defaultCapacityTypes()
 	c.defaultAMIFamily()
-	c.defaultRootVolumeOptions()
+	c.defaultBlockDeviceMappings()
 }
 
 func (c *Constraints) defaultCapacityTypes() {
@@ -68,12 +70,35 @@ func (c *Constraints) defaultAMIFamily() {
 	c.AMIFamily = &AMIFamilyAL2
 }
 
-func (c *Constraints) defaultRootVolumeOptions() {
-	if c.RootVolumeOptions != nil {
+func (c *Constraints) defaultBlockDeviceMappings() {
+	if c.LaunchTemplate != nil {
 		return
 	}
-	if c.LaunchTemplate != nil {
+	if len(c.BlockDeviceMappings) != 0 {
 		return
 	}
-	c.RootVolumeOptions = &RootVolumeOptions{Size: resource.NewScaledQuantity(20, resource.Giga)}
+	defaultEBS := BlockDevice{
+		DeleteOnTermination: aws.Bool(true),
+		Encrypted:           aws.Bool(true),
+		VolumeType:          aws.String(ec2.VolumeTypeGp3),
+		VolumeSize:          resource.NewScaledQuantity(20, resource.Giga),
+	}
+	switch aws.StringValue(c.AMIFamily) {
+	case AMIFamilyAL2, AMIFamilyUbuntu:
+		c.BlockDeviceMappings = append(c.BlockDeviceMappings, &BlockDeviceMapping{
+			DeviceName: aws.String("/dev/xvda"),
+			EBS:        &defaultEBS,
+		})
+	case AMIFamilyBottlerocket:
+		xvdaEBS := defaultEBS
+		xvdaEBS.VolumeSize = resource.NewScaledQuantity(4, resource.Giga)
+		c.BlockDeviceMappings = append(c.BlockDeviceMappings, &BlockDeviceMapping{
+			DeviceName: aws.String("/dev/xvda"),
+			EBS:        &xvdaEBS,
+		})
+		c.BlockDeviceMappings = append(c.BlockDeviceMappings, &BlockDeviceMapping{
+			DeviceName: aws.String("/dev/xvdb"),
+			EBS:        &defaultEBS,
+		})
+	}
 }

pkg/cloudprovider/aws/apis/v1alpha1/provider_validation.go

@@ -30,7 +30,12 @@ const (
 	amiFamilyPath               = "amiFamily"
 	metadataOptionsPath         = "metadataOptions"
 	instanceProfilePath         = "instanceProfile"
-	rootVolumeOptionsPath       = "rootVolumeOptions"
+	blockDeviceMappingsPath     = "blockDeviceMappings"
+)
+
+var (
+	maxVolumeSize = *resource.NewScaledQuantity(64, resource.Tera)
+	minVolumeSize = *resource.NewScaledQuantity(1, resource.Giga)
 )
 
 func (a *AWS) Validate() (errs *apis.FieldError) {
@@ -45,7 +50,7 @@ func (a *AWS) validate() (errs *apis.FieldError) {
 		a.validateTags(),
 		a.validateMetadataOptions(),
 		a.validateAMIFamily(),
-		a.validateRootVolumeOptions(),
+		a.validateBlockDeviceMappings(),
 	)
 }
 
@@ -65,8 +70,8 @@ func (a *AWS) validateLaunchTemplate() (errs *apis.FieldError) {
 	if a.InstanceProfile != nil {
 		errs = errs.Also(apis.ErrMultipleOneOf(launchTemplatePath, instanceProfilePath))
 	}
-	if a.RootVolumeOptions != nil {
-		errs = errs.Also(apis.ErrMultipleOneOf(launchTemplatePath, rootVolumeOptionsPath))
+	if len(a.BlockDeviceMappings) != 0 {
+		errs = errs.Also(apis.ErrMultipleOneOf(launchTemplatePath, blockDeviceMappingsPath))
 	}
 	return errs
 }
@@ -170,24 +175,28 @@ func (a *AWS) validateStringEnum(value, field string, validValues []string) *api
 	return apis.ErrInvalidValue(fmt.Sprintf("%s not in %v", value, strings.Join(validValues, ", ")), field)
 }
 
-func (a *AWS) validateRootVolumeOptions() *apis.FieldError {
-	if a.RootVolumeOptions == nil {
+func (a *AWS) validateBlockDeviceMappings() (errs *apis.FieldError) {
+	if a.BlockDeviceMappings == nil {
 		return nil
 	}
-	if a.RootVolumeOptions.Size == nil {
-		return apis.ErrMissingField("rootVolumeOptions.size")
-	}
-	// 64 TiB is the maximum supported EBS volume size
-	maxRootVolumeSize := *resource.NewScaledQuantity(64, resource.Tera)
-	minRootVolumeSize := *resource.NewQuantity(0, resource.DecimalSI)
-	if a.RootVolumeOptions.Size.Cmp(maxRootVolumeSize) == 1 {
-		return apis.ErrOutOfBoundsValue(a.RootVolumeOptions.Size, minRootVolumeSize, maxRootVolumeSize, "rootVolumeOptions.size")
-	}
-
-	if a.RootVolumeOptions.Type != nil {
-		if fieldErr := a.validateStringEnum(*a.RootVolumeOptions.Type, "rootVolumeOptions.type", ec2.VolumeType_Values()); fieldErr != nil {
-			return fieldErr
+	for i, blockDeviceMapping := range a.BlockDeviceMappings {
+		if blockDeviceMapping.DeviceName == nil {
+			errs = errs.Also(apis.ErrMissingField(fmt.Sprintf("%s[%d].deviceName", blockDeviceMappingsPath, i)))
+		}
+		if blockDeviceMapping.EBS == nil {
+			errs = errs.Also(apis.ErrMissingField(fmt.Sprintf("%s[%d].ebs", blockDeviceMappingsPath, i)))
+			continue
+		}
+		if blockDeviceMapping.EBS.VolumeType != nil {
+			if fieldErr := a.validateStringEnum(*blockDeviceMapping.EBS.VolumeType, fmt.Sprintf("%s[%d].ebs.volumeType", blockDeviceMappingsPath, i), ec2.VolumeType_Values()); fieldErr != nil {
+				errs = errs.Also(fieldErr)
+			}
+		}
+		if blockDeviceMapping.EBS.VolumeSize == nil {
+			errs = errs.Also(apis.ErrMissingField(fmt.Sprintf("%s[%d].ebs.volumeSize", blockDeviceMappingsPath, i)))
+		} else if blockDeviceMapping.EBS.VolumeSize.Cmp(minVolumeSize) == -1 || blockDeviceMapping.EBS.VolumeSize.Cmp(maxVolumeSize) == 1 {
+			errs = errs.Also(apis.ErrOutOfBoundsValue(blockDeviceMapping.EBS.VolumeSize.String(), minVolumeSize.String(), maxVolumeSize.String(), fmt.Sprintf("%s[%d].ebs.volumeSize", blockDeviceMappingsPath, i)))
 		}
 	}
-	return nil
+	return errs
 }