Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add EBS gp3 support #633

Merged
merged 1 commit into from
Dec 7, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,15 @@ The following CSI gRPC calls are implemented:
### CreateVolume Parameters
There are several optional parameters that could be passed into `CreateVolumeRequest.parameters` map:

| Parameters | Values | Default | Description |
|-----------------------------|-----------------------------------|----------|---------------------|
| "csi.storage.k8s.io/fsType" | xfs, ext2, ext3, ext4 | ext4 | File system type that will be formatted during volume creation |
| "type" | io1, io2, gp2, sc1, st1,standard | gp2 | EBS volume type |
| "iopsPerGB" | | | I/O operations per second per GiB. Required when io1 or io2 volume type is specified. If this value multiplied by the size of a requested volume produces a value below the minimum or above the maximum IOPs allowed for the volume type, as documented [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html), AWS will return an error and volume creation will fail |
| "encrypted" | | | Whether the volume should be encrypted or not. Valid values are "true" or "false" |
| "kmsKeyId" | | | The full ARN of the key to use when encrypting the volume. When not specified, the default KMS key is used |
| Parameters | Values | Default | Description |
|-----------------------------|----------------------------------------|----------|---------------------|
| "csi.storage.k8s.io/fsType" | xfs, ext2, ext3, ext4 | ext4 | File system type that will be formatted during volume creation |
| "type" | io1, io2, gp2, gp3, sc1, st1,standard | gp3 | EBS volume type |
| "iopsPerGB" | | | I/O operations per second per GiB. Required when io1 or io2 volume type is specified. If this value multiplied by the size of a requested volume produces a value below the minimum or above the maximum IOPs allowed for the volume type, as documented [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html), AWS will return an error and volume creation will fail |
| "iops" | | 3000 | I/O operations per second. Only effetive when gp3 volume type is specified. If empty, it will set to 3000 as documented [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html). |
| "throughput" | | 125 | Throughput in MiB/s. Only effective when gp3 volume type is specified. If empty, it will set to 125MiB/s as documented [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html). |
| "encrypted" | | | Whether the volume should be encrypted or not. Valid values are "true" or "false" |
| "kmsKeyId" | | | The full ARN of the key to use when encrypting the volume. When not specified, the default KMS key is used |

**Notes**:
* The parameters are case insensitive.
Expand Down
3 changes: 1 addition & 2 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
module github.com/kubernetes-sigs/aws-ebs-csi-driver

require (
github.com/aws/aws-sdk-go v1.35.30
github.com/aws/aws-sdk-go v1.35.37
github.com/container-storage-interface/spec v1.2.0
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
Expand All @@ -13,7 +13,6 @@ require (
github.com/onsi/ginkgo v1.10.2
github.com/onsi/gomega v1.7.0
github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect
golang.org/x/net v0.0.0-20200226051749-491c5fce7268 // indirect
google.golang.org/grpc v1.26.0
k8s.io/api v0.17.3
k8s.io/apimachinery v0.17.3
Expand Down
15 changes: 10 additions & 5 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:l
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7/go.mod h1:LWMyo4iOLWXHGdBki7NIht1kHru/0wM179h+d3g8ATM=
github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.35.30 h1:ZT+70Tw1ar5U2bL81ZyIvcLorxlD1UoxoIgjsEkismY=
github.com/aws/aws-sdk-go v1.35.30/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
github.com/aws/aws-sdk-go v1.35.37 h1:XA71k5PofXJ/eeXdWrTQiuWPEEyq8liguR+Y/QUELhI=
github.com/aws/aws-sdk-go v1.35.37/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/bazelbuild/bazel-gazelle v0.18.2/go.mod h1:D0ehMSbS+vesFsLGiD6JXu3mVEzOlfUl8wNnq+x/9p0=
github.com/bazelbuild/bazel-gazelle v0.19.1-0.20191105222053-70208cbdc798/go.mod h1:rPwzNHUqEzngx1iVBfO/2X2npKaT3tqPqqHW6rVsn/A=
github.com/bazelbuild/buildtools v0.0.0-20190731111112-f720930ceb60/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU=
Expand Down Expand Up @@ -566,6 +566,8 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0=
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
Expand Down Expand Up @@ -602,9 +604,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226051749-491c5fce7268 h1:fnuNgko6vrkrxuKfTMd+0eOz50ziv+Wi+t38KUT3j+E=
golang.org/x/net v0.0.0-20200226051749-491c5fce7268/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down Expand Up @@ -641,13 +642,17 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191220220014-0732a990476f h1:72l8qCJ1nGxMGH26QVBVIxKd/D34cfGt0OvrPtpemyY=
golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
Expand Down
15 changes: 14 additions & 1 deletion pkg/cloud/cloud.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ const (
VolumeTypeIO2 = "io2"
// VolumeTypeGP2 represents a general purpose SSD type of volume.
VolumeTypeGP2 = "gp2"
// VolumeTypeGP3 represents a general purpose SSD type of volume.
VolumeTypeGP3 = "gp3"
// VolumeTypeSC1 represents a cold HDD (sc1) type of volume.
VolumeTypeSC1 = "sc1"
// VolumeTypeST1 represents a throughput-optimized HDD type of volume.
Expand All @@ -55,6 +57,7 @@ var (
VolumeTypeIO1,
VolumeTypeIO2,
VolumeTypeGP2,
VolumeTypeGP3,
VolumeTypeSC1,
VolumeTypeST1,
VolumeTypeStandard,
Expand Down Expand Up @@ -82,7 +85,7 @@ const (
// DefaultVolumeSize represents the default volume size.
DefaultVolumeSize int64 = 100 * util.GiB
// DefaultVolumeType specifies which storage to use for newly created Volumes.
DefaultVolumeType = VolumeTypeGP2
DefaultVolumeType = VolumeTypeGP3
)

// Tags
Expand Down Expand Up @@ -138,6 +141,8 @@ type DiskOptions struct {
Tags map[string]string
VolumeType string
IOPSPerGB int
IOPS int
Throughput int
AvailabilityZone string
OutpostArn string
Encrypted bool
Expand Down Expand Up @@ -243,6 +248,7 @@ func (c *cloud) CreateDisk(ctx context.Context, volumeName string, diskOptions *
var (
createType string
iops int64
throughput int64
)
capacityGiB := util.BytesToGiB(diskOptions.CapacityBytes)

Expand All @@ -252,6 +258,10 @@ func (c *cloud) CreateDisk(ctx context.Context, volumeName string, diskOptions *
case VolumeTypeIO1, VolumeTypeIO2:
createType = diskOptions.VolumeType
iops = capacityGiB * int64(diskOptions.IOPSPerGB)
case VolumeTypeGP3:
createType = diskOptions.VolumeType
iops = int64(diskOptions.IOPS)
throughput = int64(diskOptions.Throughput)
case "":
createType = DefaultVolumeType
default:
Expand Down Expand Up @@ -299,6 +309,9 @@ func (c *cloud) CreateDisk(ctx context.Context, volumeName string, diskOptions *
if iops > 0 {
request.Iops = aws.Int64(iops)
}
if throughput > 0 && diskOptions.VolumeType == VolumeTypeGP3 {
request.Throughput = aws.Int64(throughput)
}
snapshotID := diskOptions.SnapshotID
if len(snapshotID) > 0 {
request.SnapshotId = aws.String(snapshotID)
Expand Down
33 changes: 33 additions & 0 deletions pkg/cloud/cloud_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,39 @@ func TestCreateDisk(t *testing.T) {
},
expErr: nil,
},
{
name: "success: normal with io2 options",
volumeName: "vol-test-name",
diskOptions: &DiskOptions{
CapacityBytes: util.GiBToBytes(1),
Tags: map[string]string{VolumeNameTagKey: "vol-test"},
VolumeType: VolumeTypeIO2,
IOPSPerGB: 100,
},
expDisk: &Disk{
VolumeID: "vol-test",
CapacityGiB: 1,
AvailabilityZone: defaultZone,
},
expErr: nil,
},
{
name: "success: normal with gp3 options",
volumeName: "vol-test-name",
diskOptions: &DiskOptions{
CapacityBytes: util.GiBToBytes(1),
Tags: map[string]string{VolumeNameTagKey: "vol-test"},
VolumeType: VolumeTypeGP3,
IOPS: 3000,
Throughput: 125,
},
expDisk: &Disk{
VolumeID: "vol-test",
CapacityGiB: 1,
AvailabilityZone: defaultZone,
},
expErr: nil,
},
{
name: "success: normal with provided zone",
volumeName: "vol-test-name",
Expand Down
6 changes: 6 additions & 0 deletions pkg/driver/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ const (
// IopsPerGBKey represents key for IOPS per GB
IopsPerGBKey = "iopspergb"

// Iops represents key for IOPS for volume
IopsKey = "iops"

// ThroughputKey represents key for throughput
ThroughputKey = "throughput"

// EncryptedKey represents key for whether filesystem is encrypted
EncryptedKey = "encrypted"

Expand Down
14 changes: 14 additions & 0 deletions pkg/driver/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,8 @@ func (d *controllerService) CreateVolume(ctx context.Context, req *csi.CreateVol
var (
volumeType string
iopsPerGB int
iops int
throughput int
isEncrypted bool
kmsKeyID string
volumeTags = map[string]string{
Expand All @@ -149,6 +151,16 @@ func (d *controllerService) CreateVolume(ctx context.Context, req *csi.CreateVol
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "Could not parse invalid iopsPerGB: %v", err)
}
case IopsKey:
iops, err = strconv.Atoi(value)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "Could not parse invalid iops: %v", err)
}
case ThroughputKey:
throughput, err = strconv.Atoi(value)
if err != nil {
return nil, status.Errorf(codes.InvalidArgument, "Could not parse invalid throughput: %v", err)
}
case EncryptedKey:
if value == "true" {
isEncrypted = true
Expand Down Expand Up @@ -208,6 +220,8 @@ func (d *controllerService) CreateVolume(ctx context.Context, req *csi.CreateVol
Tags: volumeTags,
VolumeType: volumeType,
IOPSPerGB: iopsPerGB,
IOPS: iops,
Throughput: throughput,
AvailabilityZone: zone,
OutpostArn: outpostArn,
Encrypted: isEncrypted,
Expand Down
126 changes: 126 additions & 0 deletions pkg/driver/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -758,6 +758,52 @@ func TestCreateVolume(t *testing.T) {
}
},
},
{
name: "success with volume type gp3",
testFunc: func(t *testing.T) {
// iops 5000 requires at least 10GB
volSize := int64(20 * 1024 * 1024 * 1024)
capRange := &csi.CapacityRange{RequiredBytes: volSize}
req := &csi.CreateVolumeRequest{
Name: "vol-test",
CapacityRange: capRange,
VolumeCapabilities: stdVolCap,
Parameters: map[string]string{
VolumeTypeKey: cloud.VolumeTypeGP3,
IopsKey: "5000",
ThroughputKey: "250",
},
}

ctx := context.Background()

mockDisk := &cloud.Disk{
VolumeID: req.Name,
AvailabilityZone: expZone,
CapacityGiB: util.BytesToGiB(volSize),
}

mockCtl := gomock.NewController(t)
defer mockCtl.Finish()

mockCloud := mocks.NewMockCloud(mockCtl)
mockCloud.EXPECT().GetDiskByName(gomock.Eq(ctx), gomock.Eq(req.Name), gomock.Eq(volSize)).Return(nil, cloud.ErrNotFound)
mockCloud.EXPECT().CreateDisk(gomock.Eq(ctx), gomock.Eq(req.Name), gomock.Any()).Return(mockDisk, nil)

awsDriver := controllerService{
cloud: mockCloud,
driverOptions: &DriverOptions{},
}

if _, err := awsDriver.CreateVolume(ctx, req); err != nil {
srvErr, ok := status.FromError(err)
if !ok {
t.Fatalf("Could not get error status code from error: %v", srvErr)
}
t.Fatalf("Unexpected error: %v", srvErr.Code())
}
},
},
{
name: "success with volume type io1",
testFunc: func(t *testing.T) {
Expand Down Expand Up @@ -1047,6 +1093,86 @@ func TestCreateVolume(t *testing.T) {
}
},
},
{
name: "fail with invalid iops parameter",
testFunc: func(t *testing.T) {
req := &csi.CreateVolumeRequest{
Name: "vol-test",
CapacityRange: stdCapRange,
VolumeCapabilities: stdVolCap,
Parameters: map[string]string{
VolumeTypeKey: cloud.VolumeTypeGP3,
IopsKey: "aaa",
},
}

ctx := context.Background()

mockCtl := gomock.NewController(t)
defer mockCtl.Finish()

mockCloud := mocks.NewMockCloud(mockCtl)
mockCloud.EXPECT().GetDiskByName(gomock.Eq(ctx), gomock.Eq(req.Name), gomock.Eq(stdVolSize)).Return(nil, cloud.ErrNotFound)

awsDriver := controllerService{
cloud: mockCloud,
driverOptions: &DriverOptions{},
}

_, err := awsDriver.CreateVolume(ctx, req)
if err == nil {
t.Fatalf("Expected CreateVolume to fail but got no error")
}

srvErr, ok := status.FromError(err)
if !ok {
t.Fatalf("Could not get error status code from error: %v", srvErr)
}
if srvErr.Code() != codes.InvalidArgument {
t.Fatalf("Expect InvalidArgument but got: %s", srvErr.Code())
}
},
},
{
name: "fail with invalid throughput parameter",
testFunc: func(t *testing.T) {
req := &csi.CreateVolumeRequest{
Name: "vol-test",
CapacityRange: stdCapRange,
VolumeCapabilities: stdVolCap,
Parameters: map[string]string{
VolumeTypeKey: cloud.VolumeTypeGP3,
ThroughputKey: "aaa",
},
}

ctx := context.Background()

mockCtl := gomock.NewController(t)
defer mockCtl.Finish()

mockCloud := mocks.NewMockCloud(mockCtl)
mockCloud.EXPECT().GetDiskByName(gomock.Eq(ctx), gomock.Eq(req.Name), gomock.Eq(stdVolSize)).Return(nil, cloud.ErrNotFound)

awsDriver := controllerService{
cloud: mockCloud,
driverOptions: &DriverOptions{},
}

_, err := awsDriver.CreateVolume(ctx, req)
if err == nil {
t.Fatalf("Expected CreateVolume to fail but got no error")
}

srvErr, ok := status.FromError(err)
if !ok {
t.Fatalf("Could not get error status code from error: %v", srvErr)
}
if srvErr.Code() != codes.InvalidArgument {
t.Fatalf("Expect InvalidArgument but got: %s", srvErr.Code())
}
},
},
{
name: "success when volume exists and contains VolumeContext and AccessibleTopology",
testFunc: func(t *testing.T) {
Expand Down
Loading