From 12fed137facdb3b24d610f97b84a3b8956c799b5 Mon Sep 17 00:00:00 2001 From: Fabio Bertinatto Date: Wed, 28 Nov 2018 13:39:46 +0100 Subject: [PATCH] Wait for volume to become available --- deploy/kubernetes/v1.12+/provisioner.yaml | 1 + pkg/cloud/cloud.go | 24 +++++++++-------------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/deploy/kubernetes/v1.12+/provisioner.yaml b/deploy/kubernetes/v1.12+/provisioner.yaml index 5c7e8bc76f..6247d8bd87 100644 --- a/deploy/kubernetes/v1.12+/provisioner.yaml +++ b/deploy/kubernetes/v1.12+/provisioner.yaml @@ -99,6 +99,7 @@ spec: - --csi-address=$(ADDRESS) - --v=5 - --feature-gates=Topology=true + - --connection-timeout=20s env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 5227b2ecda..79f8a281d5 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -242,13 +242,8 @@ func (c *cloud) CreateDisk(ctx context.Context, volumeName string, diskOptions * return nil, fmt.Errorf("disk size was not returned by CreateVolume") } - if len(diskOptions.KmsKeyID) > 0 { - err := c.waitForCreate(ctx, volumeID) - if err != nil { - if isAWSErrorVolumeNotFound(err) { - return nil, fmt.Errorf("failed to create encrypted volume: the volume disappeared after creation, most likely due to inaccessible KMS encryption key") - } - } + if err := c.waitForVolume(ctx, volumeID); err != nil { + return nil, fmt.Errorf("failed to get an available volume in EC2: %v", err) } return &Disk{CapacityGiB: size, VolumeID: volumeID, AvailabilityZone: zone}, nil @@ -500,13 +495,14 @@ func (c *cloud) waitForAttachmentState(ctx context.Context, volumeID, state stri return wait.ExponentialBackoff(backoff, verifyVolumeFunc) } -// waitForCreate waits for volume to be created for encrypted volume only -// it polls for created volume to check it has not been silently removed by AWS. +// waitForVolume waits for volume to be in the "available" state. // On a random AWS account (shared among several developers) it took 4s on average. -func (c *cloud) waitForCreate(ctx context.Context, volumeID string) error { +// Also, we assume that the default timeout in the controller is 20s, so we +// make our retry timeout lower in order to avoid exceeding the controller's one. +func (c *cloud) waitForVolume(ctx context.Context, volumeID string) error { var ( - checkInterval = 1 * time.Second - checkTimeout = 30 * time.Second + checkInterval = 3 * time.Second + checkTimeout = 15 * time.Second ) request := &ec2.DescribeVolumesInput{ @@ -523,12 +519,11 @@ func (c *cloud) waitForCreate(ctx context.Context, volumeID string) error { if vol.State != nil { switch *vol.State { case "available": - // The volume is Available, it won't be deleted now. return true, nil case "creating": return false, nil default: - return true, fmt.Errorf("unexpected State of newly created AWS EBS volume %s: %q", volumeID, *vol.State) + return true, fmt.Errorf("unexpected state for volume %s: %q", volumeID, *vol.State) } } return false, nil @@ -546,6 +541,5 @@ func isAWSErrorVolumeNotFound(err error) bool { return true } } - return false }