delete leaked volume if driver don't know the volume status

kubernetes-sigs · Feb 25, 2021 · 82a2c99 · 82a2c99
1 parent 5b31c93
commit 82a2c99
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 10 deletions.
diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go
@@ -343,6 +343,13 @@ func (c *cloud) CreateDisk(ctx context.Context, volumeName string, diskOptions *
 	}
 
 	if err := c.waitForVolume(ctx, volumeID); err != nil {
+		// To avoid leaking volume, we should delete the volume just created
+		if _, error := c.DeleteDisk(ctx, volumeID); error != nil {
+			// TODO: Need to figure out how to handle this scenario instead of just log the error
+			klog.Errorf("%v failed to be deleted, this may cause volume leak", volumeID)
+		} else {
+			klog.V(5).Infof("%v is deleted because it is not in desired state within retry limit", volumeID)
+		}
 		return nil, fmt.Errorf("failed to get an available volume in EC2: %v", err)
 	}
 

diff --git a/pkg/cloud/cloud_test.go b/pkg/cloud/cloud_test.go
@@ -41,14 +41,15 @@ const (
 
 func TestCreateDisk(t *testing.T) {
 	testCases := []struct {
-		name               string
-		volumeName         string
-		volState           string
-		diskOptions        *DiskOptions
-		expDisk            *Disk
-		expErr             error
-		expCreateVolumeErr error
-		expDescVolumeErr   error
+		name                string
+		volumeName          string
+		volState            string
+		diskOptions         *DiskOptions
+		expDisk             *Disk
+		cleanUpFailedVolume bool
+		expErr              error
+		expCreateVolumeErr  error
+		expDescVolumeErr    error
 	}{
 		{
 			name:       "success: normal",
@@ -194,7 +195,8 @@ func TestCreateDisk(t *testing.T) {
 				Tags:             map[string]string{VolumeNameTagKey: "vol-test"},
 				AvailabilityZone: "",
 			},
-			expErr: fmt.Errorf("failed to get an available volume in EC2: timed out waiting for the condition"),
+			cleanUpFailedVolume: true,
+			expErr:              fmt.Errorf("failed to get an available volume in EC2: timed out waiting for the condition"),
 		},
 		{
 			name:       "success: normal from snapshot",
@@ -243,7 +245,9 @@ func TestCreateDisk(t *testing.T) {
 			if len(tc.diskOptions.SnapshotID) > 0 {
 				mockEC2.EXPECT().DescribeSnapshotsWithContext(gomock.Eq(ctx), gomock.Any()).Return(&ec2.DescribeSnapshotsOutput{Snapshots: []*ec2.Snapshot{snapshot}}, nil).AnyTimes()
 			}
-
+			if tc.cleanUpFailedVolume {
+				mockEC2.EXPECT().DeleteVolumeWithContext(gomock.Eq(ctx), gomock.Any()).Return(&ec2.DeleteVolumeOutput{}, nil)
+			}
 			if len(tc.diskOptions.AvailabilityZone) == 0 {
 				mockEC2.EXPECT().DescribeAvailabilityZonesWithContext(gomock.Eq(ctx), gomock.Any()).Return(&ec2.DescribeAvailabilityZonesOutput{
 					AvailabilityZones: []*ec2.AvailabilityZone{