[WIP] Wait for Hyperplane-attached ENIs for lambdas to move to 'available' state before detaching

aws/resource_aws_network_interface.go

@@ -16,6 +16,10 @@ import (
 	"github.com/hashicorp/terraform/helper/schema"
 )
 
+const (
+	networkInterfaceStatusDeleted = "deleted"
+)
+
 func resourceAwsNetworkInterface() *schema.Resource {
 	return &schema.Resource{
 		Create: resourceAwsNetworkInterfaceCreate,
@@ -444,3 +448,124 @@ func resourceAwsEniAttachmentHash(v interface{}) int {
 	buf.WriteString(fmt.Sprintf("%d-", m["device_index"].(int)))
 	return hashcode.String(buf.String())
 }
+
+func deleteNetworkInterface(conn *ec2.EC2, eniId string) error {
+	_, err := conn.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{
+		NetworkInterfaceId: aws.String(eniId),
+	})
+
+	if isAWSErr(err, "InvalidNetworkInterfaceID.NotFound", "") {
+		return nil
+	}
+
+	if err != nil {
+		return fmt.Errorf("error deleting ENI (%s): %s", eniId, err)
+	}
+
+	return nil
+}
+
+func detachNetworkInterface(conn *ec2.EC2, eni *ec2.NetworkInterface, timeout time.Duration) error {
+	eniId := aws.StringValue(eni.NetworkInterfaceId)
+	if eni.Attachment == nil {
+		log.Printf("[DEBUG] ENI %s is already detached", eniId)
+		return nil
+	}
+
+	_, err := conn.DetachNetworkInterface(&ec2.DetachNetworkInterfaceInput{
+		AttachmentId: eni.Attachment.AttachmentId,
+		Force:        aws.Bool(true),
+	})
+
+	if isAWSErr(err, "InvalidAttachmentID.NotFound", "") {
+		return nil
+	}
+
+	if err != nil {
+		return fmt.Errorf("error detaching ENI (%s): %s", eniId, err)
+	}
+
+	stateConf := &resource.StateChangeConf{
+		Pending: []string{
+			ec2.AttachmentStatusAttaching,
+			ec2.AttachmentStatusAttached,
+			ec2.AttachmentStatusDetaching,
+		},
+		Target: []string{
+			ec2.AttachmentStatusDetached,
+		},
+		Refresh:    networkInterfaceAttachmentStateRefresh(conn, eniId),
+		Timeout:    timeout,
+		Delay:      10 * time.Second,
+		MinTimeout: 5 * time.Second,
+	}
+
+	log.Printf("[DEBUG] Waiting for ENI (%s) to become detached", eniId)
+	_, err = stateConf.WaitForState()
+	if err != nil {
+		return fmt.Errorf("error waiting for ENI (%s) to become detached: %s", eniId, err)
+	}
+
+	return nil
+}
+
+func networkInterfaceAttachmentStateRefresh(conn *ec2.EC2, eniId string) resource.StateRefreshFunc {
+	return func() (interface{}, string, error) {
+		resp, err := conn.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{
+			NetworkInterfaceIds: aws.StringSlice([]string{eniId}),
+		})
+
+		if isAWSErr(err, "InvalidNetworkInterfaceID.NotFound", "") {
+			return "", ec2.AttachmentStatusDetached, nil
+		}
+
+		if err != nil {
+			return nil, "", fmt.Errorf("error describing ENI (%s): %s", eniId, err)
+		}
+
+		n := len(resp.NetworkInterfaces)
+		switch n {
+		case 0:
+			return "", ec2.AttachmentStatusDetached, nil
+
+		case 1:
+			attachment := resp.NetworkInterfaces[0].Attachment
+			if attachment == nil {
+				return "", ec2.AttachmentStatusDetached, nil
+			}
+			return attachment, aws.StringValue(attachment.Status), nil
+
+		default:
+			return nil, "", fmt.Errorf("found %d ENIs for %s, expected 1", n, eniId)
+		}
+	}
+}
+
+func networkInterfaceStateRefresh(conn *ec2.EC2, eniId string) resource.StateRefreshFunc {
+	return func() (interface{}, string, error) {
+		resp, err := conn.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{
+			NetworkInterfaceIds: aws.StringSlice([]string{eniId}),
+		})
+
+		if isAWSErr(err, "InvalidNetworkInterfaceID.NotFound", "") {
+			return "", networkInterfaceStatusDeleted, nil
+		}
+
+		if err != nil {
+			return nil, "", fmt.Errorf("error describing ENI (%s): %s", eniId, err)
+		}
+
+		n := len(resp.NetworkInterfaces)
+		switch n {
+		case 0:
+			return "", networkInterfaceStatusDeleted, nil
+
+		case 1:
+			eni := resp.NetworkInterfaces[0]
+			return eni, aws.StringValue(eni.Status), nil
+
+		default:
+			return nil, "", fmt.Errorf("found %d ENIs for %s, expected 1", n, eniId)
+		}
+	}
+}

aws/resource_aws_security_group.go

@@ -31,7 +31,7 @@ func resourceAwsSecurityGroup() *schema.Resource {
 
 		Timeouts: &schema.ResourceTimeout{
 			Create: schema.DefaultTimeout(10 * time.Minute),
-			Delete: schema.DefaultTimeout(10 * time.Minute),
+			Delete: schema.DefaultTimeout(30 * time.Minute),
 		},
 
 		SchemaVersion: 1,
@@ -448,8 +448,8 @@ func resourceAwsSecurityGroupDelete(d *schema.ResourceData, meta interface{}) er
 
 	log.Printf("[DEBUG] Security Group destroy: %v", d.Id())
 
-	if err := deleteLingeringLambdaENIs(conn, d, "group-id"); err != nil {
-		return fmt.Errorf("Failed to delete Lambda ENIs: %s", err)
+	if err := deleteLingeringLambdaENIs(conn, "group-id", d.Id(), d.Timeout(schema.TimeoutDelete)); err != nil {
+		return fmt.Errorf("error deleting Lambda ENIs using Security Group (%s): %s", d.Id(), err)
 	}
 
 	// conditionally revoke rules first before attempting to delete the group
@@ -1402,98 +1402,62 @@ func sgProtocolIntegers() map[string]int {
 
 // The AWS Lambda service creates ENIs behind the scenes and keeps these around for a while
 // which would prevent SGs attached to such ENIs from being destroyed
-func deleteLingeringLambdaENIs(conn *ec2.EC2, d *schema.ResourceData, filterName string) error {
-	// Here we carefully find the offenders
-	params := &ec2.DescribeNetworkInterfacesInput{
-		Filters: []*ec2.Filter{
-			{
-				Name:   aws.String(filterName),
-				Values: []*string{aws.String(d.Id())},
-			},
-			{
-				Name:   aws.String("description"),
-				Values: []*string{aws.String("AWS Lambda VPC ENI: *")},
-			},
-		},
-	}
-	networkInterfaceResp, err := conn.DescribeNetworkInterfaces(params)
-
-	if isAWSErr(err, "InvalidNetworkInterfaceID.NotFound", "") {
-		return nil
-	}
+func deleteLingeringLambdaENIs(conn *ec2.EC2, filterName, resourceId string, timeout time.Duration) error {
+	resp, err := conn.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{
+		Filters: buildEC2AttributeFilterList(map[string]string{
+			filterName:    resourceId,
+			"description": "AWS Lambda VPC ENI*",
+		}),
+	})
 
 	if err != nil {
-		return err
+		return fmt.Errorf("error describing ENIs: %s", err)
 	}
 
-	// Then we detach and finally delete those
-	v := networkInterfaceResp.NetworkInterfaces
-	for _, eni := range v {
-		if eni.Attachment != nil {
-			detachNetworkInterfaceParams := &ec2.DetachNetworkInterfaceInput{
-				AttachmentId: eni.Attachment.AttachmentId,
-			}
-			_, detachNetworkInterfaceErr := conn.DetachNetworkInterface(detachNetworkInterfaceParams)
+	for _, eni := range resp.NetworkInterfaces {
+		eniId := aws.StringValue(eni.NetworkInterfaceId)
 
-			if isAWSErr(detachNetworkInterfaceErr, "InvalidNetworkInterfaceID.NotFound", "") {
-				return nil
-			}
-
-			if detachNetworkInterfaceErr != nil {
-				return detachNetworkInterfaceErr
-			}
-
-			log.Printf("[DEBUG] Waiting for ENI (%s) to become detached", *eni.NetworkInterfaceId)
+		if eni.Attachment != nil && aws.StringValue(eni.Attachment.InstanceOwnerId) == "amazon-aws" {
+			// Hyperplane attached ENI.
+			// Wait for it to be moved into a removable state.
 			stateConf := &resource.StateChangeConf{
-				Pending: []string{"true"},
-				Target:  []string{"false"},
-				Refresh: networkInterfaceAttachedRefreshFunc(conn, *eni.NetworkInterfaceId),
-				Timeout: d.Timeout(schema.TimeoutDelete),
-			}
-			if _, err := stateConf.WaitForState(); err != nil {
-				return fmt.Errorf(
-					"Error waiting for ENI (%s) to become detached: %s", *eni.NetworkInterfaceId, err)
+				Pending: []string{
+					ec2.NetworkInterfaceStatusInUse,
+				},
+				Target: []string{
+					ec2.NetworkInterfaceStatusAvailable,
+					networkInterfaceStatusDeleted,
+				},
+				Refresh:                   networkInterfaceStateRefresh(conn, eniId),
+				Timeout:                   timeout,
+				Delay:                     10 * time.Second,
+				MinTimeout:                5 * time.Second,
+				ContinuousTargetOccurence: 10,
 			}
-		}
 
-		deleteNetworkInterfaceParams := &ec2.DeleteNetworkInterfaceInput{
-			NetworkInterfaceId: eni.NetworkInterfaceId,
-		}
-		_, deleteNetworkInterfaceErr := conn.DeleteNetworkInterface(deleteNetworkInterfaceParams)
+			eniRaw, err := stateConf.WaitForState()
 
-		if isAWSErr(deleteNetworkInterfaceErr, "InvalidNetworkInterfaceID.NotFound", "") {
-			return nil
-		}
+			if err != nil {
+				return fmt.Errorf("error waiting for ENI (%s) to become available: %s", eniId, err)
+			}
 
-		if deleteNetworkInterfaceErr != nil {
-			return deleteNetworkInterfaceErr
+			eni = eniRaw.(*ec2.NetworkInterface)
 		}
-	}
-
-	return nil
-}
 
-func networkInterfaceAttachedRefreshFunc(conn *ec2.EC2, id string) resource.StateRefreshFunc {
-	return func() (interface{}, string, error) {
+		err = detachNetworkInterface(conn, eni, timeout)
 
-		describe_network_interfaces_request := &ec2.DescribeNetworkInterfacesInput{
-			NetworkInterfaceIds: []*string{aws.String(id)},
+		if err != nil {
+			return err
 		}
-		describeResp, err := conn.DescribeNetworkInterfaces(describe_network_interfaces_request)
 
-		if isAWSErr(err, "InvalidNetworkInterfaceID.NotFound", "") {
-			return 42, "false", nil
-		}
+		err = deleteNetworkInterface(conn, eniId)
 
 		if err != nil {
-			return nil, "", err
+			return err
 		}
-
-		eni := describeResp.NetworkInterfaces[0]
-		hasAttachment := strconv.FormatBool(eni.Attachment != nil)
-		log.Printf("[DEBUG] ENI %s has attachment state %s", id, hasAttachment)
-		return eni, hasAttachment, nil
 	}
+
+	return nil
 }
 
 func initSecurityGroupRule(ruleMap map[string]map[string]interface{}, perm *ec2.IpPermission, desc string) map[string]interface{} {

aws/resource_aws_subnet.go

@@ -24,7 +24,7 @@ func resourceAwsSubnet() *schema.Resource {
 
 		Timeouts: &schema.ResourceTimeout{
 			Create: schema.DefaultTimeout(10 * time.Minute),
-			Delete: schema.DefaultTimeout(20 * time.Minute),
+			Delete: schema.DefaultTimeout(30 * time.Minute),
 		},
 
 		SchemaVersion: 1,
@@ -319,8 +319,8 @@ func resourceAwsSubnetDelete(d *schema.ResourceData, meta interface{}) error {
 
 	log.Printf("[INFO] Deleting subnet: %s", d.Id())
 
-	if err := deleteLingeringLambdaENIs(conn, d, "subnet-id"); err != nil {
-		return fmt.Errorf("Failed to delete Lambda ENIs: %s", err)
+	if err := deleteLingeringLambdaENIs(conn, "subnet-id", d.Id(), d.Timeout(schema.TimeoutDelete)); err != nil {
+		return fmt.Errorf("error deleting Lambda ENIs using subnet (%s): %s", d.Id(), err)
 	}
 
 	req := &ec2.DeleteSubnetInput{

website/docs/r/security_group.html.markdown

@@ -186,7 +186,7 @@ In addition to all arguments above, the following attributes are exported:
 configuration options:
 
 - `create` - (Default `10 minutes`) How long to wait for a security group to be created.
-- `delete` - (Default `10 minutes`) How long to wait for a security group to be deleted.
+- `delete` - (Default `30 minutes`) How long to wait for a security group to be deleted.
 
 ## Import
 

website/docs/r/subnet.html.markdown

@@ -69,6 +69,14 @@ In addition to all arguments above, the following attributes are exported:
 * `ipv6_cidr_block_association_id` - The association ID for the IPv6 CIDR block.
 * `owner_id` - The ID of the AWS account that owns the subnet.
 
+## Timeouts
+
+`aws_subnet` provides the following [Timeouts](/docs/configuration/resources.html#timeouts)
+configuration options:
+
+- `create` - (Default `10 minutes`) How long to wait for a subnet to be created.
+- `delete` - (Default `30 minutes`) How long to wait for a subnet to be deleted.
+
 ## Import
 
 Subnets can be imported using the `subnet id`, e.g.