add custom networking e2e test suite

test/e2e/custom-networking/custom_networking_suite_test.go

@@ -124,18 +124,18 @@ var _ = BeforeSuite(func() {
 			Build()
 		Expect(err).ToNot(HaveOccurred())
 
+		// For deleting later
+		customNetworkingSubnetIDList = append(customNetworkingSubnetIDList, subnetID)
+		eniConfigList = append(eniConfigList, eniConfig.DeepCopy())
+
 		By("creating the ENIConfig with az name")
 		err = f.K8sResourceManagers.CustomResourceManager().CreateResource(eniConfig)
 		Expect(err).ToNot(HaveOccurred())
-
-		// For deleting later
-		customNetworkingSubnetIDList = append(customNetworkingSubnetIDList, subnetID)
-		eniConfigList = append(eniConfigList, eniConfig)
 	}
 
 	By("enabling custom networking on aws-node DaemonSet")
-	k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, "aws-node",
-		"kube-system", "aws-node", map[string]string{
+	k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName,
+		utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{
 			"AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG": "true",
 			"ENI_CONFIG_LABEL_DEF":               "failure-domain.beta.kubernetes.io/zone",
 			"WARM_ENI_TARGET":                    "0",
@@ -186,8 +186,8 @@ var _ = AfterSuite(func() {
 	Expect(err).ToNot(HaveOccurred())
 
 	By("disabling custom networking on aws-node DaemonSet")
-	k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, "aws-node",
-		"kube-system", "aws-node", map[string]struct{}{
+	k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName,
+		utils.AwsNodeNamespace, utils.AwsNodeName, map[string]struct{}{
 			"AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG": {},
 			"ENI_CONFIG_LABEL_DEF":               {},
 			"WARM_ENI_TARGET":                    {},

test/e2e/custom-networking/custom_networking_test.go

@@ -17,8 +17,11 @@ import (
 	"fmt"
 	"net"
 	"strconv"
+	"time"
 
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
+	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -39,7 +42,6 @@ var _ = Describe("Custom Networking Test", func() {
 	)
 
 	Context("when creating deployment targeted using ENIConfig", func() {
-
 		BeforeEach(func() {
 			podLabelKey = "role"
 			podLabelVal = "custom-networking-test"
@@ -59,7 +61,7 @@ var _ = Describe("Custom Networking Test", func() {
 				Build()
 
 			deployment, err = f.K8sResourceManagers.DeploymentManager().
-				CreateAndWaitTillDeploymentIsReady(deployment)
+				CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout)
 			Expect(err).ToNot(HaveOccurred())
 
 			podList, err = f.K8sResourceManagers.PodManager().
@@ -127,4 +129,64 @@ var _ = Describe("Custom Networking Test", func() {
 			It("should fail to connect", func() {})
 		})
 	})
+
+	Context("when creating deployment on nodes that don't have ENIConfig", func() {
+		JustBeforeEach(func() {
+			By("deleting all existing ENIConfigs")
+			for _, eniConfig := range eniConfigList {
+				err = f.K8sResourceManagers.CustomResourceManager().
+					DeleteResource(eniConfig)
+				Expect(err).ToNot(HaveOccurred())
+			}
+		})
+
+		JustAfterEach(func() {
+			By("creating the deleted ENIConfigs")
+			for _, eniConfig := range eniConfigList {
+				err = f.K8sResourceManagers.CustomResourceManager().
+					CreateResource(eniConfig)
+				Expect(err).ToNot(HaveOccurred())
+			}
+		})
+
+		It("deployment should not become ready", func() {
+			By("getting the list of nodes created")
+			nodeList, err := f.K8sResourceManagers.NodeManager().
+				GetNodes(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal)
+			Expect(err).ToNot(HaveOccurred())
+
+			var instanceIDs []string
+			for _, node := range nodeList.Items {
+				instanceIDs = append(instanceIDs, k8sUtils.GetInstanceIDFromNode(node))
+			}
+
+			By("terminating all the nodes")
+			err = f.CloudServices.EC2().TerminateInstance(instanceIDs)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("waiting for the node to be removed")
+			time.Sleep(time.Second * 120)
+
+			By("waiting for all nodes to become ready")
+			err = f.K8sResourceManagers.NodeManager().
+				WaitTillNodesReady(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal,
+					nodeGroupProperties.AsgSize)
+			Expect(err).ToNot(HaveOccurred())
+
+			deployment := manifest.NewBusyBoxDeploymentBuilder().
+				Replicas(2).
+				NodeSelector(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal).
+				Build()
+
+			By("verifying deployment should not succeed")
+			deployment, err = f.K8sResourceManagers.DeploymentManager().
+				CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout)
+			Expect(err).To(HaveOccurred())
+
+			By("deleting the failed deployment")
+			err = f.K8sResourceManagers.DeploymentManager().
+				DeleteAndWaitTillDeploymentIsDeleted(deployment)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
 })

test/framework/resources/aws/services/ec2.go

@@ -31,6 +31,7 @@ type EC2 interface {
 	AuthorizeSecurityGroupEgress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error
 	RevokeSecurityGroupEgress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error
 	AssociateVPCCIDRBlock(vpcId string, cidrBlock string) (*ec2.AssociateVpcCidrBlockOutput, error)
+	TerminateInstance(instanceIDs []string) error
 	DisAssociateVPCCIDRBlock(associationID string) error
 	DescribeSubnet(subnetID string) (*ec2.DescribeSubnetsOutput, error)
 	CreateSubnet(cidrBlock string, vpcID string, az string) (*ec2.CreateSubnetOutput, error)
@@ -258,6 +259,15 @@ func (d *defaultEC2) DeleteKey(keyName string) error {
 	return err
 }
 
+func (d *defaultEC2) TerminateInstance(instanceIDs []string) error {
+	terminateInstanceInput := &ec2.TerminateInstancesInput{
+		DryRun:      nil,
+		InstanceIds: aws.StringSlice(instanceIDs),
+	}
+	_, err := d.EC2API.TerminateInstances(terminateInstanceInput)
+	return err
+}
+
 func NewEC2(session *session.Session) EC2 {
 	return &defaultEC2{
 		EC2API: ec2.New(session),

test/framework/resources/k8s/resources/deployment.go

@@ -26,15 +26,17 @@ import (
 )
 
 type DeploymentManager interface {
-	CreateAndWaitTillDeploymentIsReady(deployment *v1.Deployment) (*v1.Deployment, error)
+	CreateAndWaitTillDeploymentIsReady(deployment *v1.Deployment, timeout time.Duration) (*v1.Deployment, error)
 	DeleteAndWaitTillDeploymentIsDeleted(deployment *v1.Deployment) error
 }
 
 type defaultDeploymentManager struct {
 	k8sClient client.DelegatingClient
 }
 
-func (d defaultDeploymentManager) CreateAndWaitTillDeploymentIsReady(deployment *v1.Deployment) (*v1.Deployment, error) {
+// CreateAndWaitTillDeploymentIsReady creates and waits for deployment to become ready or timeout
+// with error if deployment doesn't become ready.
+func (d defaultDeploymentManager) CreateAndWaitTillDeploymentIsReady(deployment *v1.Deployment, timeout time.Duration) (*v1.Deployment, error) {
 	ctx := context.Background()
 	err := d.k8sClient.Create(ctx, deployment)
 	if err != nil {
@@ -45,7 +47,7 @@ func (d defaultDeploymentManager) CreateAndWaitTillDeploymentIsReady(deployment
 	time.Sleep(utils.PollIntervalShort)
 
 	observed := &v1.Deployment{}
-	return observed, wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+	return observed, wait.PollImmediate(utils.PollIntervalShort, timeout, func() (bool, error) {
 		if err := d.k8sClient.Get(ctx, utils.NamespacedName(deployment), observed); err != nil {
 			return false, err
 		}
@@ -56,9 +58,10 @@ func (d defaultDeploymentManager) CreateAndWaitTillDeploymentIsReady(deployment
 			return true, nil
 		}
 		return false, nil
-	}, ctx.Done())
+	})
 }
 
+//
 func (d defaultDeploymentManager) DeleteAndWaitTillDeploymentIsDeleted(deployment *v1.Deployment) error {
 	ctx := context.Background()
 	err := d.k8sClient.Delete(ctx, deployment)

test/framework/resources/k8s/resources/node.go

@@ -25,6 +25,7 @@ import (
 
 type NodeManager interface {
 	GetNodes(nodeLabelKey string, nodeLabelVal string) (v1.NodeList, error)
+	UpdateNode(oldNode *v1.Node, newNode *v1.Node) error
 	WaitTillNodesReady(nodeLabelKey string, nodeLabelVal string, asgSize int) error
 }
 
@@ -45,6 +46,10 @@ func (d *defaultNodeManager) GetNodes(nodeLabelKey string, nodeLabelVal string)
 	return nodeList, err
 }
 
+func (d *defaultNodeManager) UpdateNode(oldNode *v1.Node, newNode *v1.Node) error {
+	return d.k8sClient.Patch(context.Background(), newNode, client.MergeFrom(oldNode))
+}
+
 func (d *defaultNodeManager) WaitTillNodesReady(nodeLabelKey string, nodeLabelVal string, asgSize int) error {
 	return wait.PollImmediateUntil(utils.PollIntervalLong, func() (done bool, err error) {
 		nodeList, err := d.GetNodes(nodeLabelKey, nodeLabelVal)

test/framework/utils/const.go

@@ -25,4 +25,6 @@ const (
 	PollIntervalShort  = time.Second * 2
 	PollIntervalMedium = time.Second * 5
 	PollIntervalLong   = time.Second * 20
+
+	DefaultDeploymentReadyTimeout = time.Second * 120
 )

test/integration-new/cni/pod_networking_test.go

@@ -17,6 +17,8 @@ import (
 	"fmt"
 	"strconv"
 
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
 	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
 
@@ -96,7 +98,7 @@ var _ = Describe("test pod networking", func() {
 
 		primaryNodeDeployment, err = f.K8sResourceManagers.
 			DeploymentManager().
-			CreateAndWaitTillDeploymentIsReady(primaryNodeDeployment)
+			CreateAndWaitTillDeploymentIsReady(primaryNodeDeployment, utils.DefaultDeploymentReadyTimeout)
 		Expect(err).ToNot(HaveOccurred())
 
 		interfaceToPodListOnPrimaryNode =
@@ -121,7 +123,7 @@ var _ = Describe("test pod networking", func() {
 
 		secondaryNodeDeployment, err = f.K8sResourceManagers.
 			DeploymentManager().
-			CreateAndWaitTillDeploymentIsReady(secondaryNodeDeployment)
+			CreateAndWaitTillDeploymentIsReady(secondaryNodeDeployment, utils.DefaultDeploymentReadyTimeout)
 		Expect(err).ToNot(HaveOccurred())
 
 		interfaceToPodListOnSecondaryNode =

test/integration-new/cni/service_connectivity_test.go

@@ -71,7 +71,7 @@ var _ = Describe("test service connectivity", func() {
 
 		By("creating and waiting for deployment to be ready")
 		deployment, err = f.K8sResourceManagers.DeploymentManager().
-			CreateAndWaitTillDeploymentIsReady(deployment)
+			CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout)
 		Expect(err).ToNot(HaveOccurred())
 
 		service = manifest.NewHTTPService().