Add VlanId in the cmdAdd Result struct

This VlanId will appear in the prevResult during cmdDel request

CleanUp Pod Network using vlanId from prevResult in CNI itself

Skip processing Delete request if prevResult is nil
Add Logging vlanId to ipamd

Add support to test with containerd nodegroup in pod-eni test

Add check for empty Netns() in cni

cmd/routed-eni-cni-plugin/cni.go

@@ -22,6 +22,7 @@ import (
 	"net"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
 
 	"github.com/containernetworking/cni/pkg/skel"
@@ -43,6 +44,7 @@ import (
 )
 
 const ipamdAddress = "127.0.0.1:50051"
+const vlanInterfaceName = "vlanId"
 
 var version string
 
@@ -95,6 +97,12 @@ func LoadNetConf(bytes []byte) (*NetConf, logger.Logger, error) {
 		return nil, nil, errors.Wrap(err, "add cmd: error loading config from args")
 	}
 
+	if conf.RawPrevResult != nil {
+		if err := cniSpecVersion.ParsePrevResult(&conf.NetConf); err != nil {
+			return nil, nil, fmt.Errorf("could not parse prevResult: %v", err)
+		}
+	}
+
 	logConfig := logger.Configuration{
 		LogLevel:    conf.PluginLogLevel,
 		LogLocation: conf.PluginLogFile,
@@ -122,6 +130,8 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 	log.Infof("Received CNI add request: ContainerID(%s) Netns(%s) IfName(%s) Args(%s) Path(%s) argsStdinData(%s)",
 		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData)
 
+	log.Infof("Prev Result: %v\n", conf.PrevResult)
+
 	var k8sArgs K8sArgs
 	if err := cniTypes.LoadArgs(args.Args, &k8sArgs); err != nil {
 		log.Errorf("Failed to load k8s config from arg: %v", err)
@@ -194,14 +204,12 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 	var hostVethName string
 	if r.PodVlanId != 0 {
 		hostVethName = generateHostVethName("vlan", string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-
 		err = driverClient.SetupPodENINetwork(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.PodVlanId), r.PodENIMAC,
 			r.PodENISubnetGW, int(r.ParentIfIndex), mtu, log)
 	} else {
 		// build hostVethName
 		// Note: the maximum length for linux interface name is 15
 		hostVethName = generateHostVethName(conf.VethPrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-
 		err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.DeviceNumber), r.VPCv4CIDRs, r.UseExternalSNAT, mtu, log)
 	}
 
@@ -241,12 +249,15 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 
 	hostInterface := &current.Interface{Name: hostVethName}
 	containerInterface := &current.Interface{Name: args.IfName, Sandbox: args.Netns}
+	vlanInterface := &current.Interface{Name: vlanInterfaceName, Mac: fmt.Sprint(r.PodVlanId)}
+	log.Infof("Using vlanInterface: %v", vlanInterface)
 
 	result := &current.Result{
 		IPs: ips,
 		Interfaces: []*current.Interface{
 			hostInterface,
 			containerInterface,
+			vlanInterface,
 		},
 	}
 
@@ -270,6 +281,8 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 	driverClient driver.NetworkAPIs) error {
 
 	conf, log, err := LoadNetConf(args.StdinData)
+	log.Infof("Prev Result: %v\n", conf.PrevResult)
+
 	if err != nil {
 		return errors.Wrap(err, "add cmd: error loading config from args")
 	}
@@ -283,6 +296,39 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 		return errors.Wrap(err, "del cmd: failed to load k8s config from args")
 	}
 
+	if args.Netns == "" {
+		log.Info("Netns() is empty, so network already cleanedup. Nothing to do")
+		return nil
+	}
+	prevResult, ok := conf.PrevResult.(*current.Result)
+
+	// Try to use prevResult if available
+	// prevResult might not be availabe, if we are still using older cni spec < 0.4.0.
+	// So we should fallback to the old clean up method
+	if ok {
+		for _, iface := range prevResult.Interfaces {
+			if iface.Name == vlanInterfaceName {
+				podVlanId, err := strconv.Atoi(iface.Mac)
+				if err != nil {
+					return errors.Wrap(err, "Failed to parse vlanId from prevResult")
+				}
+				// podVlanId == 0 means pod is not using branch ENI
+				// then fallback to existing cleanup
+				if podVlanId == 0 {
+					break
+				}
+				// if podVlanId != 0 means pod is using branch ENI
+				err = cleanUpPodENI(podVlanId, log, args.ContainerID, driverClient)
+				if err != nil {
+					return err
+				}
+				log.Infof("Received del network response for pod %s namespace %s sandbox %s with vlanId: %v", string(k8sArgs.K8S_POD_NAME),
+					string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), podVlanId)
+				return nil
+			}
+		}
+	}
+
 	// notify local IP address manager to free secondary IP
 	// Set up a connection to the server.
 	conn, err := grpcClient.Dial(ipamdAddress, grpc.WithInsecure())
@@ -362,6 +408,16 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 	return nil
 }
 
+func cleanUpPodENI(podVlanId int, log logger.Logger, containerId string, driverClient driver.NetworkAPIs) error {
+	err := driverClient.TeardownPodENINetwork(podVlanId, log)
+	if err != nil {
+		log.Errorf("Failed on TeardownPodNetwork for container ID %s: %v",
+			containerId, err)
+		return errors.Wrap(err, "del cmd: failed on tear down pod network")
+	}
+	return nil
+}
+
 func main() {
 	log := logger.DefaultLogger()
 	about := fmt.Sprintf("AWS CNI %s", version)

misc/10-aws.conflist

@@ -1,5 +1,5 @@
 {
-  "cniVersion": "0.3.1",
+  "cniVersion": "0.4.0",
   "name": "aws-cni",
   "plugins": [
     {

pkg/ipamd/rpc_handler.go

@@ -110,6 +110,8 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp
 					ipv4Addr = firstENI.PrivateIP
 					branchENIMAC = firstENI.IfAddress
 					vlanID = firstENI.VlanID
+					log.Infof("Pod vlandId: %d", vlanID)
+
 					if ipv4Addr == "" || branchENIMAC == "" || vlanID == 0 {
 						log.Errorf("Failed to parse pod-ENI annotation: %s", val)
 						return &failureResponse, nil

test/e2e/pod-eni/securiy_group_per_pod_suite_test.go

@@ -23,8 +23,9 @@ import (
 	awsUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/utils"
 	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
-
 	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/vpc"
+	v1 "k8s.io/api/core/v1"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -52,6 +53,8 @@ var (
 	clusterRoleName string
 	// NodeSecurityGroupId for Node-Node communication
 	nodeSecurityGroupID string
+
+	node v1.Node
 )
 
 func TestSecurityGroupForPods(t *testing.T) {
@@ -116,7 +119,7 @@ var _ = BeforeSuite(func() {
 
 	// Get ref to any node from newly created nodegroup
 	By("Getting providerID of the node")
-	node := nodeList.Items[0]
+	node = nodeList.Items[0]
 	providerID := node.Spec.ProviderID
 	Expect(len(providerID)).To(BeNumerically(">", 0))
 

test/e2e/pod-eni/securiy_group_per_pod_test.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input"
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/agent"
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
 	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
@@ -32,13 +33,21 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
+type TestType int
+
+const (
+	NetworkingTearDownSucceeds TestType = iota
+	NetworkingSetupSucceeds
+)
+
 var _ = Describe("Security Group for Pods Test", func() {
 	var (
 		// The Pod labels for client and server in order to retrieve the
 		// client and server Pods belonging to a Deployment/Jobs
-		labelKey          = "app"
-		serverPodLabelVal = "server-pod"
-		clientPodLabelVal = "client-pod"
+		labelKey           = "app"
+		serverPodLabelVal  = "server-pod"
+		clientPodLabelVal  = "client-pod"
+		busyboxPodLabelVal = "busybox-pod"
 		// The Security Group Policy take list of Pod Label Value and if the
 		// Pod has any label in the list, it should get Branch ENI
 		branchPodLabelVal       []string
@@ -264,8 +273,107 @@ var _ = Describe("Security Group for Pods Test", func() {
 			It("TCP liveness probe will succeed", func() {})
 		})
 	})
+
+	Context("Verify HostNetworking", func() {
+		BeforeEach(func() {
+			// BusyBox Pods will get Branch ENI
+			branchPodLabelVal = []string{busyboxPodLabelVal}
+		})
+		It("Deploy BusyBox Pods with branch ENI and verify HostNetworking", func() {
+			deployment := manifest.NewBusyBoxDeploymentBuilder().
+				Replicas(totalBranchInterface/asgSize).
+				PodLabel(labelKey, busyboxPodLabelVal).
+				NodeName(node.Name).
+				Build()
+
+			By("creating a deployment to launch pod using Branch ENI")
+			_, err = f.K8sResourceManagers.DeploymentManager().
+				CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("getting the list of pods using BranchENI")
+			podList, err := f.K8sResourceManagers.
+				PodManager().
+				GetPodsWithLabelSelector(labelKey, busyboxPodLabelVal)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("generating the pod networking validation input to be passed to tester")
+			input, err := GetPodNetworkingValidationInput(podList).Serialize()
+			Expect(err).NotTo(HaveOccurred())
+
+			By("validating host networking setup is setup correctly")
+			ValidateHostNetworking(NetworkingSetupSucceeds, input)
+
+			By("deleting the deployment to test teardown")
+			err = f.K8sResourceManagers.DeploymentManager().
+				DeleteAndWaitTillDeploymentIsDeleted(deployment)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("waiting to allow CNI to tear down networking for terminated pods")
+			time.Sleep(time.Second * 60)
+
+			By("validating host networking is teared down correctly")
+			ValidateHostNetworking(NetworkingTearDownSucceeds, input)
+		})
+	})
 })
 
+func GetPodNetworkingValidationInput(podList v1.PodList) input.PodNetworkingValidationInput {
+	ip := input.PodNetworkingValidationInput{
+		VethPrefix:  "vlan",
+		PodList:     []input.Pod{},
+		ValidateMTU: true,
+		MTU:         9001,
+	}
+
+	for _, pod := range podList.Items {
+		ip.PodList = append(ip.PodList, input.Pod{
+			PodName:        pod.Name,
+			PodNamespace:   pod.Namespace,
+			PodIPv4Address: pod.Status.PodIP,
+		})
+	}
+	return ip
+}
+
+func ValidateHostNetworking(testType TestType, podValidationInputString string) {
+	testerArgs := []string{fmt.Sprintf("-pod-networking-validation-input=%s",
+		podValidationInputString)}
+
+	if NetworkingSetupSucceeds == testType {
+		testerArgs = append(testerArgs, "-test-setup=true", "-test-ppsg=true")
+	} else if NetworkingTearDownSucceeds == testType {
+		testerArgs = append(testerArgs, "-test-cleanup=true", "-test-ppsg=true")
+	}
+
+	testContainer := manifest.NewTestHelperContainer().
+		Command([]string{"./networking"}).
+		Args(testerArgs).
+		Build()
+
+	testPod := manifest.NewDefaultPodBuilder().
+		Container(testContainer).
+		NodeName(node.Name).
+		HostNetwork(true).
+		Build()
+
+	By("creating pod to test host networking setup")
+	testPod, err := f.K8sResourceManagers.PodManager().
+		CreateAndWaitTillPodCompleted(testPod)
+	Expect(err).ToNot(HaveOccurred())
+
+	logs, errLogs := f.K8sResourceManagers.PodManager().
+		PodLogs(testPod.Namespace, testPod.Name)
+	Expect(errLogs).ToNot(HaveOccurred())
+
+	fmt.Fprintln(GinkgoWriter, logs)
+
+	By("deleting the host networking setup pod")
+	err = f.K8sResourceManagers.PodManager().
+		DeleteAndWaitTillPodDeleted(testPod)
+	Expect(err).ToNot(HaveOccurred())
+}
+
 func ValidatePodsHaveBranchENI(podList v1.PodList) error {
 	for _, pod := range podList.Items {
 		if val, ok := pod.Annotations["vpc.amazonaws.com/pod-eni"]; ok {

test/framework/resources/aws/utils/nodegroup.go

@@ -30,6 +30,11 @@ import (
 
 const CreateNodeGroupCFNTemplateURL = "https://raw.githubusercontent.com/awslabs/amazon-eks-ami/master/amazon-eks-nodegroup.yaml"
 
+// Docker will be default, if not specified
+const (
+	CONTAINERD = "containerd"
+)
+
 type NodeGroupProperties struct {
 	// Required to verify the node is up and ready
 	NgLabelKey string
@@ -44,6 +49,9 @@ type NodeGroupProperties struct {
 	Subnet       []string
 	InstanceType string
 	KeyPairName  string
+
+	// optional: specify container runtime
+	ContainerRuntime string
 }
 
 type ClusterVPCConfig struct {
@@ -95,6 +103,11 @@ func CreateAndWaitTillSelfManagedNGReady(f *framework.Framework, properties Node
 		kubeletExtraArgs += fmt.Sprintf(" --max-pods=%d", maxPods)
 	}
 
+	containerRuntime := properties.ContainerRuntime
+	if containerRuntime != "" {
+		bootstrapArgs += fmt.Sprintf(" --container-runtime %s", containerRuntime)
+	}
+
 	asgSizeString := strconv.Itoa(properties.AsgSize)
 
 	createNgStackParams := []*cloudformation.Parameter{

test/framework/resources/k8s/utils/daemonset.go

@@ -76,7 +76,7 @@ func updateDaemonsetEnvVarsAndWait(f *framework.Framework, dsName string, dsName
 	if dsName != utils.MultusNodeName {
 		_, err := f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(dsNamespace, utils.MultusNodeName)
 		if err == nil {
-			By("Restarting Multus daemonset")
+			By("Restarting Multus daemonset to use the update aws-node changes")
 			td := time.Now()
 			updateDaemonsetEnvVarsAndWait(f, utils.MultusNodeName, dsNamespace, utils.MultusContainerName, map[string]string{
 				"forceUpdatedAt": td.String(),