diff --git a/test/agent/cmd/networking/tester/network.go b/test/agent/cmd/networking/tester/network.go index de9ccfa929..6320693dd0 100644 --- a/test/agent/cmd/networking/tester/network.go +++ b/test/agent/cmd/networking/tester/network.go @@ -69,6 +69,16 @@ func TestNetworkingSetupForRegularPod(podNetworkingValidationInput input.PodNetw continue } + // Validate MTU value if it is set to true + if podNetworkingValidationInput.ValidateMTU { + if link.Attrs().MTU != podNetworkingValidationInput.MTU { + validationErrors = append(validationErrors, + fmt.Errorf("MTU value %v for pod: %s on veth pair: %s failed to match the expected value: %v", link.Attrs().MTU, pod.PodName, hostVethName, podNetworkingValidationInput.MTU)) + } else { + log.Printf("Found Valid MTU value:%d for pod: %s on veth Pair: %s\n", link.Attrs().MTU, pod.PodName, hostVethName) + } + } + // Verify IP Link for the Pod is UP isLinkUp := strings.Contains(link.Attrs().Flags.String(), "up") if !isLinkUp { diff --git a/test/agent/pkg/input/input.go b/test/agent/pkg/input/input.go index 1fb187b475..f31f1ec596 100644 --- a/test/agent/pkg/input/input.go +++ b/test/agent/pkg/input/input.go @@ -13,6 +13,8 @@ package input +import "encoding/json" + type TestStatus struct { SuccessCount int FailureCount int @@ -32,6 +34,10 @@ type PodNetworkingValidationInput struct { VethPrefix string // List of pod to validate the networking PodList []Pod + // Should Validate MTU value, by default it will false + ValidateMTU bool + // Expected MTU value + MTU int } type Pod struct { @@ -45,3 +51,11 @@ type Pod struct { // from the Secondary ENI IsIPFromSecondaryENI bool } + +func (ip PodNetworkingValidationInput) Serialize() (string, error) { + inputBytes, err := json.Marshal(ip) + if err != nil { + return "", err + } + return string(inputBytes), nil +} diff --git a/test/framework/utils/utils.go b/test/framework/utils/utils.go index 6fea0f723f..34b105283e 100644 --- a/test/framework/utils/utils.go +++ b/test/framework/utils/utils.go @@ -14,6 +14,7 @@ package utils import ( + appsV1 "k8s.io/api/apps/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) @@ -25,3 +26,13 @@ func NamespacedName(obj v1.Object) types.NamespacedName { Name: obj.GetName(), } } + +func GetEnvValueForKeyFromDaemonSet(key string, ds *appsV1.DaemonSet) string { + envVar := ds.Spec.Template.Spec.Containers[0].Env + for _, env := range envVar { + if env.Name == key { + return env.Value + } + } + return "" +} diff --git a/test/integration-new/cni/host_networking_test.go b/test/integration-new/cni/host_networking_test.go index 71cd292818..f9761a1e22 100644 --- a/test/integration-new/cni/host_networking_test.go +++ b/test/integration-new/cni/host_networking_test.go @@ -14,14 +14,15 @@ package cni import ( - "encoding/json" "fmt" + "strconv" "time" "github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input" v1 "k8s.io/api/core/v1" "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" . "github.com/onsi/ginkgo" @@ -39,6 +40,13 @@ const ( NetworkingSetupFails ) +const ( + AWS_VPC_ENI_MTU = "AWS_VPC_ENI_MTU" + AWS_VPC_K8S_CNI_VETHPREFIX = "AWS_VPC_K8S_CNI_VETHPREFIX" + NEW_MTU_VAL = 1300 + NEW_VETH_PREFIX = "veth" +) + var _ = Describe("test host networking", func() { var err error var podLabelKey = "app" @@ -71,7 +79,8 @@ var _ = Describe("test host networking", func() { Should(BeNumerically(">", 0)) By("generating the pod networking validation input to be passed to tester") - input := GetPodNetworkingValidationInput(interfaceTypeToPodList) + input, err := GetPodNetworkingValidationInput(interfaceTypeToPodList).Serialize() + Expect(err).NotTo(HaveOccurred()) By("validating host networking setup is setup correctly") ValidateHostNetworking(NetworkingSetupSucceeds, input) @@ -87,6 +96,64 @@ var _ = Describe("test host networking", func() { By("validating host networking is teared down correctly") ValidateHostNetworking(NetworkingTearDownSucceeds, input) }) + + It("Validate Host Networking setup after changing MTU and Veth Prefix", func() { + deployment := manifest.NewBusyBoxDeploymentBuilder(). + Replicas(6). + PodLabel(podLabelKey, podLabelVal). + NodeName(primaryNode.Name). + Build() + + By("Configuring Veth Prefix and MTU value on aws-node daemonset") + ds, err := f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(utils.AwsNodeNamespace, utils.AwsNodeName) + Expect(err).NotTo(HaveOccurred()) + + oldMTU := utils.GetEnvValueForKeyFromDaemonSet(AWS_VPC_ENI_MTU, ds) + oldVethPrefix := utils.GetEnvValueForKeyFromDaemonSet(AWS_VPC_K8S_CNI_VETHPREFIX, ds) + + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + AWS_VPC_ENI_MTU: strconv.Itoa(NEW_MTU_VAL), + AWS_VPC_K8S_CNI_VETHPREFIX: NEW_VETH_PREFIX, + }) + + By("creating a deployment to launch pods") + deployment, err = f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + + By("getting the list of pods using IP from primary and secondary ENI") + interfaceTypeToPodList := + GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal) + + By("generating the pod networking validation input to be passed to tester") + podNetworkingValidationInput := GetPodNetworkingValidationInput(interfaceTypeToPodList) + podNetworkingValidationInput.VethPrefix = NEW_VETH_PREFIX + podNetworkingValidationInput.ValidateMTU = true + podNetworkingValidationInput.MTU = NEW_MTU_VAL + input, err := podNetworkingValidationInput.Serialize() + Expect(err).NotTo(HaveOccurred()) + + By("validating host networking setup is setup correctly with MTU check as well") + ValidateHostNetworking(NetworkingSetupSucceeds, input) + + // Restore MTU and Veth Prefix + By("Restoring MTU value and Veth Prefix to old values") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName, map[string]string{ + AWS_VPC_ENI_MTU: oldMTU, + AWS_VPC_K8S_CNI_VETHPREFIX: oldVethPrefix, + }) + + By("deleting the deployment to test teardown") + err = f.K8sResourceManagers.DeploymentManager(). + DeleteAndWaitTillDeploymentIsDeleted(deployment) + Expect(err).ToNot(HaveOccurred()) + + By("waiting to allow CNI to tear down networking for terminated pods") + time.Sleep(time.Second * 60) + + By("validating host networking is teared down correctly") + ValidateHostNetworking(NetworkingTearDownSucceeds, input) + }) }) Context("when host networking is tested on invalid input", func() { @@ -103,9 +170,10 @@ var _ = Describe("test host networking", func() { CreatAndWaitTillRunning(parkingPod) Expect(err).ToNot(HaveOccurred()) - validInput := GetPodNetworkingValidationInput(InterfaceTypeToPodList{ + validInput, err := GetPodNetworkingValidationInput(InterfaceTypeToPodList{ PodsOnPrimaryENI: []v1.Pod{*parkingPod}, - }) + }).Serialize() + Expect(err).NotTo(HaveOccurred()) By("first validating the tester work on valid input") ValidateHostNetworking(NetworkingSetupSucceeds, validInput) @@ -114,9 +182,11 @@ var _ = Describe("test host networking", func() { invalidPod := parkingPod.DeepCopy() invalidPod.Status.PodIP = "1.1.1.1" - invalidInput := GetPodNetworkingValidationInput(InterfaceTypeToPodList{ + invalidInput, err := GetPodNetworkingValidationInput(InterfaceTypeToPodList{ PodsOnPrimaryENI: []v1.Pod{*invalidPod}, - }) + }).Serialize() + Expect(err).NotTo(HaveOccurred()) + ValidateHostNetworking(NetworkingSetupFails, invalidInput) By("validating the tester fails when invalid namespace is passed") @@ -124,9 +194,10 @@ var _ = Describe("test host networking", func() { // veth pair name is generated using namespace+name so the test should fail invalidPod.Namespace = "different" - invalidInput = GetPodNetworkingValidationInput(InterfaceTypeToPodList{ + invalidInput, err = GetPodNetworkingValidationInput(InterfaceTypeToPodList{ PodsOnPrimaryENI: []v1.Pod{*invalidPod}, - }) + }).Serialize() + Expect(err).NotTo(HaveOccurred()) ValidateHostNetworking(NetworkingSetupFails, invalidInput) @@ -194,12 +265,13 @@ func ValidateHostNetworking(testType TestType, podValidationInputString string) // GetPodNetworkingValidationInput returns input string containing the list of pods for which // the host networking has to be tested -func GetPodNetworkingValidationInput(interfaceTypeToPodList InterfaceTypeToPodList) string { +func GetPodNetworkingValidationInput(interfaceTypeToPodList InterfaceTypeToPodList) input.PodNetworkingValidationInput { ip := input.PodNetworkingValidationInput{ VPCCidrRange: vpcCIDRs, VethPrefix: "eni", PodList: []input.Pod{}, + ValidateMTU: false, } for _, primaryENIPod := range interfaceTypeToPodList.PodsOnPrimaryENI { @@ -220,8 +292,5 @@ func GetPodNetworkingValidationInput(interfaceTypeToPodList InterfaceTypeToPodLi }) } - inputBytes, err := json.Marshal(ip) - Expect(err).ToNot(HaveOccurred()) - - return string(inputBytes) + return ip } diff --git a/test/integration-new/ipamd/env_vars_test.go b/test/integration-new/ipamd/env_vars_test.go index 32f0de50e2..ae4f6c1786 100644 --- a/test/integration-new/ipamd/env_vars_test.go +++ b/test/integration-new/ipamd/env_vars_test.go @@ -1,8 +1,6 @@ package ipamd import ( - "regexp" - "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" @@ -13,89 +11,19 @@ import ( ) var ( - ds *appsV1.DaemonSet - hostNetworkDeploymentSpec *appsV1.Deployment - hostNetworkPod v1.Pod + ds *appsV1.DaemonSet ) const ( - AWS_VPC_ENI_MTU = "AWS_VPC_ENI_MTU" - AWS_VPC_K8S_CNI_LOG_FILE = "AWS_VPC_K8S_CNI_LOG_FILE" - AWS_VPC_K8S_CNI_VETHPREFIX = "AWS_VPC_K8S_CNI_VETHPREFIX" - POD_VOL_LABEL_KEY = "MountVolume" - POD_VOL_LABEL_VAL = "true" - VOLUME_NAME = "ipamd-logs" - VOLUME_MOUNT_PATH = "/var/log/aws-routed-eni/" + AWS_VPC_K8S_CNI_LOG_FILE = "AWS_VPC_K8S_CNI_LOG_FILE" + POD_VOL_LABEL_KEY = "MountVolume" + POD_VOL_LABEL_VAL = "true" + VOLUME_NAME = "ipamd-logs" + VOLUME_MOUNT_PATH = "/var/log/aws-routed-eni/" ) var _ = Describe("cni env test", func() { Context("CNI Environment Variables", func() { - It("Changing AWS_VPC_ENI_MTU and AWS_VPC_K8S_CNI_VETHPREFIX", func() { - setupHostNetworkPod() - - currMTUVal := getEnvValueForKey(AWS_VPC_ENI_MTU) - Expect(currMTUVal).NotTo(Equal("")) - - currVETHPrefix := getEnvValueForKey(AWS_VPC_K8S_CNI_VETHPREFIX) - Expect(currVETHPrefix).NotTo(Equal("")) - - k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, DAEMONSET, NAMESPACE, DAEMONSET, map[string]string{ - AWS_VPC_ENI_MTU: "1300", - AWS_VPC_K8S_CNI_VETHPREFIX: "veth", - }) - - By("Deploying a BusyBox deployment") - deploymentSpec := manifest.NewBusyBoxDeploymentBuilder(). - Namespace("default"). - Name("busybox"). - Replicas(1). - NodeName(primaryNode.Name). - Build() - - _, err := f.K8sResourceManagers. - DeploymentManager(). - CreateAndWaitTillDeploymentIsReady(deploymentSpec, utils.DefaultDeploymentReadyTimeout) - Expect(err).ToNot(HaveOccurred()) - - stdout, _, err := f.K8sResourceManagers.PodManager().PodExec("default", hostNetworkPod.Name, []string{"ifconfig"}) - Expect(err).NotTo(HaveOccurred()) - - re := regexp.MustCompile(`\n`) - input := re.ReplaceAllString(stdout, "") - - re = regexp.MustCompile(`eth.*lo`) - eth := re.FindStringSubmatch(input)[0] - - re = regexp.MustCompile(`MTU:[0-9]*`) - mtus := re.FindAllStringSubmatch(eth, -1) - - By("Validating new MTU value") - // Validate MTU - for _, m := range mtus { - Expect(m[0]).To(Equal("MTU:1300")) - } - - By("Validating new VETH Prefix") - // Validate VETH Prefix - // Adding the new MTU value to below regex ensures that we are checking the recently created - // veth and not any older entries - re = regexp.MustCompile(`veth.*MTU:1300`) - veth := re.FindAllString(input, -1) - - Expect(len(veth)).NotTo(Equal(0)) - - By("Deleting BusyBox Deployment") - err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deploymentSpec) - Expect(err).NotTo(HaveOccurred()) - - By("Restoring old value on daemonset") - restoreOldValues(map[string]string{ - AWS_VPC_ENI_MTU: currMTUVal, - AWS_VPC_K8S_CNI_VETHPREFIX: currVETHPrefix, - }) - cleanupHostNetworkPod() - }) - It("Changing AWS_VPC_K8S_CNI_LOG_FILE", func() { By("Deploying a host network deployment with Volume mount") curlContainer := manifest.NewBusyBoxContainerBuilder().Image("curlimages/curl:7.76.1").Name("curler").Build() @@ -139,7 +67,11 @@ var _ = Describe("cni env test", func() { Expect(len(pods.Items)).Should(BeNumerically(">", 0)) podWithVol := pods.Items[0] - currLogFilepath := getEnvValueForKey(AWS_VPC_K8S_CNI_LOG_FILE) + + ds, err = f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(NAMESPACE, DAEMONSET) + Expect(err).NotTo(HaveOccurred()) + + currLogFilepath := utils.GetEnvValueForKeyFromDaemonSet(AWS_VPC_K8S_CNI_LOG_FILE, ds) Expect(currLogFilepath).NotTo(Equal("")) newLogFile := "ipamd_test.log" @@ -163,50 +95,6 @@ var _ = Describe("cni env test", func() { }) }) -func getEnvValueForKey(key string) string { - ds, err = f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(NAMESPACE, DAEMONSET) - Expect(err).NotTo(HaveOccurred()) - - envVar := ds.Spec.Template.Spec.Containers[0].Env - for _, env := range envVar { - if env.Name == key { - return env.Value - } - } - return "" -} - -func setupHostNetworkPod() { - By("Deploying a Host Network Pod") - curlContainer := manifest.NewBusyBoxContainerBuilder().Image("curlimages/curl:7.76.1").Name("curler").Build() - - hostNetworkDeploymentSpec = manifest.NewDefaultDeploymentBuilder(). - Namespace("default"). - Name("host-network"). - Replicas(1). - HostNetwork(true). - Container(curlContainer). - PodLabel(HOST_POD_LABEL_KEY, HOST_POD_LABEL_VAL). - NodeName(primaryNode.Name). - Build() - - _, err := f.K8sResourceManagers. - DeploymentManager(). - CreateAndWaitTillDeploymentIsReady(hostNetworkDeploymentSpec, utils.DefaultDeploymentReadyTimeout) - Expect(err).NotTo(HaveOccurred()) - - pods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(HOST_POD_LABEL_KEY, HOST_POD_LABEL_VAL) - Expect(err).NotTo(HaveOccurred()) - - hostNetworkPod = pods.Items[0] -} - -func cleanupHostNetworkPod() { - err = f.K8sResourceManagers.DeploymentManager(). - DeleteAndWaitTillDeploymentIsDeleted(hostNetworkDeploymentSpec) - Expect(err).ToNot(HaveOccurred()) -} - func restoreOldValues(oldVals map[string]string) { k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, DAEMONSET, NAMESPACE, DAEMONSET, oldVals) }