From efaeccdd1e771cc063beb94213e34684b9a9eb49 Mon Sep 17 00:00:00 2001 From: mmirecki Date: Mon, 1 Jun 2020 16:47:36 +0200 Subject: [PATCH] Discovery mode for tests --- test/conformance/tests/sriov_operator.go | 1076 +++++++++++----------- test/util/execute/discovery.go | 12 + test/util/namespaces/namespaces.go | 48 +- 3 files changed, 590 insertions(+), 546 deletions(-) create mode 100644 test/util/execute/discovery.go diff --git a/test/conformance/tests/sriov_operator.go b/test/conformance/tests/sriov_operator.go index fb6d70078..a0417afca 100644 --- a/test/conformance/tests/sriov_operator.go +++ b/test/conformance/tests/sriov_operator.go @@ -62,279 +62,70 @@ var _ = Describe("[sriov] operator", func() { Expect(err).ToNot(HaveOccurred()) }) - BeforeEach(func() { - err := namespaces.Clean(operatorNamespace, namespaces.Test, clients) - Expect(err).ToNot(HaveOccurred()) - waitForSRIOVStable() - }) - - Describe("Configuration", func() { - - Context("SR-IOV network config daemon can be set by nodeselector", func() { - // 26186 - It("Should schedule the config daemon on selected nodes", func() { - - By("Checking that a daemon is scheduled on each worker node") - Eventually(func() bool { - return daemonsScheduledOnNodes("node-role.kubernetes.io/worker=") - }, 3*time.Minute, 1*time.Second).Should(Equal(true)) - - By("Labelling one worker node with the label needed for the daemon") - allNodes, err := clients.Nodes().List(metav1.ListOptions{ - LabelSelector: "node-role.kubernetes.io/worker", - }) - Expect(len(allNodes.Items)).To(BeNumerically(">", 0), "There must be at least one worker") - candidate := allNodes.Items[0] - candidate.Labels["sriovenabled"] = "true" - _, err = clients.Nodes().Update(&candidate) - Expect(err).ToNot(HaveOccurred()) - - By("Setting the node selector for each daemon") - cfg := sriovv1.SriovOperatorConfig{} - err = clients.Get(context.TODO(), runtimeclient.ObjectKey{ - Name: "default", - Namespace: operatorNamespace, - }, &cfg) - Expect(err).ToNot(HaveOccurred()) - cfg.Spec.ConfigDaemonNodeSelector = map[string]string{ - "sriovenabled": "true", - } - err = clients.Update(context.TODO(), &cfg) - Expect(err).ToNot(HaveOccurred()) - - By("Checking that a daemon is scheduled only on selected node") - Eventually(func() bool { - return !daemonsScheduledOnNodes("sriovenabled!=true") && - daemonsScheduledOnNodes("sriovenabled=true") - }, 1*time.Minute, 1*time.Second).Should(Equal(true)) - - By("Restoring the node selector for daemons") - err = clients.Get(context.TODO(), runtimeclient.ObjectKey{ - Name: "default", - Namespace: operatorNamespace, - }, &cfg) - Expect(err).ToNot(HaveOccurred()) - cfg.Spec.ConfigDaemonNodeSelector = map[string]string{} - err = clients.Update(context.TODO(), &cfg) - Expect(err).ToNot(HaveOccurred()) - - By("Checking that a daemon is scheduled on each worker node") - Eventually(func() bool { - return daemonsScheduledOnNodes("node-role.kubernetes.io/worker") - }, 1*time.Minute, 1*time.Second).Should(Equal(true)) - - }) - }) - - Context("PF Partitioning", func() { - // 27633 - It("Should be possible to partition the pf's vfs", func() { - node := sriovInfos.Nodes[0] - intf, err := sriovInfos.FindOneSriovDevice(node) - Expect(err).ToNot(HaveOccurred()) - - firstConfig := &sriovv1.SriovNetworkNodePolicy{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-policy", - Namespace: operatorNamespace, - }, - - Spec: sriovv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{ - "kubernetes.io/hostname": node, - }, - NumVfs: 5, - ResourceName: "testresource", - Priority: 99, - NicSelector: sriovv1.SriovNetworkNicSelector{ - PfNames: []string{intf.Name + "#2-4"}, - }, - DeviceType: "netdevice", - }, - } - - err = clients.Create(context.Background(), firstConfig) + Describe("Generic SriovNetworkNodePolicy", func() { + numVfs := 5 + resourceName := "testresource" + var node string + + execute.BeforeAll(func() { + node = sriovInfos.Nodes[0] + + // For the context of tests is better to use a Mellanox card + // as they support all the virtual function flags + // if we don't find a Mellanox card we fall back to any sriov + // capability interface and skip the rate limit test. + intf, err := sriovInfos.FindOneMellanoxSriovDevice(node) + if err != nil { + intf, err = sriovInfos.FindOneSriovDevice(node) Expect(err).ToNot(HaveOccurred()) + } - Eventually(func() sriovv1.Interfaces { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(node, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) - return nodeState.Spec.Interfaces - }, 1*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( - IgnoreExtras, - Fields{ - "Name": Equal(intf.Name), - "NumVfs": Equal(5), - "VfGroups": ContainElement( - MatchFields( - IgnoreExtras, - Fields{ - "ResourceName": Equal("testresource"), - "DeviceType": Equal("netdevice"), - "VfRange": Equal("2-4"), - })), - }))) - - waitForSRIOVStable() + config := &sriovv1.SriovNetworkNodePolicy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-policy", + Namespace: operatorNamespace, + }, - Eventually(func() int64 { - testedNode, err := clients.Nodes().Get(node, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) - resNum, _ := testedNode.Status.Allocatable["openshift.io/testresource"] - capacity, _ := resNum.AsInt64() - return capacity - }, 3*time.Minute, time.Second).Should(Equal(int64(3))) - - secondConfig := &sriovv1.SriovNetworkNodePolicy{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-policy", - Namespace: operatorNamespace, + Spec: sriovv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{ + "kubernetes.io/hostname": node, }, - - Spec: sriovv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{ - "kubernetes.io/hostname": node, - }, - NumVfs: 5, - ResourceName: "testresource1", - Priority: 99, - NicSelector: sriovv1.SriovNetworkNicSelector{ - PfNames: []string{intf.Name + "#0-1"}, - }, - DeviceType: "vfio-pci", - }, - } - - err = clients.Create(context.Background(), secondConfig) - Expect(err).ToNot(HaveOccurred()) - - Eventually(func() sriovv1.Interfaces { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(node, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) - return nodeState.Spec.Interfaces - }, 3*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( - IgnoreExtras, - Fields{ - "Name": Equal(intf.Name), - "NumVfs": Equal(5), - "VfGroups": SatisfyAll( - ContainElement( - MatchFields( - IgnoreExtras, - Fields{ - "ResourceName": Equal("testresource"), - "DeviceType": Equal("netdevice"), - "VfRange": Equal("2-4"), - })), - ContainElement( - MatchFields( - IgnoreExtras, - Fields{ - "ResourceName": Equal("testresource1"), - "DeviceType": Equal("vfio-pci"), - "VfRange": Equal("0-1"), - })), - ), - }, - ))) - - // The node may reset here so we put a larger timeout here - Eventually(func() bool { - res, err := cluster.SriovStable(operatorNamespace, clients) - Expect(err).ToNot(HaveOccurred()) - return res - }, 15*time.Minute, 5*time.Second).Should(BeTrue()) - - Eventually(func() map[string]int64 { - testedNode, err := clients.Nodes().Get(node, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) - resNum, _ := testedNode.Status.Allocatable["openshift.io/testresource"] - capacity, _ := resNum.AsInt64() - res := make(map[string]int64) - res["openshift.io/testresource"] = capacity - resNum, _ = testedNode.Status.Allocatable["openshift.io/testresource1"] - capacity, _ = resNum.AsInt64() - res["openshift.io/testresource1"] = capacity - return res - }, 2*time.Minute, time.Second).Should(Equal(map[string]int64{ - "openshift.io/testresource": int64(3), - "openshift.io/testresource1": int64(2), - })) - }) - - // 27630 - /*It("Should not be possible to have overlapping pf ranges", func() { - // Skipping this test as blocking the override will - // be implemented in 4.5, as per bz #1798880 - Skip("Overlapping is still not blocked") - node := sriovInfos.Nodes[0] - intf, err := sriovInfos.FindOneSriovDevice(node) - Expect(err).ToNot(HaveOccurred()) - - firstConfig := &sriovv1.SriovNetworkNodePolicy{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-policy", - Namespace: operatorNamespace, + NumVfs: numVfs, + ResourceName: resourceName, + Priority: 99, + NicSelector: sriovv1.SriovNetworkNicSelector{ + PfNames: []string{intf.Name}, }, + DeviceType: "netdevice", + }, + } - Spec: sriovv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{ - "kubernetes.io/hostname": node, - }, - NumVfs: 5, - ResourceName: "testresource", - Priority: 99, - NicSelector: sriovv1.SriovNetworkNicSelector{ - PfNames: []string{intf.Name + "#1-4"}, - }, - DeviceType: "netdevice", - }, - } + err = clients.Create(context.Background(), config) + Expect(err).ToNot(HaveOccurred()) - err = clients.Create(context.Background(), firstConfig) + Eventually(func() sriovv1.Interfaces { + nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(node, metav1.GetOptions{}) Expect(err).ToNot(HaveOccurred()) - - Eventually(func() sriovv1.Interfaces { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(node, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) - return nodeState.Spec.Interfaces - }, 1*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( - IgnoreExtras, - Fields{ - "Name": Equal(intf.Name), - "NumVfs": Equal(5), - "VfGroups": ContainElement(sriovv1.VfGroup{ResourceName: "testresource", DeviceType: "netdevice", VfRange: "1-4", PolicyName: firstConfig.Name}), - }))) - - secondConfig := &sriovv1.SriovNetworkNodePolicy{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-policy", - Namespace: operatorNamespace, - }, - - Spec: sriovv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{ - "kubernetes.io/hostname": node, - }, - NumVfs: 5, - ResourceName: "testresource1", - Priority: 99, - NicSelector: sriovv1.SriovNetworkNicSelector{ - PfNames: []string{intf.Name + "#0-2"}, - }, - DeviceType: "vfio-pci", - }, - } - - err = clients.Create(context.Background(), secondConfig) - Expect(err).To(HaveOccurred()) - })*/ + return nodeState.Spec.Interfaces + }, 1*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( + IgnoreExtras, + Fields{ + "Name": Equal(intf.Name), + "NumVfs": Equal(numVfs), + }))) + + waitForSRIOVStable() }) + BeforeEach(func() { + err := namespaces.CleanPods(namespaces.Test, clients) + Expect(err).ToNot(HaveOccurred()) + err = namespaces.CleanNetworks(operatorNamespace, clients) + Expect(err).ToNot(HaveOccurred()) + }) Context("VF flags", func() { hostNetPod := &corev1.Pod{} // Initialized in BeforeEach intf := &sriovv1.InterfaceExt{} - numVfs := 5 validationFunction := func(networks []string, containsFunc func(line string) bool) { podObj := pod.DefineWithNetworks(networks) @@ -391,55 +182,7 @@ var _ = Describe("[sriov] operator", func() { } BeforeEach(func() { - var err error node := sriovInfos.Nodes[0] - - // For the context of tests is better to use a Mellanox card - // as they support all the virtual function flags - // if we don't find a Mellanox card we fall back to any sriov - // capability interface and skip the rate limit test. - intf, err = sriovInfos.FindOneMellanoxSriovDevice(node) - if err != nil { - intf, err = sriovInfos.FindOneSriovDevice(node) - Expect(err).ToNot(HaveOccurred()) - } - - config := &sriovv1.SriovNetworkNodePolicy{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-policy", - Namespace: operatorNamespace, - }, - - Spec: sriovv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{ - "kubernetes.io/hostname": node, - }, - NumVfs: numVfs, - ResourceName: "testresource", - Priority: 99, - NicSelector: sriovv1.SriovNetworkNicSelector{ - PfNames: []string{intf.Name}, - }, - DeviceType: "netdevice", - }, - } - - err = clients.Create(context.Background(), config) - Expect(err).ToNot(HaveOccurred()) - - Eventually(func() sriovv1.Interfaces { - nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(node, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) - return nodeState.Spec.Interfaces - }, 1*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( - IgnoreExtras, - Fields{ - "Name": Equal(intf.Name), - "NumVfs": Equal(numVfs), - }))) - - waitForSRIOVStable() - Eventually(func() int64 { testedNode, err := clients.Nodes().Get(node, metav1.GetOptions{}) Expect(err).ToNot(HaveOccurred()) @@ -449,7 +192,7 @@ var _ = Describe("[sriov] operator", func() { }, 3*time.Minute, time.Second).Should(Equal(int64(numVfs))) hostNetPod = pod.DefineWithHostNetwork(node) - err = clients.Create(context.Background(), hostNetPod) + err := clients.Create(context.Background(), hostNetPod) Expect(err).ToNot(HaveOccurred()) Eventually(func() corev1.PodPhase { hostNetPod, err = clients.Pods(namespaces.Test).Get(hostNetPod.Name, metav1.GetOptions{}) @@ -463,13 +206,13 @@ var _ = Describe("[sriov] operator", func() { sriovNetwork := &sriovv1.SriovNetwork{ ObjectMeta: metav1.ObjectMeta{Name: "test-spoofnetwork", Namespace: operatorNamespace}, Spec: sriovv1.SriovNetworkSpec{ - ResourceName: "testresource", + ResourceName: resourceName, IPAM: `{"type":"host-local", - "subnet":"10.10.10.0/24", - "rangeStart":"10.10.10.171", - "rangeEnd":"10.10.10.181", - "routes":[{"dst":"0.0.0.0/0"}], - "gateway":"10.10.10.1"}`, + "subnet":"10.10.10.0/24", + "rangeStart":"10.10.10.171", + "rangeEnd":"10.10.10.181", + "routes":[{"dst":"0.0.0.0/0"}], + "gateway":"10.10.10.1"}`, NetworkNamespace: namespaces.Test, }} @@ -508,13 +251,13 @@ var _ = Describe("[sriov] operator", func() { sriovNetwork := &sriovv1.SriovNetwork{ ObjectMeta: metav1.ObjectMeta{Name: "test-trustnetwork", Namespace: operatorNamespace}, Spec: sriovv1.SriovNetworkSpec{ - ResourceName: "testresource", + ResourceName: resourceName, IPAM: `{"type":"host-local", - "subnet":"10.10.10.0/24", - "rangeStart":"10.10.10.171", - "rangeEnd":"10.10.10.181", - "routes":[{"dst":"0.0.0.0/0"}], - "gateway":"10.10.10.1"}`, + "subnet":"10.10.10.0/24", + "rangeStart":"10.10.10.171", + "rangeEnd":"10.10.10.181", + "routes":[{"dst":"0.0.0.0/0"}], + "gateway":"10.10.10.1"}`, NetworkNamespace: namespaces.Test, }} @@ -552,13 +295,13 @@ var _ = Describe("[sriov] operator", func() { sriovNetwork := &sriovv1.SriovNetwork{ ObjectMeta: metav1.ObjectMeta{Name: "test-statenetwork", Namespace: operatorNamespace}, Spec: sriovv1.SriovNetworkSpec{ - ResourceName: "testresource", + ResourceName: resourceName, IPAM: `{"type":"host-local", - "subnet":"10.10.10.0/24", - "rangeStart":"10.10.10.171", - "rangeEnd":"10.10.10.181", - "routes":[{"dst":"0.0.0.0/0"}], - "gateway":"10.10.10.1"}`, + "subnet":"10.10.10.0/24", + "rangeStart":"10.10.10.171", + "rangeEnd":"10.10.10.181", + "routes":[{"dst":"0.0.0.0/0"}], + "gateway":"10.10.10.1"}`, NetworkNamespace: namespaces.Test, }} @@ -625,13 +368,13 @@ var _ = Describe("[sriov] operator", func() { var minTxRate = 40 sriovNetwork := &sriovv1.SriovNetwork{ObjectMeta: metav1.ObjectMeta{Name: "test-ratenetwork", Namespace: operatorNamespace}, Spec: sriovv1.SriovNetworkSpec{ - ResourceName: "testresource", + ResourceName: resourceName, IPAM: `{"type":"host-local", - "subnet":"10.10.10.0/24", - "rangeStart":"10.10.10.171", - "rangeEnd":"10.10.10.181", - "routes":[{"dst":"0.0.0.0/0"}], - "gateway":"10.10.10.1"}`, + "subnet":"10.10.10.0/24", + "rangeStart":"10.10.10.171", + "rangeEnd":"10.10.10.181", + "routes":[{"dst":"0.0.0.0/0"}], + "gateway":"10.10.10.1"}`, MaxTxRate: &maxTxRate, MinTxRate: &minTxRate, NetworkNamespace: namespaces.Test, @@ -661,13 +404,13 @@ var _ = Describe("[sriov] operator", func() { It("Should configure the requested vlan and Qos vlan flags under the vf", func() { sriovNetwork := &sriovv1.SriovNetwork{ObjectMeta: metav1.ObjectMeta{Name: "test-quosnetwork", Namespace: operatorNamespace}, Spec: sriovv1.SriovNetworkSpec{ - ResourceName: "testresource", + ResourceName: resourceName, IPAM: `{"type":"host-local", - "subnet":"10.10.10.0/24", - "rangeStart":"10.10.10.171", - "rangeEnd":"10.10.10.181", - "routes":[{"dst":"0.0.0.0/0"}], - "gateway":"10.10.10.1"}`, + "subnet":"10.10.10.0/24", + "rangeStart":"10.10.10.171", + "rangeEnd":"10.10.10.181", + "routes":[{"dst":"0.0.0.0/0"}], + "gateway":"10.10.10.1"}`, Vlan: 1, VlanQoS: 2, NetworkNamespace: namespaces.Test, @@ -692,258 +435,525 @@ var _ = Describe("[sriov] operator", func() { }) }) }) - Context("Resource Injector", func() { - // 25815 - It("Should inject downward api volume", func() { - node := sriovInfos.Nodes[0] - intf, err := sriovInfos.FindOneSriovDevice(node) - Expect(err).ToNot(HaveOccurred()) - - nodePolicy := &sriovv1.SriovNetworkNodePolicy{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-apivolumepolicy", - Namespace: operatorNamespace, - }, - Spec: sriovv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{ - "kubernetes.io/hostname": node, - }, - NumVfs: 5, - ResourceName: "apivolresource", - Priority: 99, - NicSelector: sriovv1.SriovNetworkNicSelector{ - PfNames: []string{intf.Name}, - }, - DeviceType: "netdevice", - }, - } - - err = clients.Create(context.Background(), nodePolicy) + Context("Multiple sriov device and attachment", func() { + // 25834 + It("Should configure multiple network attachments", func() { + sriovNetworkName := "test-sriovnetwork" + sriovDevice, err := sriovInfos.FindOneSriovDevice(node) + ipam := `{"type": "host-local","ranges": [[{"subnet": "1.1.1.0/24"}]],"dataDir": "/run/my-orchestrator/container-ipam-state"}` + err = network.CreateSriovNetwork(clients, sriovDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam) Expect(err).ToNot(HaveOccurred()) + Eventually(func() error { + netAttDef := &netattdefv1.NetworkAttachmentDefinition{} + return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: namespaces.Test}, netAttDef) + }, 10*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) - waitForSRIOVStable() + pod := createTestPod(node, []string{sriovNetworkName, sriovNetworkName}) + nics, err := network.GetNicsByPrefix(pod, "net") + Expect(err).ToNot(HaveOccurred()) + Expect(len(nics)).To(Equal(2), "No sriov network interfaces found.") + }) + }) - Eventually(func() int64 { - testedNode, err := clients.Nodes().Get(node, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) - resNum, _ := testedNode.Status.Allocatable["openshift.io/apivolresource"] - capacity, _ := resNum.AsInt64() - return capacity - }, 3*time.Minute, time.Second).Should(Equal(int64(5))) + Context("IPv6 configured secondary interfaces on pods", func() { + // 25874 + It("should be able to ping each other", func() { + ipv6NetworkName := "test-ipv6network" + testNode := sriovInfos.Nodes[0] + sriovDevice, err := sriovInfos.FindOneSriovDevice(node) - sriovNetwork := &sriovv1.SriovNetwork{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-apivolnetwork", - Namespace: operatorNamespace, - }, - Spec: sriovv1.SriovNetworkSpec{ - ResourceName: "apivolresource", - IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181","routes":[{"dst":"0.0.0.0/0"}],"gateway":"10.10.10.1"}`, - NetworkNamespace: namespaces.Test, - }} - err = clients.Create(context.Background(), sriovNetwork) + ipam := `{"type": "host-local","ranges": [[{"subnet": "3ffe:ffff:0:01ff::/64"}]],"dataDir": "/run/my-orchestrator/container-ipam-state"}` + err = network.CreateSriovNetwork(clients, sriovDevice, ipv6NetworkName, namespaces.Test, operatorNamespace, resourceName, ipam) Expect(err).ToNot(HaveOccurred()) - Eventually(func() error { netAttDef := &netattdefv1.NetworkAttachmentDefinition{} - return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: "test-apivolnetwork", Namespace: namespaces.Test}, netAttDef) + return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: ipv6NetworkName, Namespace: namespaces.Test}, netAttDef) }, 10*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) - podDefinition := pod.DefineWithNetworks([]string{sriovNetwork.Name}) - created, err := clients.Pods(namespaces.Test).Create(podDefinition) + pod := createTestPod(testNode, []string{ipv6NetworkName}) + ips, err := network.GetSriovNicIPs(pod, "net1") Expect(err).ToNot(HaveOccurred()) + Expect(ips).NotTo(BeNil(), "No sriov network interface found.") + Expect(len(ips)).Should(Equal(1)) + for _, ip := range ips { + pingPod(ip, testNode, ipv6NetworkName) + } + }) + }) + }) - var runningPod *corev1.Pod - Eventually(func() corev1.PodPhase { - runningPod, err = clients.Pods(namespaces.Test).Get(created.Name, metav1.GetOptions{}) + Describe("Custom SriovNetworkNodePolicy", func() { + + BeforeEach(func() { + err := namespaces.Clean(operatorNamespace, namespaces.Test, clients) + Expect(err).ToNot(HaveOccurred()) + waitForSRIOVStable() + }) + + Describe("Configuration", func() { + + Context("SR-IOV network config daemon can be set by nodeselector", func() { + // 26186 + It("Should schedule the config daemon on selected nodes", func() { + + By("Checking that a daemon is scheduled on each worker node") + Eventually(func() bool { + return daemonsScheduledOnNodes("node-role.kubernetes.io/worker=") + }, 3*time.Minute, 1*time.Second).Should(Equal(true)) + + By("Labelling one worker node with the label needed for the daemon") + allNodes, err := clients.Nodes().List(metav1.ListOptions{ + LabelSelector: "node-role.kubernetes.io/worker", + }) + Expect(len(allNodes.Items)).To(BeNumerically(">", 0), "There must be at least one worker") + candidate := allNodes.Items[0] + candidate.Labels["sriovenabled"] = "true" + _, err = clients.Nodes().Update(&candidate) Expect(err).ToNot(HaveOccurred()) - return runningPod.Status.Phase - }, 3*time.Minute, time.Second).Should(Equal(corev1.PodRunning)) - var downwardVolume *corev1.Volume - for _, v := range runningPod.Spec.Volumes { - if v.Name == "podnetinfo" { - downwardVolume = v.DeepCopy() - break + By("Setting the node selector for each daemon") + cfg := sriovv1.SriovOperatorConfig{} + err = clients.Get(context.TODO(), runtimeclient.ObjectKey{ + Name: "default", + Namespace: operatorNamespace, + }, &cfg) + Expect(err).ToNot(HaveOccurred()) + cfg.Spec.ConfigDaemonNodeSelector = map[string]string{ + "sriovenabled": "true", } - } + err = clients.Update(context.TODO(), &cfg) + Expect(err).ToNot(HaveOccurred()) + + By("Checking that a daemon is scheduled only on selected node") + Eventually(func() bool { + return !daemonsScheduledOnNodes("sriovenabled!=true") && + daemonsScheduledOnNodes("sriovenabled=true") + }, 1*time.Minute, 1*time.Second).Should(Equal(true)) + + By("Restoring the node selector for daemons") + err = clients.Get(context.TODO(), runtimeclient.ObjectKey{ + Name: "default", + Namespace: operatorNamespace, + }, &cfg) + Expect(err).ToNot(HaveOccurred()) + cfg.Spec.ConfigDaemonNodeSelector = map[string]string{} + err = clients.Update(context.TODO(), &cfg) + Expect(err).ToNot(HaveOccurred()) + + By("Checking that a daemon is scheduled on each worker node") + Eventually(func() bool { + return daemonsScheduledOnNodes("node-role.kubernetes.io/worker") + }, 1*time.Minute, 1*time.Second).Should(Equal(true)) - Expect(downwardVolume).ToNot(BeNil(), "Downward volume not found") - Expect(downwardVolume.DownwardAPI).ToNot(BeNil(), "Downward api not found in volume") - Expect(downwardVolume.DownwardAPI.Items).To(SatisfyAll( - ContainElement(corev1.DownwardAPIVolumeFile{ - Path: "labels", - FieldRef: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "metadata.labels", + }) + }) + + Context("PF Partitioning", func() { + // 27633 + It("Should be possible to partition the pf's vfs", func() { + node := sriovInfos.Nodes[0] + intf, err := sriovInfos.FindOneSriovDevice(node) + Expect(err).ToNot(HaveOccurred()) + + firstConfig := &sriovv1.SriovNetworkNodePolicy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-policy", + Namespace: operatorNamespace, }, - }), ContainElement(corev1.DownwardAPIVolumeFile{ - Path: "annotations", - FieldRef: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "metadata.annotations", + + Spec: sriovv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{ + "kubernetes.io/hostname": node, + }, + NumVfs: 5, + ResourceName: "testresource", + Priority: 99, + NicSelector: sriovv1.SriovNetworkNicSelector{ + PfNames: []string{intf.Name + "#2-4"}, + }, + DeviceType: "netdevice", }, - }))) - }) - Context("Multiple sriov device and attachment", func() { - // 25834 - It("Should configure multiple network attachments", func() { - resourceName := "sriovnic" - sriovNetworkName := "test-sriovnetwork" - testNode := sriovInfos.Nodes[0] + } - sriovDevice, err := sriovInfos.FindOneSriovDevice(testNode) + err = clients.Create(context.Background(), firstConfig) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() sriovv1.Interfaces { + nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(node, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + return nodeState.Spec.Interfaces + }, 1*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( + IgnoreExtras, + Fields{ + "Name": Equal(intf.Name), + "NumVfs": Equal(5), + "VfGroups": ContainElement( + MatchFields( + IgnoreExtras, + Fields{ + "ResourceName": Equal("testresource"), + "DeviceType": Equal("netdevice"), + "VfRange": Equal("2-4"), + })), + }))) + + waitForSRIOVStable() + + Eventually(func() int64 { + testedNode, err := clients.Nodes().Get(node, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + resNum, _ := testedNode.Status.Allocatable["openshift.io/testresource"] + capacity, _ := resNum.AsInt64() + return capacity + }, 3*time.Minute, time.Second).Should(Equal(int64(3))) + + secondConfig := &sriovv1.SriovNetworkNodePolicy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-policy", + Namespace: operatorNamespace, + }, + + Spec: sriovv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{ + "kubernetes.io/hostname": node, + }, + NumVfs: 5, + ResourceName: "testresource1", + Priority: 99, + NicSelector: sriovv1.SriovNetworkNicSelector{ + PfNames: []string{intf.Name + "#0-1"}, + }, + DeviceType: "vfio-pci", + }, + } + + err = clients.Create(context.Background(), secondConfig) Expect(err).ToNot(HaveOccurred()) - createSriovPolicy(sriovDevice.Name, testNode, 5, resourceName) - ipam := `{"type": "host-local","ranges": [[{"subnet": "1.1.1.0/24"}]],"dataDir": "/run/my-orchestrator/container-ipam-state"}` - err = network.CreateSriovNetwork(clients, sriovDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam) + Eventually(func() sriovv1.Interfaces { + nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(node, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + return nodeState.Spec.Interfaces + }, 3*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( + IgnoreExtras, + Fields{ + "Name": Equal(intf.Name), + "NumVfs": Equal(5), + "VfGroups": SatisfyAll( + ContainElement( + MatchFields( + IgnoreExtras, + Fields{ + "ResourceName": Equal("testresource"), + "DeviceType": Equal("netdevice"), + "VfRange": Equal("2-4"), + })), + ContainElement( + MatchFields( + IgnoreExtras, + Fields{ + "ResourceName": Equal("testresource1"), + "DeviceType": Equal("vfio-pci"), + "VfRange": Equal("0-1"), + })), + ), + }, + ))) + + // The node may reset here so we put a larger timeout here + Eventually(func() bool { + res, err := cluster.SriovStable(operatorNamespace, clients) + Expect(err).ToNot(HaveOccurred()) + return res + }, 15*time.Minute, 5*time.Second).Should(BeTrue()) + + Eventually(func() map[string]int64 { + testedNode, err := clients.Nodes().Get(node, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + resNum, _ := testedNode.Status.Allocatable["openshift.io/testresource"] + capacity, _ := resNum.AsInt64() + res := make(map[string]int64) + res["openshift.io/testresource"] = capacity + resNum, _ = testedNode.Status.Allocatable["openshift.io/testresource1"] + capacity, _ = resNum.AsInt64() + res["openshift.io/testresource1"] = capacity + return res + }, 2*time.Minute, time.Second).Should(Equal(map[string]int64{ + "openshift.io/testresource": int64(3), + "openshift.io/testresource1": int64(2), + })) + }) + + // 27630 + /*It("Should not be possible to have overlapping pf ranges", func() { + // Skipping this test as blocking the override will + // be implemented in 4.5, as per bz #1798880 + Skip("Overlapping is still not blocked") + node := sriovInfos.Nodes[0] + intf, err := sriovInfos.FindOneSriovDevice(node) Expect(err).ToNot(HaveOccurred()) - Eventually(func() error { - netAttDef := &netattdefv1.NetworkAttachmentDefinition{} - return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: namespaces.Test}, netAttDef) - }, 10*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) - pod := createTestPod(testNode, []string{sriovNetworkName, sriovNetworkName}) - nics, err := network.GetNicsByPrefix(pod, "net") + firstConfig := &sriovv1.SriovNetworkNodePolicy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-policy", + Namespace: operatorNamespace, + }, + + Spec: sriovv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{ + "kubernetes.io/hostname": node, + }, + NumVfs: 5, + ResourceName: "testresource", + Priority: 99, + NicSelector: sriovv1.SriovNetworkNicSelector{ + PfNames: []string{intf.Name + "#1-4"}, + }, + DeviceType: "netdevice", + }, + } + + err = clients.Create(context.Background(), firstConfig) Expect(err).ToNot(HaveOccurred()) - Expect(len(nics)).To(Equal(2), "No sriov network interfaces found.") - }) + Eventually(func() sriovv1.Interfaces { + nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(node, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + return nodeState.Spec.Interfaces + }, 1*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( + IgnoreExtras, + Fields{ + "Name": Equal(intf.Name), + "NumVfs": Equal(5), + "VfGroups": ContainElement(sriovv1.VfGroup{ResourceName: "testresource", DeviceType: "netdevice", VfRange: "1-4", PolicyName: firstConfig.Name}), + }))) + + secondConfig := &sriovv1.SriovNetworkNodePolicy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-policy", + Namespace: operatorNamespace, + }, + + Spec: sriovv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{ + "kubernetes.io/hostname": node, + }, + NumVfs: 5, + ResourceName: "testresource1", + Priority: 99, + NicSelector: sriovv1.SriovNetworkNicSelector{ + PfNames: []string{intf.Name + "#0-2"}, + }, + DeviceType: "vfio-pci", + }, + } + + err = clients.Create(context.Background(), secondConfig) + Expect(err).To(HaveOccurred()) + })*/ }) - Context("IPv6 configured secondary interfaces on pods", func() { - // 25874 - It("should be able to ping each other", func() { - resourceName := "sriovnic" - ipv6NetworkName := "test-ipv6network" - testNode := sriovInfos.Nodes[0] - sriovDevice, err := sriovInfos.FindOneSriovDevice(testNode) + + Context("Resource Injector", func() { + // 25815 + It("Should inject downward api volume", func() { + node := sriovInfos.Nodes[0] + intf, err := sriovInfos.FindOneSriovDevice(node) + Expect(err).ToNot(HaveOccurred()) + + nodePolicy := &sriovv1.SriovNetworkNodePolicy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-apivolumepolicy", + Namespace: operatorNamespace, + }, + + Spec: sriovv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{ + "kubernetes.io/hostname": node, + }, + NumVfs: 5, + ResourceName: "apivolresource", + Priority: 99, + NicSelector: sriovv1.SriovNetworkNicSelector{ + PfNames: []string{intf.Name}, + }, + DeviceType: "netdevice", + }, + } + + err = clients.Create(context.Background(), nodePolicy) Expect(err).ToNot(HaveOccurred()) - createSriovPolicy(sriovDevice.Name, testNode, 5, resourceName) - ipam := `{"type": "host-local","ranges": [[{"subnet": "3ffe:ffff:0:01ff::/64"}]],"dataDir": "/run/my-orchestrator/container-ipam-state"}` - err = network.CreateSriovNetwork(clients, sriovDevice, ipv6NetworkName, namespaces.Test, operatorNamespace, resourceName, ipam) + waitForSRIOVStable() + + Eventually(func() int64 { + testedNode, err := clients.Nodes().Get(node, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + resNum, _ := testedNode.Status.Allocatable["openshift.io/apivolresource"] + capacity, _ := resNum.AsInt64() + return capacity + }, 3*time.Minute, time.Second).Should(Equal(int64(5))) + + sriovNetwork := &sriovv1.SriovNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-apivolnetwork", + Namespace: operatorNamespace, + }, + Spec: sriovv1.SriovNetworkSpec{ + ResourceName: "apivolresource", + IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181","routes":[{"dst":"0.0.0.0/0"}],"gateway":"10.10.10.1"}`, + NetworkNamespace: namespaces.Test, + }} + err = clients.Create(context.Background(), sriovNetwork) Expect(err).ToNot(HaveOccurred()) + Eventually(func() error { netAttDef := &netattdefv1.NetworkAttachmentDefinition{} - return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: ipv6NetworkName, Namespace: namespaces.Test}, netAttDef) + return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: "test-apivolnetwork", Namespace: namespaces.Test}, netAttDef) }, 10*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) - pod := createTestPod(testNode, []string{ipv6NetworkName}) - ips, err := network.GetSriovNicIPs(pod, "net1") + podDefinition := pod.DefineWithNetworks([]string{sriovNetwork.Name}) + created, err := clients.Pods(namespaces.Test).Create(podDefinition) Expect(err).ToNot(HaveOccurred()) - Expect(ips).NotTo(BeNil(), "No sriov network interface found.") - Expect(len(ips)).Should(Equal(1)) - for _, ip := range ips { - pingPod(ip, testNode, ipv6NetworkName) + + var runningPod *corev1.Pod + Eventually(func() corev1.PodPhase { + runningPod, err = clients.Pods(namespaces.Test).Get(created.Name, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + return runningPod.Status.Phase + }, 3*time.Minute, time.Second).Should(Equal(corev1.PodRunning)) + + var downwardVolume *corev1.Volume + for _, v := range runningPod.Spec.Volumes { + if v.Name == "podnetinfo" { + downwardVolume = v.DeepCopy() + break + } } + + Expect(downwardVolume).ToNot(BeNil(), "Downward volume not found") + Expect(downwardVolume.DownwardAPI).ToNot(BeNil(), "Downward api not found in volume") + Expect(downwardVolume.DownwardAPI.Items).To(SatisfyAll( + ContainElement(corev1.DownwardAPIVolumeFile{ + Path: "labels", + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.labels", + }, + }), ContainElement(corev1.DownwardAPIVolumeFile{ + Path: "annotations", + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.annotations", + }, + }))) }) + }) - }) - Context("MTU", func() { - BeforeEach(func() { - node := sriovInfos.Nodes[0] - intf, err := sriovInfos.FindOneSriovDevice(node) - Expect(err).ToNot(HaveOccurred()) - mtuPolicy := &sriovv1.SriovNetworkNodePolicy{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-mtupolicy", - Namespace: operatorNamespace, - }, + Context("MTU", func() { + BeforeEach(func() { + node := sriovInfos.Nodes[0] + intf, err := sriovInfos.FindOneSriovDevice(node) + Expect(err).ToNot(HaveOccurred()) - Spec: sriovv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{ - "kubernetes.io/hostname": node, + mtuPolicy := &sriovv1.SriovNetworkNodePolicy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-mtupolicy", + Namespace: operatorNamespace, }, - Mtu: 9000, - NumVfs: 5, - ResourceName: "mturesource", - Priority: 99, - NicSelector: sriovv1.SriovNetworkNicSelector{ - PfNames: []string{intf.Name}, + + Spec: sriovv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{ + "kubernetes.io/hostname": node, + }, + Mtu: 9000, + NumVfs: 5, + ResourceName: "mturesource", + Priority: 99, + NicSelector: sriovv1.SriovNetworkNicSelector{ + PfNames: []string{intf.Name}, + }, + DeviceType: "netdevice", }, - DeviceType: "netdevice", - }, - } + } - err = clients.Create(context.Background(), mtuPolicy) - Expect(err).ToNot(HaveOccurred()) + err = clients.Create(context.Background(), mtuPolicy) + Expect(err).ToNot(HaveOccurred()) - waitForSRIOVStable() + waitForSRIOVStable() - sriovNetwork := &sriovv1.SriovNetwork{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-mtuvolnetwork", - Namespace: operatorNamespace, - }, - Spec: sriovv1.SriovNetworkSpec{ - ResourceName: "mturesource", - IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181","routes":[{"dst":"0.0.0.0/0"}],"gateway":"10.10.10.1"}`, - NetworkNamespace: namespaces.Test, - LinkState: "enable", - }} + sriovNetwork := &sriovv1.SriovNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-mtuvolnetwork", + Namespace: operatorNamespace, + }, + Spec: sriovv1.SriovNetworkSpec{ + ResourceName: "mturesource", + IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181","routes":[{"dst":"0.0.0.0/0"}],"gateway":"10.10.10.1"}`, + NetworkNamespace: namespaces.Test, + LinkState: "enable", + }} - // We need this to be able to run the connectivity checks on Mellanox cards - if intf.DeviceID == "1015" { - sriovNetwork.Spec.SpoofChk = "off" - } + // We need this to be able to run the connectivity checks on Mellanox cards + if intf.DeviceID == "1015" { + sriovNetwork.Spec.SpoofChk = "off" + } - err = clients.Create(context.Background(), sriovNetwork) + err = clients.Create(context.Background(), sriovNetwork) - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred()) - Eventually(func() error { - netAttDef := &netattdefv1.NetworkAttachmentDefinition{} - return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: "test-mtuvolnetwork", Namespace: namespaces.Test}, netAttDef) - }, 10*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + Eventually(func() error { + netAttDef := &netattdefv1.NetworkAttachmentDefinition{} + return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: "test-mtuvolnetwork", Namespace: namespaces.Test}, netAttDef) + }, 10*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) - }) + }) - // 27662 - It("Should support jumbo frames", func() { - podDefinition := pod.DefineWithNetworks([]string{"test-mtuvolnetwork"}) - firstPod, err := clients.Pods(namespaces.Test).Create(podDefinition) - Expect(err).ToNot(HaveOccurred()) + // 27662 + It("Should support jumbo frames", func() { + podDefinition := pod.DefineWithNetworks([]string{"test-mtuvolnetwork"}) + firstPod, err := clients.Pods(namespaces.Test).Create(podDefinition) + Expect(err).ToNot(HaveOccurred()) - Eventually(func() corev1.PodPhase { - firstPod, _ = clients.Pods(namespaces.Test).Get(firstPod.Name, metav1.GetOptions{}) - return firstPod.Status.Phase - }, 3*time.Minute, time.Second).Should(Equal(corev1.PodRunning)) + Eventually(func() corev1.PodPhase { + firstPod, _ = clients.Pods(namespaces.Test).Get(firstPod.Name, metav1.GetOptions{}) + return firstPod.Status.Phase + }, 3*time.Minute, time.Second).Should(Equal(corev1.PodRunning)) - var stdout, stderr string - Eventually(func() error { - stdout, stderr, err = pod.ExecCommand(clients, firstPod, "ip", "link", "show", "net1") - if stdout == "" { - return fmt.Errorf("empty response from pod exec") - } + var stdout, stderr string + Eventually(func() error { + stdout, stderr, err = pod.ExecCommand(clients, firstPod, "ip", "link", "show", "net1") + if stdout == "" { + return fmt.Errorf("empty response from pod exec") + } - if err != nil { - return fmt.Errorf("Failed to show net1") - } + if err != nil { + return fmt.Errorf("Failed to show net1") + } - return nil - }, 1*time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) - Expect(stdout).To(ContainSubstring("mtu 9000")) - firstPodIPs, err := network.GetSriovNicIPs(firstPod, "net1") - Expect(err).ToNot(HaveOccurred()) - Expect(len(firstPodIPs)).To(Equal(1)) + return nil + }, 1*time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) + Expect(stdout).To(ContainSubstring("mtu 9000")) + firstPodIPs, err := network.GetSriovNicIPs(firstPod, "net1") + Expect(err).ToNot(HaveOccurred()) + Expect(len(firstPodIPs)).To(Equal(1)) - podDefinition = pod.DefineWithNetworks([]string{"test-mtuvolnetwork"}) - secondPod, err := clients.Pods(namespaces.Test).Create(podDefinition) - Expect(err).ToNot(HaveOccurred()) + podDefinition = pod.DefineWithNetworks([]string{"test-mtuvolnetwork"}) + secondPod, err := clients.Pods(namespaces.Test).Create(podDefinition) + Expect(err).ToNot(HaveOccurred()) - Eventually(func() corev1.PodPhase { - secondPod, _ = clients.Pods(namespaces.Test).Get(secondPod.Name, metav1.GetOptions{}) - return secondPod.Status.Phase - }, 3*time.Minute, time.Second).Should(Equal(corev1.PodRunning)) + Eventually(func() corev1.PodPhase { + secondPod, _ = clients.Pods(namespaces.Test).Get(secondPod.Name, metav1.GetOptions{}) + return secondPod.Status.Phase + }, 3*time.Minute, time.Second).Should(Equal(corev1.PodRunning)) - stdout, stderr, err = pod.ExecCommand(clients, secondPod, - "ping", firstPodIPs[0], "-s", "8972", "-M", "do", "-c", "2") - Expect(err).ToNot(HaveOccurred(), "Failed to ping first pod", stderr) - Expect(stdout).To(ContainSubstring("2 packets transmitted, 2 received, 0% packet loss")) + stdout, stderr, err = pod.ExecCommand(clients, secondPod, + "ping", firstPodIPs[0], "-s", "8972", "-M", "do", "-c", "2") + Expect(err).ToNot(HaveOccurred(), "Failed to ping first pod", stderr) + Expect(stdout).To(ContainSubstring("2 packets transmitted, 2 received, 0% packet loss")) + }) }) }) diff --git a/test/util/execute/discovery.go b/test/util/execute/discovery.go new file mode 100644 index 000000000..e54bde3a7 --- /dev/null +++ b/test/util/execute/discovery.go @@ -0,0 +1,12 @@ +package execute + +import ( + "os" + "strconv" +) + +// DiscoveryModeEnabled indicates whether test discovery mode is enabled. +func DiscoveryModeEnabled() bool { + discoveryMode, _ := strconv.ParseBool(os.Getenv("DISCOVERY_MODE")) + return discoveryMode +} diff --git a/test/util/namespaces/namespaces.go b/test/util/namespaces/namespaces.go index b81a04eae..07289d154 100644 --- a/test/util/namespaces/namespaces.go +++ b/test/util/namespaces/namespaces.go @@ -55,29 +55,35 @@ func DeleteAndWait(cs *testclient.ClientSet, namespace string, timeout time.Dura return WaitForDeletion(cs, namespace, timeout) } -// Clean cleans all dangling objects from the given namespace. -func Clean(operatorNamespace, namespace string, cs *testclient.ClientSet) error { +func namespaceExists(namespace string, cs *testclient.ClientSet) bool { _, err := cs.Namespaces().Get(namespace, metav1.GetOptions{}) - if err != nil && k8serrors.IsNotFound(err) { + return err == nil || !k8serrors.IsNotFound(err) +} + +// CleanPods deletes all pods in namespace +func CleanPods(namespace string, cs *testclient.ClientSet) error { + if !namespaceExists(namespace, cs) { return nil } - - err = cs.Pods(namespace).DeleteCollection(&metav1.DeleteOptions{ + err := cs.Pods(namespace).DeleteCollection(&metav1.DeleteOptions{ GracePeriodSeconds: pointer.Int64Ptr(0), }, metav1.ListOptions{}) if err != nil { return fmt.Errorf("Failed to delete pods %v", err) } + return err +} +// CleanPolicies deletes all SriovNetworkNodePolicies in operatorNamespace +func CleanPolicies(operatorNamespace string, cs *testclient.ClientSet) error { policies := sriovv1.SriovNetworkNodePolicyList{} - err = cs.List(context.Background(), + err := cs.List(context.Background(), &policies, - runtimeclient.InNamespace(operatorNamespace)) - + runtimeclient.InNamespace(operatorNamespace), + ) if err != nil { return err } - for _, p := range policies.Items { if p.Name != "default" && strings.HasPrefix(p.Name, "test-") { err := cs.Delete(context.Background(), &p) @@ -86,16 +92,18 @@ func Clean(operatorNamespace, namespace string, cs *testclient.ClientSet) error } } } + return err +} +// CleanNetworks deletes all network in operatorNamespace +func CleanNetworks(operatorNamespace string, cs *testclient.ClientSet) error { networks := sriovv1.SriovNetworkList{} - err = cs.List(context.Background(), + err := cs.List(context.Background(), &networks, runtimeclient.InNamespace(operatorNamespace)) - if err != nil { return err } - for _, n := range networks.Items { if strings.HasPrefix(n.Name, "test-") { err := cs.Delete(context.Background(), &n) @@ -104,5 +112,19 @@ func Clean(operatorNamespace, namespace string, cs *testclient.ClientSet) error } } } - return nil + return err +} + +// Clean cleans all dangling objects from the given namespace. +func Clean(operatorNamespace, namespace string, cs *testclient.ClientSet) error { + err := CleanPods(namespace, cs) + if err != nil { + return err + } + err = CleanPolicies(operatorNamespace, cs) + if err != nil { + return err + } + err = CleanNetworks(operatorNamespace, cs) + return err }