From 0c4edb3577f6065851653f239435c183ffc53add Mon Sep 17 00:00:00 2001 From: Sebastian Sch Date: Mon, 16 Dec 2024 17:16:56 +0200 Subject: [PATCH] functest: add retry for rdma functest this is needed because after a reboot on a single node the operator webhook may not be ready Signed-off-by: Sebastian Sch --- test/conformance/tests/test_networkpool.go | 44 ++++++++++------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/test/conformance/tests/test_networkpool.go b/test/conformance/tests/test_networkpool.go index 8e6cc2775..1129dc2ca 100644 --- a/test/conformance/tests/test_networkpool.go +++ b/test/conformance/tests/test_networkpool.go @@ -74,15 +74,13 @@ var _ = Describe("[sriov] NetworkPool", Ordered, func() { Expect(nodeState.Status.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeExclusive)) By("Checking rdma mode and kernel args") - output, _, err := runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline | grep ib_core.netns_mode=0 | wc -l") + cmdlineOutput, _, err := runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline") + errDescription := fmt.Sprintf("kernel args are not right, printing current kernel args %s", cmdlineOutput) Expect(err).ToNot(HaveOccurred()) - Expect(strings.HasPrefix(output, "1")).To(BeTrue()) - - output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline | grep ib_core.netns_mode=1 | wc -l") - Expect(err).ToNot(HaveOccurred()) - Expect(strings.HasPrefix(output, "0")).To(BeTrue()) + Expect(cmdlineOutput).To(ContainSubstring("ib_core.netns_mode=0"), errDescription) + Expect(cmdlineOutput).ToNot(ContainSubstring("ib_core.netns_mode=1"), errDescription) - output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/etc/modprobe.d/sriov_network_operator_modules_config.conf | grep mode=0 | wc -l") + output, _, err := runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/etc/modprobe.d/sriov_network_operator_modules_config.conf | grep mode=0 | wc -l") Expect(err).ToNot(HaveOccurred()) Expect(strings.HasPrefix(output, "1")).To(BeTrue()) @@ -97,21 +95,22 @@ var _ = Describe("[sriov] NetworkPool", Ordered, func() { Expect(nodeState.Status.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeShared)) By("Checking rdma mode and kernel args") - output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline | grep ib_core.netns_mode=0 | wc -l") + cmdlineOutput, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline") + errDescription = fmt.Sprintf("kernel args are not right, printing current kernel args %s", cmdlineOutput) Expect(err).ToNot(HaveOccurred()) - Expect(strings.HasPrefix(output, "0")).To(BeTrue()) - - output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline | grep ib_core.netns_mode=1 | wc -l") - Expect(err).ToNot(HaveOccurred()) - Expect(strings.HasPrefix(output, "1")).To(BeTrue()) + Expect(cmdlineOutput).ToNot(ContainSubstring("ib_core.netns_mode=0"), errDescription) + Expect(cmdlineOutput).To(ContainSubstring("ib_core.netns_mode=1"), errDescription) output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/etc/modprobe.d/sriov_network_operator_modules_config.conf | grep mode=1 | wc -l") Expect(err).ToNot(HaveOccurred()) - Expect(strings.HasPrefix(output, "1")).To(BeTrue()) + Expect(strings.HasPrefix(output, "1")).To(BeTrue(), fmt.Sprintf("kernel args are not right, printing current kernel args %s", cmdlineOutput)) By("removing rdma mode configuration") - err = clients.Delete(context.Background(), networkPool) - Expect(err).ToNot(HaveOccurred()) + Eventually(func(g Gomega) { + err = clients.Delete(context.Background(), networkPool) + g.Expect(err).ToNot(HaveOccurred()) + }, 5*time.Minute, 5*time.Second).Should(Succeed()) + WaitForSRIOVStable() err = clients.Get(context.Background(), client.ObjectKey{Name: testNode, Namespace: operatorNamespace}, nodeState) @@ -120,17 +119,14 @@ var _ = Describe("[sriov] NetworkPool", Ordered, func() { Expect(nodeState.Status.System.RdmaMode).To(Equal(consts.RdmaSubsystemModeShared)) By("Checking rdma mode and kernel args") - output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline | grep ib_core.netns_mode=0 | wc -l") - Expect(err).ToNot(HaveOccurred()) - Expect(strings.HasPrefix(output, "0")).To(BeTrue()) - - output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline | grep ib_core.netns_mode=1 | wc -l") - Expect(err).ToNot(HaveOccurred()) - Expect(strings.HasPrefix(output, "0")).To(BeTrue()) + cmdlineOutput, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "cat /host/proc/cmdline") + errDescription = fmt.Sprintf("kernel args are not right, printing current kernel args %s", cmdlineOutput) + Expect(cmdlineOutput).ToNot(ContainSubstring("ib_core.netns_mode=0"), errDescription) + Expect(cmdlineOutput).ToNot(ContainSubstring("ib_core.netns_mode=1"), errDescription) output, _, err = runCommandOnConfigDaemon(testNode, "/bin/bash", "-c", "ls /host/etc/modprobe.d | grep sriov_network_operator_modules_config.conf | wc -l") Expect(err).ToNot(HaveOccurred()) - Expect(strings.HasPrefix(output, "0")).To(BeTrue()) + Expect(strings.HasPrefix(output, "0")).To(BeTrue(), fmt.Sprintf("kernel args are not right, printing current kernel args %s", cmdlineOutput)) }) })