Skip to content

Commit

Permalink
e2e: Fix Debug logging should be visible in multus pod flake
Browse files Browse the repository at this point in the history 
Once the SriovNetwork has been created, test routine must wait for NetAttachDefinition
to be created by the operator before spawing pods. Create a function `waitForNetAttachDef(...)`
and use it in similar situations.

The flake error this commit is supposed to fix is:
```
• [FAILED] [2.695 seconds]
[sriov] operator Generic SriovNetworkNodePolicy CNI Logging level [It] Debug logging should be visible in multus pod
/root/opr-k8s-1/data/sriov-network-operator/sriov-network-operator/test/conformance/tests/test_sriov_operator.go:1077

  [FAILED] Unexpected error:
      <*errors.StatusError | 0xc000fd8640>:
      admission webhook "network-resources-injector-config.k8s.io" denied the request: could not find network attachment definition 'sriov-conformance-testing/test-log-level-debug-no-file': could not get Network Attachment Definition sriov-conformance-testing/test-log-level-debug-no-file: the server could not find the requested resource
      {
          ErrStatus: {
              TypeMeta: {Kind: "", APIVersion: ""},
              ListMeta: {
                  SelfLink: "",
                  ResourceVersion: "",
                  Continue: "",
                  RemainingItemCount: nil,
              },
              Status: "Failure",
              Message: "admission webhook \"network-resources-injector-config.k8s.io\" denied the request: could not find network attachment definition 'sriov-conformance-testing/test-log-level-debug-no-file': could not get Network Attachment Definition sriov-conformance-testing/test-log-level-debug-no-file: the server could not find the requested resource",
              Reason: "",
              Details: nil,
              Code: 400,
          },
      }
  occurred
  In [It] at: /root/opr-k8s-1/data/sriov-network-operator/sriov-network-operator/test/conformance/tests/test_sriov_operator.go:2505 @ 02/01/24 12:44:25.702
```

Signed-off-by: Andrea Panattoni <[email protected]>
  • Loading branch information
@zeeke
zeeke committed Feb 2, 2024
1 parent 1a3019e commit 20c08f5
Showing 1 changed file with 19 additions and 40 deletions.
59 changes: 19 additions & 40 deletions test/conformance/tests/test_sriov_operator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -776,10 +776,7 @@ var _ = Describe("[sriov] operator", func() {
ipam := ipamIpv4
err := network.CreateSriovNetwork(clients, sriovDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam)
Expect(err).ToNot(HaveOccurred())
Eventually(func() error {
netAttDef := &netattdefv1.NetworkAttachmentDefinition{}
return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: namespaces.Test}, netAttDef)
}, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred())
waitForNetAttachDef(sriovNetworkName, namespaces.Test)

pod := createTestPod(node, []string{sriovNetworkName, sriovNetworkName})
nics, err := network.GetNicsByPrefix(pod, "net")
Expand All @@ -795,10 +792,7 @@ var _ = Describe("[sriov] operator", func() {
ipam := ipamIpv6
err := network.CreateSriovNetwork(clients, sriovDevice, ipv6NetworkName, namespaces.Test, operatorNamespace, resourceName, ipam)
Expect(err).ToNot(HaveOccurred())
Eventually(func() error {
netAttDef := &netattdefv1.NetworkAttachmentDefinition{}
return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: ipv6NetworkName, Namespace: namespaces.Test}, netAttDef)
}, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred())
waitForNetAttachDef(ipv6NetworkName, namespaces.Test)

pod := createTestPod(node, []string{ipv6NetworkName})
ips, err := network.GetSriovNicIPs(pod, "net1")
Expand Down Expand Up @@ -826,10 +820,7 @@ var _ = Describe("[sriov] operator", func() {
ipam := ipamIpv4
err = network.CreateSriovNetwork(clients, sriovDevice, sriovNetworkName, ns1, operatorNamespace, resourceName, ipam)
Expect(err).ToNot(HaveOccurred())
Eventually(func() error {
netAttDef := &netattdefv1.NetworkAttachmentDefinition{}
return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: ns1}, netAttDef)
}, (30+snoTimeoutMultiplier*90)*time.Second, 1*time.Second).ShouldNot(HaveOccurred())
waitForNetAttachDef(sriovNetworkName, ns1)

body, _ := json.Marshal([]patchBody{{
Op: "replace",
Expand All @@ -838,10 +829,8 @@ var _ = Describe("[sriov] operator", func() {
}})
clients.SriovnetworkV1Interface.RESTClient().Patch(types.JSONPatchType).Namespace(operatorNamespace).Resource("sriovnetworks").Name(sriovNetworkName).Body(body).Do(context.Background())

Eventually(func() error {
netAttDef := &netattdefv1.NetworkAttachmentDefinition{}
return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: ns2}, netAttDef)
}, (30+snoTimeoutMultiplier*90)*time.Second, 1*time.Second).ShouldNot(HaveOccurred())
waitForNetAttachDef(sriovNetworkName, ns2)

Consistently(func() error {
netAttDef := &netattdefv1.NetworkAttachmentDefinition{}
return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: ns1}, netAttDef)
Expand Down Expand Up @@ -891,10 +880,7 @@ var _ = Describe("[sriov] operator", func() {
ipam := ipamIpv4
err := network.CreateSriovNetwork(clients, sriovDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam)
Expect(err).ToNot(HaveOccurred())
Eventually(func() error {
netAttDef := &netattdefv1.NetworkAttachmentDefinition{}
return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: namespaces.Test}, netAttDef)
}, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred())
waitForNetAttachDef(sriovNetworkName, namespaces.Test)

macvlanNadName := "macvlan-nad"
macvlanNad := network.CreateMacvlanNetworkAttachmentDefinition(macvlanNadName, namespaces.Test)
Expand Down Expand Up @@ -934,10 +920,7 @@ var _ = Describe("[sriov] operator", func() {
}
err := network.CreateSriovNetwork(clients, sriovDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam, []network.SriovNetworkOptions{config}...)
Expect(err).ToNot(HaveOccurred())
Eventually(func() error {
netAttDef := &netattdefv1.NetworkAttachmentDefinition{}
return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: namespaces.Test}, netAttDef)
}, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred())
waitForNetAttachDef(sriovNetworkName, namespaces.Test)

testPod := createTestPod(node, []string{sriovNetworkName})
stdout, _, err := pod.ExecCommand(clients, testPod, "more", "/proc/sys/net/ipv4/conf/net1/accept_redirects")
Expand All @@ -964,10 +947,7 @@ var _ = Describe("[sriov] operator", func() {
ipam := ipamIpv6
err = network.CreateSriovNetwork(clients, sriovDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam)
Expect(err).ToNot(HaveOccurred())
Eventually(func() error {
netAttDef := &netattdefv1.NetworkAttachmentDefinition{}
return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: namespaces.Test}, netAttDef)
}, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred())
waitForNetAttachDef(sriovNetworkName, namespaces.Test)

testPodA := pod.RedefineWithNodeSelector(
pod.DefineWithNetworks([]string{sriovNetworkName, sriovNetworkName, sriovNetworkName, sriovNetworkName, sriovNetworkName}),
Expand Down Expand Up @@ -1031,6 +1011,7 @@ var _ = Describe("[sriov] operator", func() {
sn.Spec.LogLevel = "debug"
})
Expect(err).ToNot(HaveOccurred())
waitForNetAttachDef(sriovNetworkName, namespaces.Test)

podDeployTime := time.Now()

Expand Down Expand Up @@ -1381,10 +1362,7 @@ var _ = Describe("[sriov] operator", func() {
ipam := ipamIpv4
err := network.CreateSriovNetwork(clients, mainDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam)
Expect(err).ToNot(HaveOccurred())
Eventually(func() error {
netAttDef := &netattdefv1.NetworkAttachmentDefinition{}
return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: namespaces.Test}, netAttDef)
}, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred())
waitForNetAttachDef(sriovNetworkName, namespaces.Test)
createTestPod(nodeToTest, []string{sriovNetworkName})
})
})
Expand Down Expand Up @@ -1430,10 +1408,7 @@ var _ = Describe("[sriov] operator", func() {

err = network.CreateSriovNetwork(clients, unusedSriovDevice, sriovNetworkName, namespaces.Test, operatorNamespace, resourceName, ipam)
Expect(err).ToNot(HaveOccurred())
Eventually(func() error {
netAttDef := &netattdefv1.NetworkAttachmentDefinition{}
return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: sriovNetworkName, Namespace: namespaces.Test}, netAttDef)
}, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred())
waitForNetAttachDef(sriovNetworkName, namespaces.Test)
changeNodeInterfaceState(testNode, unusedSriovDevice.Name, false)
pod := createTestPod(testNode, []string{sriovNetworkName})
ips, err := network.GetSriovNicIPs(pod, "net1")
Expand Down Expand Up @@ -1794,10 +1769,7 @@ var _ = Describe("[sriov] operator", func() {
ipam := ipamIpv4
err = network.CreateSriovNetwork(clients, &nic, ipv6NetworkName, namespaces.Test, operatorNamespace, resourceName, ipam)
Expect(err).ToNot(HaveOccurred())
Eventually(func() error {
netAttDef := &netattdefv1.NetworkAttachmentDefinition{}
return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: ipv6NetworkName, Namespace: namespaces.Test}, netAttDef)
}, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred())
waitForNetAttachDef(ipv6NetworkName, namespaces.Test)

By("creating a pod")
pod := createTestPod(node, []string{ipv6NetworkName})
Expand Down Expand Up @@ -2719,3 +2691,10 @@ func getMultusPodLogs(nodeName string, since time.Time) []string {

return strings.Split(string(rawLogs), "\n")
}

func waitForNetAttachDef(name, namespace string) {
Eventually(func() error {
netAttDef := &netattdefv1.NetworkAttachmentDefinition{}
return clients.Get(context.Background(), runtimeclient.ObjectKey{Name: name, Namespace: namespace}, netAttDef)
}, (10+snoTimeoutMultiplier*110)*time.Second, 1*time.Second).ShouldNot(HaveOccurred())
}

0 comments on commit 20c08f5

Please sign in to comment.