From fb0b85e4bc4d8aeedf98c2e8ec6a93f2c24c7ae0 Mon Sep 17 00:00:00 2001 From: Sebastian Sch Date: Wed, 14 Jun 2023 14:09:31 +0300 Subject: [PATCH] virtual cluster e2e tests changes Signed-off-by: Sebastian Sch --- .github/workflows/virtual-cluster.yml | 33 ++ Makefile | 12 + deploy/configmap.yaml | 1 + .../templates/configmap.yaml | 1 + doc/testing-virtual-machine.md | 58 +++ hack/run-e2e-conformance-virtual-cluster.sh | 350 ++++++++++++++++++ hack/run-e2e-conformance-virtual-ocp.sh | 225 +++++++++++ hack/run-e2e-conformance.sh | 4 +- test/conformance/test_suite_test.go | 3 + test/conformance/tests/test_sriov_operator.go | 53 ++- test/util/cluster/cluster.go | 12 +- test/util/network/network.go | 5 + test/util/pod/pod.go | 5 + 13 files changed, 740 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/virtual-cluster.yml create mode 100644 doc/testing-virtual-machine.md create mode 100755 hack/run-e2e-conformance-virtual-cluster.sh create mode 100755 hack/run-e2e-conformance-virtual-ocp.sh diff --git a/.github/workflows/virtual-cluster.yml b/.github/workflows/virtual-cluster.yml new file mode 100644 index 0000000000..9eb55bf82c --- /dev/null +++ b/.github/workflows/virtual-cluster.yml @@ -0,0 +1,33 @@ +name: sriov-operator-test +on: [pull_request] + +jobs: + virtual-k8s-cluster: + name: k8s + runs-on: [sriov] + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Set up Go 1.20 + uses: actions/setup-go@v3 + with: + go-version: 1.20.x + + - name: run test + run: make test-e2e-conformance-virtual-k8s-cluster-ci + + virtual-ocp: + name: ocp + runs-on: [ ocp ] + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Set up Go 1.20 + uses: actions/setup-go@v3 + with: + go-version: 1.20.x + + - name: run test + run: make test-e2e-conformance-virtual-ocp-cluster-ci \ No newline at end of file diff --git a/Makefile b/Makefile index f48baa6813..79ca433eb1 100644 --- a/Makefile +++ b/Makefile @@ -181,6 +181,18 @@ deploy-setup-k8s: deploy-setup test-e2e-conformance: SUITE=./test/conformance ./hack/run-e2e-conformance.sh +test-e2e-conformance-virtual-k8s-cluster-ci: + ./hack/run-e2e-conformance-virtual-cluster.sh + +test-e2e-conformance-virtual-k8s-cluster: + SKIP_DELETE=TRUE ./hack/run-e2e-conformance-virtual-cluster.sh + +test-e2e-conformance-virtual-ocp-cluster-ci: + ./hack/run-e2e-conformance-virtual-ocp.sh + +test-e2e-conformance-virtual-ocp-cluster: + SKIP_DELETE=TRUE ./hack/run-e2e-conformance-virtual-ocp.sh + test-e2e-validation-only: SUITE=./test/validation ./hack/run-e2e-conformance.sh diff --git a/deploy/configmap.yaml b/deploy/configmap.yaml index 9d7cf8596e..41ce4c8095 100644 --- a/deploy/configmap.yaml +++ b/deploy/configmap.yaml @@ -7,6 +7,7 @@ data: Intel_i40e_25G_SFP28: "8086 158b 154c" Intel_i40e_10G_X710_SFP: "8086 1572 154c" Intel_ixgbe_10G_X550: "8086 1563 1565" + Intel_ixgbe_82576: "8086 10c9 10ca" Intel_i40e_X710_X557_AT_10G: "8086 1589 154c" Intel_i40e_10G_X710_BACKPLANE: "8086 1581 154c" Intel_i40e_10G_X710_BASE_T: "8086 15ff 154c" diff --git a/deployment/sriov-network-operator/templates/configmap.yaml b/deployment/sriov-network-operator/templates/configmap.yaml index 9d7cf8596e..41ce4c8095 100644 --- a/deployment/sriov-network-operator/templates/configmap.yaml +++ b/deployment/sriov-network-operator/templates/configmap.yaml @@ -7,6 +7,7 @@ data: Intel_i40e_25G_SFP28: "8086 158b 154c" Intel_i40e_10G_X710_SFP: "8086 1572 154c" Intel_ixgbe_10G_X550: "8086 1563 1565" + Intel_ixgbe_82576: "8086 10c9 10ca" Intel_i40e_X710_X557_AT_10G: "8086 1589 154c" Intel_i40e_10G_X710_BACKPLANE: "8086 1581 154c" Intel_i40e_10G_X710_BASE_T: "8086 15ff 154c" diff --git a/doc/testing-virtual-machine.md b/doc/testing-virtual-machine.md new file mode 100644 index 0000000000..31fce2457a --- /dev/null +++ b/doc/testing-virtual-machine.md @@ -0,0 +1,58 @@ +## E2E conformance test + +It's possible to use QEMU to test the SR-IOV operator on a virtual kubernetes/openshift cluster. +Using the IGB model network driver allow to create virtual functions on the virtual system + +## How to test + +First you will need to enable the `DEV_MODE` via the operator environment variable. +Second step is to add the intel virtual nic to the supported nics configmap. + +Another requirement is to load the vfio kernel module with no_iommu configuration. Example systemd: + +``` +[Unit] +Description=vfio no-iommu +Before=kubelet.service crio.service node-valid-hostname.service + +[Service] +# Need oneshot to delay kubelet +Type=oneshot +ExecStart=/usr/bin/bash -c "modprobe vfio enable_unsafe_noiommu_mode=1" +StandardOutput=journal+console +StandardError=journal+console + +[Install] +WantedBy=network-online.target +``` + +### Prerequisites +* kcli - deployment tool (https://github.com/karmab/kcli) +* virsh +* qemu > 8.1 +* libvirt > 9 +* podman +* make +* go + +## Deploy the cluster + +use the deployment [script](../hack/run-e2e-conformance-virtual-cluster.sh), this will deploy a k8s cluster +compile the operator images and run the e2e tests. + +example: +``` +SKIP_DELETE=TRUE make test-e2e-conformance-virtual-k8s-cluster +``` + +It's also possible to skip the tests and only deploy the cluster running + +``` +SKIP_TEST=TRUE SKIP_DELETE=TRUE make test-e2e-conformance-virtual-k8s-cluster +``` + +To use the cluster after the deployment you need to export the kubeconfig + +``` +export KUBECONFIG=$HOME/.kcli/clusters/virtual/auth/kubeconfig +``` \ No newline at end of file diff --git a/hack/run-e2e-conformance-virtual-cluster.sh b/hack/run-e2e-conformance-virtual-cluster.sh new file mode 100755 index 0000000000..ffe1948980 --- /dev/null +++ b/hack/run-e2e-conformance-virtual-cluster.sh @@ -0,0 +1,350 @@ +#!/usr/bin/env bash +set -xeo pipefail + +cluster_name=${CLUSTER_NAME:-virtual} +domain_name=$cluster_name.lab + +api_ip=${API_IP:-192.168.122.250} +virtual_router_id=${VIRTUAL_ROUTER_ID:-250} +HOME="/root" + +here="$(dirname "$(readlink --canonicalize "${BASH_SOURCE[0]}")")" +root="$(readlink --canonicalize "$here/..")" + +check_requirements() { + for cmd in kcli virsh virt-edit podman make go; do + if ! command -v "$cmd" &> /dev/null; then + echo "$cmd is not available" + exit 1 + fi + done + return 0 +} + +echo "## checking requirements" +check_requirements +echo "## delete existing cluster name $cluster_name" +kcli delete cluster $cluster_name -y +kcli delete network $cluster_name -y + +function cleanup { + kcli delete cluster $cluster_name -y + kcli delete network $cluster_name -y +} + +if [ -z $SKIP_DELETE ]; then + trap cleanup EXIT +fi + +kcli create network -c 192.168.${virtual_router_id}.0/24 --nodhcp -i $cluster_name + +cat < ./${cluster_name}-plan.yaml +ctlplane_memory: 4096 +worker_memory: 4096 +pool: default +disk_size: 50 +network: default +api_ip: $api_ip +virtual_router_id: $virtual_router_id +domain: $domain_name +ctlplanes: 1 +workers: 2 +ingress: false +machine: q35 +engine: crio +sdn: flannel +autolabeller: false +vmrules: + - $cluster_name-worker-.*: + nets: + - name: default + type: igb + vfio: true + noconf: true + numa: 0 + - name: $cluster_name + type: igb + vfio: true + noconf: true + numa: 1 + numcpus: 6 + numa: + - id: 0 + vcpus: 0,2,4 + memory: 2048 + - id: 1 + vcpus: 1,3,5 + memory: 2048 + +EOF + +kcli create cluster generic --paramfile ./${cluster_name}-plan.yaml $cluster_name + +export KUBECONFIG=$HOME/.kcli/clusters/$cluster_name/auth/kubeconfig +export PATH=$PWD:$PATH + +ATTEMPTS=0 +MAX_ATTEMPTS=72 +ready=false +sleep_time=10 + +until $ready || [ $ATTEMPTS -eq $MAX_ATTEMPTS ] +do + echo "waiting for cluster to be ready" + if [ `kubectl get node | grep Ready | wc -l` == 3 ]; then + echo "cluster is ready" + ready=true + else + echo "cluster is not ready yet" + sleep $sleep_time + fi + ATTEMPTS=$((ATTEMPTS+1)) +done + +if ! $ready; then + echo "Timed out waiting for cluster to be ready" + kubectl get nodes + exit 1 +fi + +echo "## label cluster workers as sriov capable" +kubectl label node $cluster_name-worker-0.$domain_name feature.node.kubernetes.io/network-sriov.capable=true --overwrite +kubectl label node $cluster_name-worker-1.$domain_name feature.node.kubernetes.io/network-sriov.capable=true --overwrite + +echo "## label cluster worker as worker" +kubectl label node $cluster_name-worker-0.$domain_name node-role.kubernetes.io/worker= --overwrite +kubectl label node $cluster_name-worker-1.$domain_name node-role.kubernetes.io/worker= --overwrite + +controller_ip=`kubectl get node -o wide | grep ctlp | awk '{print $6}'` +insecure_registry="[[registry]] +location = \"$controller_ip:5000\" +insecure = true +" + +cat << EOF > /etc/containers/registries.conf.d/003-${cluster_name}.conf +$insecure_registry +EOF + +kcli ssh $cluster_name-ctlplane-0 << EOF +sudo su +echo '$insecure_registry' > /etc/containers/registries.conf.d/003-internal.conf +systemctl restart crio +EOF + +kcli ssh $cluster_name-worker-0 << EOF +sudo su +echo '$insecure_registry' > /etc/containers/registries.conf.d/003-internal.conf +systemctl restart crio +EOF + +kcli ssh $cluster_name-worker-1 << EOF +sudo su +echo '$insecure_registry' > /etc/containers/registries.conf.d/003-internal.conf +systemctl restart crio +EOF + +kubectl create namespace container-registry + +echo "## deploy internal registry" +cat < /dev/null; then + echo "$cmd is not available" + exit 1 + fi + done + return 0 +} + +echo "## checking requirements" +check_requirements +echo "## delete existing cluster name $cluster_name" +kcli delete cluster $cluster_name -y +kcli delete network $cluster_name -y + +function cleanup { + kcli delete cluster $cluster_name -y + kcli delete network $cluster_name -y + podman logout $registry +} + +if [ -z $SKIP_DELETE ]; then + trap cleanup EXIT +fi + +kcli create network -c 192.168.${virtual_router_id}.0/24 --nodhcp -i $cluster_name + +cat < ./${cluster_name}-plan.yaml +tag: 4.14.0-rc.1 +ctlplane_memory: 24576 +worker_memory: 8192 +pool: default +disk_size: 50 +network: default +api_ip: $api_ip +virtual_router_id: $virtual_router_id +domain: $domain_name +ctlplanes: 1 +workers: 2 +machine: q35 +network_type: OVNKubernetes +pull_secret: /root/openshift_pull.json +vmrules: + - $cluster_name-worker-.*: + nets: + - name: default + numa: 0 + - name: $cluster_name + type: igb + vfio: true + noconf: true + numa: 0 + - name: $cluster_name + type: igb + vfio: true + noconf: true + numa: 1 + numcpus: 6 + numa: + - id: 0 + vcpus: 0,2,4 + memory: 4096 + - id: 1 + vcpus: 1,3,5 + memory: 4096 + +EOF + +kcli create cluster openshift --paramfile ./${cluster_name}-plan.yaml $cluster_name +kubectl create clusterrolebinding authenticated-registry-viewer --clusterrole registry-viewer --group system:unauthenticated + +export KUBECONFIG=$HOME/.kcli/clusters/$cluster_name/auth/kubeconfig +export PATH=$PWD:$PATH + + +ATTEMPTS=0 +MAX_ATTEMPTS=72 +ready=false +sleep_time=10 + +until $ready || [ $ATTEMPTS -eq $MAX_ATTEMPTS ] +do + echo "waiting for cluster to be ready" + if [ `kubectl get node | grep Ready | wc -l` == 3 ]; then + echo "cluster is ready" + ready=true + else + echo "cluster is not ready yet" + sleep $sleep_time + fi + ATTEMPTS=$((ATTEMPTS+1)) +done + +if ! $ready; then + echo "Timed out waiting for cluster to be ready" + kubectl get nodes + exit 1 +fi + +echo "## label cluster workers as sriov capable" +kubectl label node $cluster_name-worker-0.$domain_name feature.node.kubernetes.io/network-sriov.capable=true --overwrite +kubectl label node $cluster_name-worker-1.$domain_name feature.node.kubernetes.io/network-sriov.capable=true --overwrite + +controller_ip=`kubectl get node -o wide | grep ctlp | awk '{print $6}'` + +if [ `cat /etc/hosts | grep ${api_ip} | grep "default-route-openshift-image-registry.apps.${cluster_name}.${domain_name}" | wc -l` == 0 ]; then + echo "adding registry to hosts" + sed -i "s/${api_ip}/${api_ip} default-route-openshift-image-registry.apps.${cluster_name}.${domain_name}/g" /etc/hosts +fi + + +cat < registry-login.conf + +pass=$( jq .\"$registry\".password registry-login.conf ) +podman login -u serviceaccount -p ${pass:1:-1} $registry --tls-verify=false + +podman push --tls-verify=false "${SRIOV_NETWORK_OPERATOR_IMAGE}" +podman push --tls-verify=false "${SRIOV_NETWORK_CONFIG_DAEMON_IMAGE}" +podman push --tls-verify=false "${SRIOV_NETWORK_WEBHOOK_IMAGE}" + +podman logout $registry + +echo "## apply CRDs" +kubectl apply -k $root/config/crd + +export SRIOV_NETWORK_OPERATOR_IMAGE="image-registry.openshift-image-registry.svc:5000/$NAMESPACE/sriov-network-operator:latest" +export SRIOV_NETWORK_CONFIG_DAEMON_IMAGE="image-registry.openshift-image-registry.svc:5000/$NAMESPACE/sriov-network-config-daemon:latest" +export SRIOV_NETWORK_WEBHOOK_IMAGE="image-registry.openshift-image-registry.svc:5000/$NAMESPACE/sriov-network-operator-webhook:latest" + +echo "## deploying SRIOV Network Operator" +hack/deploy-setup.sh $NAMESPACE + +echo "## wait for sriov operator to be ready" + +echo "## wait for sriov operator to be ready" +hack/deploy-wait.sh + +if [ -z $SKIP_TEST ]; then + echo "## run sriov e2e conformance tests" + SUITE=./test/conformance hack/run-e2e-conformance.sh +fi diff --git a/hack/run-e2e-conformance.sh b/hack/run-e2e-conformance.sh index bc4a459509..df1ce9baef 100755 --- a/hack/run-e2e-conformance.sh +++ b/hack/run-e2e-conformance.sh @@ -8,7 +8,7 @@ if [ $? -ne 0 ]; then GINKGO_TMP_DIR=$(mktemp -d) cd $GINKGO_TMP_DIR go mod init tmp - go install -mod=readonly github.com/onsi/ginkgo/v2/ginkgo@v2.5.0 + go install -mod=readonly github.com/onsi/ginkgo/v2/ginkgo@v2.9.5 rm -rf $GINKGO_TMP_DIR echo "Downloading ginkgo tool" cd - @@ -18,4 +18,4 @@ GOPATH="${GOPATH:-~/go}" JUNIT_OUTPUT="${JUNIT_OUTPUT:-/tmp/artifacts}" export PATH=$PATH:$GOPATH/bin -GOFLAGS=-mod=vendor ginkgo -output-dir=$JUNIT_OUTPUT --junit-report "unit_report.xml" "$SUITE" -- -report=$JUNIT_OUTPUT +GOFLAGS=-mod=vendor ginkgo -output-dir=$JUNIT_OUTPUT --junit-report "unit_report.xml" -v "$SUITE" -- -report=$JUNIT_OUTPUT diff --git a/test/conformance/test_suite_test.go b/test/conformance/test_suite_test.go index 0bda0c0e41..af648ad421 100644 --- a/test/conformance/test_suite_test.go +++ b/test/conformance/test_suite_test.go @@ -11,6 +11,8 @@ import ( . "github.com/onsi/gomega" kniK8sReporter "github.com/openshift-kni/k8sreporter" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" // Test files in this package must not end with `_test.go` suffix, as they are imported as go package _ "github.com/k8snetworkplumbingwg/sriov-network-operator/test/conformance/tests" @@ -53,6 +55,7 @@ func TestTest(t *testing.T) { } } + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) RunSpecs(t, "SRIOV Operator conformance tests") } diff --git a/test/conformance/tests/test_sriov_operator.go b/test/conformance/tests/test_sriov_operator.go index a979ebe5f3..9bbb86d467 100644 --- a/test/conformance/tests/test_sriov_operator.go +++ b/test/conformance/tests/test_sriov_operator.go @@ -511,6 +511,11 @@ var _ = Describe("[sriov] operator", func() { // 25961 It("Should configure the the link state variable", func() { + if cluster.VirtualCluster() { + // https://bugzilla.redhat.com/show_bug.cgi?id=2214976 + Skip("Bug in IGB driver") + } + sriovNetwork := &sriovv1.SriovNetwork{ ObjectMeta: metav1.ObjectMeta{Name: "test-statenetwork", Namespace: operatorNamespace}, Spec: sriovv1.SriovNetworkSpec{ @@ -1161,6 +1166,11 @@ var _ = Describe("[sriov] operator", func() { Context("PF shutdown", func() { // 29398 It("Should be able to create pods successfully if PF is down.Pods are able to communicate with each other on the same node", func() { + if cluster.VirtualCluster() { + // https://bugzilla.redhat.com/show_bug.cgi?id=2214976 + Skip("Bug in IGB driver") + } + resourceName := testResourceName var testNode string var unusedSriovDevice *sriovv1.InterfaceExt @@ -1210,6 +1220,11 @@ var _ = Describe("[sriov] operator", func() { Context("MTU", func() { BeforeEach(func() { + if cluster.VirtualCluster() { + // https://bugzilla.redhat.com/show_bug.cgi?id=2214977 + Skip("Bug in IGB driver") + } + var node string resourceName := "mturesource" var numVfs int @@ -1305,7 +1320,6 @@ var _ = Describe("[sriov] operator", func() { ResourceName: resourceName, IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181","routes":[{"dst":"0.0.0.0/0"}],"gateway":"10.10.10.1"}`, NetworkNamespace: namespaces.Test, - LinkState: "enable", }} // We need this to be able to run the connectivity checks on Mellanox cards @@ -1395,11 +1409,11 @@ var _ = Describe("[sriov] operator", func() { }, Spec: sriovv1.SriovNetworkNodePolicySpec{ - NumVfs: 10, + NumVfs: 7, ResourceName: "resourceXXX", NodeSelector: map[string]string{"kubernetes.io/hostname": node}, NicSelector: sriovv1.SriovNetworkNicSelector{ - PfNames: []string{intf.Name + "#0-4"}, + PfNames: []string{intf.Name + "#0-3"}, }, ExcludeTopology: true, }, @@ -1412,11 +1426,11 @@ var _ = Describe("[sriov] operator", func() { }, Spec: sriovv1.SriovNetworkNodePolicySpec{ - NumVfs: 10, + NumVfs: 7, ResourceName: "resourceXXX", NodeSelector: map[string]string{"kubernetes.io/hostname": node}, NicSelector: sriovv1.SriovNetworkNicSelector{ - PfNames: []string{intf.Name + "#5-9"}, + PfNames: []string{intf.Name + "#4-6"}, }, ExcludeTopology: false, }, @@ -1429,11 +1443,11 @@ var _ = Describe("[sriov] operator", func() { }, Spec: sriovv1.SriovNetworkNodePolicySpec{ - NumVfs: 10, + NumVfs: 7, ResourceName: "resourceYYY", NodeSelector: map[string]string{"kubernetes.io/hostname": node}, NicSelector: sriovv1.SriovNetworkNicSelector{ - PfNames: []string{intf.Name + "#5-9"}, + PfNames: []string{intf.Name + "#4-6"}, }, ExcludeTopology: false, }, @@ -1447,15 +1461,15 @@ var _ = Describe("[sriov] operator", func() { Expect(err).ToNot(HaveOccurred()) assertDevicePluginConfigurationContains(node, - fmt.Sprintf(`{"resourceName":"resourceXXX","excludeTopology":true,"selectors":{"pfNames":["%s#0-4"],"IsRdma":false,"NeedVhostNet":false},"SelectorObj":null}`, intf.Name)) + fmt.Sprintf(`{"resourceName":"resourceXXX","excludeTopology":true,"selectors":{"pfNames":["%s#0-3"],"IsRdma":false,"NeedVhostNet":false},"SelectorObj":null}`, intf.Name)) err = clients.Create(context.Background(), excludeTopologyFalseResourceYYY) Expect(err).ToNot(HaveOccurred()) assertDevicePluginConfigurationContains(node, - fmt.Sprintf(`{"resourceName":"resourceXXX","excludeTopology":true,"selectors":{"pfNames":["%s#0-4"],"IsRdma":false,"NeedVhostNet":false},"SelectorObj":null}`, intf.Name)) + fmt.Sprintf(`{"resourceName":"resourceXXX","excludeTopology":true,"selectors":{"pfNames":["%s#0-3"],"IsRdma":false,"NeedVhostNet":false},"SelectorObj":null}`, intf.Name)) assertDevicePluginConfigurationContains(node, - fmt.Sprintf(`{"resourceName":"resourceYYY","selectors":{"pfNames":["%s#5-9"],"IsRdma":false,"NeedVhostNet":false},"SelectorObj":null}`, intf.Name)) + fmt.Sprintf(`{"resourceName":"resourceYYY","selectors":{"pfNames":["%s#4-6"],"IsRdma":false,"NeedVhostNet":false},"SelectorObj":null}`, intf.Name)) }) It("multiple values for the same resource should not be allowed", func() { @@ -1729,7 +1743,6 @@ var _ = Describe("[sriov] operator", func() { ResourceName: resourceName, IPAM: `{"type":"host-local","subnet":"10.10.10.0/24","rangeStart":"10.10.10.171","rangeEnd":"10.10.10.181","routes":[{"dst":"0.0.0.0/0"}],"gateway":"10.10.10.1"}`, NetworkNamespace: namespaces.Test, - LinkState: "enable", }} // We need this to be able to run the connectivity checks on Mellanox cards @@ -2084,9 +2097,9 @@ func createCustomTestPod(node string, networks []string, hostNetwork bool, podCa Expect(err).ToNot(HaveOccurred()) return runningPod.Status.Phase }, 5*time.Minute, 1*time.Second).Should(Equal(corev1.PodRunning)) - pod, err := clients.Pods(namespaces.Test).Get(context.Background(), createdPod.Name, metav1.GetOptions{}) + podObj, err := clients.Pods(namespaces.Test).Get(context.Background(), createdPod.Name, metav1.GetOptions{}) Expect(err).ToNot(HaveOccurred()) - return pod + return podObj } func pingPod(ip string, nodeSelector string, sriovNetworkAttachment string) { @@ -2095,15 +2108,19 @@ func pingPod(ip string, nodeSelector string, sriovNetworkAttachment string) { ipProtocolVersion = "4" } podDefinition := pod.RedefineWithNodeSelector( - pod.RedefineWithRestartPolicy( - pod.RedefineWithCommand( - pod.DefineWithNetworks([]string{sriovNetworkAttachment}), - []string{"sh", "-c", fmt.Sprintf("ping -%s -c 3 %s", ipProtocolVersion, ip)}, []string{}, + pod.RedefineWithCapabilities( + pod.RedefineWithRestartPolicy( + pod.RedefineWithCommand( + pod.DefineWithNetworks([]string{sriovNetworkAttachment}), + []string{"sh", "-c", fmt.Sprintf("ping -%s -c 3 %s", ipProtocolVersion, ip)}, []string{}, + ), + corev1.RestartPolicyNever, ), - corev1.RestartPolicyNever, + []corev1.Capability{"NET_RAW"}, ), nodeSelector, ) + createdPod, err := clients.Pods(namespaces.Test).Create(context.Background(), podDefinition, metav1.CreateOptions{}) Expect(err).ToNot(HaveOccurred()) diff --git a/test/util/cluster/cluster.go b/test/util/cluster/cluster.go index d5b9895ca2..004f61e463 100644 --- a/test/util/cluster/cluster.go +++ b/test/util/cluster/cluster.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "os" "strings" "time" @@ -28,8 +29,8 @@ type EnabledNodes struct { } var ( - supportedPFDrivers = []string{"mlx5_core", "i40e", "ixgbe", "ice"} - supportedVFDrivers = []string{"iavf", "vfio-pci", "mlx5_core"} + supportedPFDrivers = []string{"mlx5_core", "i40e", "ixgbe", "ice", "igb"} + supportedVFDrivers = []string{"iavf", "vfio-pci", "mlx5_core", "igbvf"} mlxVendorID = "15b3" intelVendorID = "8086" ) @@ -331,3 +332,10 @@ func GetNodeSecureBootState(clients *testclient.ClientSet, nodeName, namespace s return strings.Contains(stdout, "[integrity]") || strings.Contains(stdout, "[confidentiality]"), nil } + +func VirtualCluster() bool { + if v, exist := os.LookupEnv("CLUSTER_HAS_EMULATED_PF"); exist && v != "" { + return true + } + return false +} diff --git a/test/util/network/network.go b/test/util/network/network.go index 623f873c01..7b67777d7d 100644 --- a/test/util/network/network.go +++ b/test/util/network/network.go @@ -12,6 +12,7 @@ import ( sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" testclient "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/client" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/cluster" ) // Needed for parsing of podinfo @@ -37,6 +38,10 @@ func CreateSriovNetwork(clientSet *testclient.ClientSet, intf *sriovv1.Interface LinkState: "enable", }} + if cluster.VirtualCluster() { + sriovNetwork.Spec.LinkState = "" + } + for _, o := range options { o(sriovNetwork) } diff --git a/test/util/pod/pod.go b/test/util/pod/pod.go index 32234ac648..32a1549b9d 100644 --- a/test/util/pod/pod.go +++ b/test/util/pod/pod.go @@ -95,6 +95,11 @@ func RedefineWithRestartPolicy(pod *corev1.Pod, restartPolicy corev1.RestartPoli return pod } +func RedefineWithCapabilities(pod *corev1.Pod, capabilitiesList []corev1.Capability) *corev1.Pod { + pod.Spec.Containers[0].SecurityContext = &corev1.SecurityContext{Capabilities: &corev1.Capabilities{Add: capabilitiesList}} + return pod +} + // ExecCommand runs command in the pod and returns buffer output func ExecCommand(cs *testclient.ClientSet, pod *corev1.Pod, command ...string) (string, string, error) { var buf, errbuf bytes.Buffer