vhost-vdpa support

SriovNetworkNodePolicy API: VDPA device type can take now the following values: "virtio" and "vhost" Vhost/vdpa: Same behaviour of virtio/vdpa apart for the drivers that are loaded in the kernel (vhost-vdpa instead of virtio-vdpa) Signed-off-by: Leonardo Milleri <[email protected]>
k8snetworkplumbingwg · Jul 18, 2023 · 25222d7 · 25222d7
1 parent 48349d8
commit 25222d7
Show file tree

Hide file tree

Showing 10 changed files with 168 additions and 20 deletions.
diff --git a/api/v1/sriovnetworknodepolicy_types.go b/api/v1/sriovnetworknodepolicy_types.go
@@ -54,8 +54,8 @@ type SriovNetworkNodePolicySpec struct {
 	// +kubebuilder:validation:Enum=legacy;switchdev
 	// NIC Device Mode. Allowed value "legacy","switchdev".
 	EswitchMode string `json:"eSwitchMode,omitempty"`
-	// +kubebuilder:validation:Enum=virtio
-	// VDPA device type. Allowed value "virtio"
+	// +kubebuilder:validation:Enum=virtio;vhost
+	// VDPA device type. Allowed value "virtio", "vhost"
 	VdpaType string `json:"vdpaType,omitempty"`
 	// Exclude device's NUMA node when advertising this resource by SRIOV network device plugin. Default to false.
 	ExcludeTopology bool `json:"excludeTopology,omitempty"`

diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml
@@ -117,9 +117,10 @@ spec:
                 description: SRIOV Network device plugin endpoint resource name
                 type: string
               vdpaType:
-                description: VDPA device type. Allowed value "virtio"
+                description: VDPA device type. Allowed value "virtio", "vhost"
                 enum:
                 - virtio
+                - vhost
                 type: string
             required:
             - nicSelector

diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml
@@ -42,8 +42,9 @@ spec:
                 description: NodeSelector selects the nodes to be configured
                 type: object
               configurationMode:
-                description: Flag to enable the sriov-network-config-daemon to use
-                  a systemd mode instead of the regular method
+                description: 'Flag to enable the sriov-network-config-daemon to use
+                  a systemd service to configure SR-IOV devices on boot Default mode:
+                  daemon'
                 enum:
                 - daemon
                 - systemd

diff --git a/...yment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/...yment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml
@@ -117,9 +117,10 @@ spec:
                 description: SRIOV Network device plugin endpoint resource name
                 type: string
               vdpaType:
-                description: VDPA device type. Allowed value "virtio"
+                description: VDPA device type. Allowed value "virtio", "vhost"
                 enum:
                 - virtio
+                - vhost
                 type: string
             required:
             - nicSelector

diff --git a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml
@@ -42,8 +42,9 @@ spec:
                 description: NodeSelector selects the nodes to be configured
                 type: object
               configurationMode:
-                description: Flag to enable the sriov-network-config-daemon to use
-                  a systemd mode instead of the regular method
+                description: 'Flag to enable the sriov-network-config-daemon to use
+                  a systemd service to configure SR-IOV devices on boot Default mode:
+                  daemon'
                 enum:
                 - daemon
                 - systemd

diff --git a/pkg/consts/constants.go b/pkg/consts/constants.go
@@ -32,4 +32,5 @@ const (
 	DeviceTypeVfioPci   = "vfio-pci"
 	DeviceTypeNetDevice = "netdevice"
 	VdpaTypeVirtio      = "virtio"
+	VdpaTypeVhost       = "vhost"
 )
diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go
@@ -23,12 +23,14 @@ var PluginName = "generic_plugin"
 const (
 	Vfio = iota
 	VirtioVdpa
+	VhostVdpa
 )
 
 // driver name
 const (
 	vfioPciDriver    = "vfio_pci"
 	virtioVdpaDriver = "virtio_vdpa"
+	vhostVdpaDriver  = "vhost_vdpa"
 )
 
 // driver state
@@ -80,6 +82,13 @@ func NewGenericPlugin(runningOnHost bool) (plugin.VendorPlugin, error) {
 		NeedDriverFunc: needDriverCheckVdpaType,
 		State:          Unloaded,
 	}
+	driverStateMap[VhostVdpa] = &DriverState{
+		DriverName:     vhostVdpaDriver,
+		DeviceType:     constants.DeviceTypeNetDevice,
+		VdpaType:       constants.VdpaTypeVhost,
+		NeedDriverFunc: needDriverCheckVdpaType,
+		State:          Unloaded,
+	}
 
 	return &GenericPlugin{
 		PluginName:     PluginName,

diff --git a/pkg/plugins/generic/generic_plugin_test.go b/pkg/plugins/generic/generic_plugin_test.go
@@ -245,6 +245,66 @@ var _ = Describe("Generic plugin", func() {
 			driverState := driverStateMap[generic.VirtioVdpa]
 			Expect(driverState.State).To(Equal(uint(generic.Loading)))
 		})
+
+		It("should load vhost_vdpa driver", func() {
+			networkNodeState := &sriovnetworkv1.SriovNetworkNodeState{
+				Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{
+					Interfaces: sriovnetworkv1.Interfaces{{
+						PciAddress: "0000:00:00.0",
+						NumVfs:     2,
+						Mtu:        1500,
+						VfGroups: []sriovnetworkv1.VfGroup{{
+							DeviceType:   "netdevice",
+							VdpaType:     "vhost",
+							PolicyName:   "policy-1",
+							ResourceName: "resource-1",
+							VfRange:      "0-1",
+							Mtu:          1500,
+						}}}},
+				},
+				Status: sriovnetworkv1.SriovNetworkNodeStateStatus{
+					Interfaces: sriovnetworkv1.InterfaceExts{{
+						PciAddress:  "0000:00:00.0",
+						NumVfs:      2,
+						TotalVfs:    2,
+						DeviceID:    "1015",
+						Vendor:      "15b3",
+						Name:        "sriovif1",
+						Mtu:         1500,
+						Mac:         "0c:42:a1:55:ee:46",
+						Driver:      "mlx5_core",
+						EswitchMode: "legacy",
+						LinkSpeed:   "25000 Mb/s",
+						LinkType:    "ETH",
+						VFs: []sriovnetworkv1.VirtualFunction{{
+							PciAddress: "0000:00:00.1",
+							DeviceID:   "1016",
+							Vendor:     "15b3",
+							VfID:       0,
+							Driver:     "mlx5_core",
+							Name:       "sriovif1v0",
+							Mtu:        1500,
+							Mac:        "8e:d6:2c:62:87:1b",
+						}, {
+							PciAddress: "0000:00:00.2",
+							DeviceID:   "1016",
+							Vendor:     "15b3",
+							VfID:       0,
+							Driver:     "mlx5_core",
+						}},
+					}},
+				},
+			}
+
+			needDrain, needReboot, err := genericPlugin.OnNodeStateChange(networkNodeState)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(needReboot).To(BeFalse())
+			Expect(needDrain).To(BeFalse())
+			concretePlugin := genericPlugin.(*generic.GenericPlugin)
+			driverStateMap := concretePlugin.GetDriverStateMap()
+			driverState := driverStateMap[generic.VhostVdpa]
+			Expect(driverState.State).To(Equal(uint(generic.Loading)))
+		})
 	})
 
 })
diff --git a/pkg/webhook/validate.go b/pkg/webhook/validate.go
@@ -203,12 +203,12 @@ func staticValidateSriovNetworkNodePolicy(cr *sriovnetworkv1.SriovNetworkNodePol
 	}
 
 	// vdpa: deviceType must be set to 'netdevice'
-	if cr.Spec.DeviceType != constants.DeviceTypeNetDevice && cr.Spec.VdpaType == constants.VdpaTypeVirtio {
-		return false, fmt.Errorf("'deviceType: %s' conflicts with 'vdpaType: virtio'; Set 'deviceType' to (string)'netdevice' Or Remove 'vdpaType'", cr.Spec.DeviceType)
+	if cr.Spec.DeviceType != constants.DeviceTypeNetDevice && (cr.Spec.VdpaType == constants.VdpaTypeVirtio || cr.Spec.VdpaType == constants.VdpaTypeVhost) {
+		return false, fmt.Errorf("'deviceType: %s' conflicts with '%s'; Set 'deviceType' to (string)'netdevice' Or Remove 'vdpaType'", cr.Spec.DeviceType, cr.Spec.VdpaType)
 	}
 	// vdpa: device must be configured in switchdev mode
-	if cr.Spec.VdpaType == constants.VdpaTypeVirtio && cr.Spec.EswitchMode != sriovnetworkv1.ESwithModeSwitchDev {
-		return false, fmt.Errorf("virtio/vdpa requires the device to be configured in switchdev mode")
+	if (cr.Spec.VdpaType == constants.VdpaTypeVirtio || cr.Spec.VdpaType == constants.VdpaTypeVhost) && cr.Spec.EswitchMode != sriovnetworkv1.ESwithModeSwitchDev {
+		return false, fmt.Errorf("vdpa requires the device to be configured in switchdev mode")
 	}
 	return true, nil
 }
@@ -286,8 +286,8 @@ func validatePolicyForNodeState(policy *sriovnetworkv1.SriovNetworkNodePolicy, s
 				return fmt.Errorf("numVfs(%d) in CR %s exceed the maximum allowed value(%d)", policy.Spec.NumVfs, policy.GetName(), MlxMaxVFs)
 			}
 			// vdpa: only mellanox cards are supported
-			if policy.Spec.VdpaType == constants.VdpaTypeVirtio && iface.Vendor != MellanoxID {
-				return fmt.Errorf("vendor(%s) in CR %s not supported for virtio-vdpa", iface.Vendor, policy.GetName())
+			if (policy.Spec.VdpaType == constants.VdpaTypeVirtio || policy.Spec.VdpaType == constants.VdpaTypeVhost) && iface.Vendor != MellanoxID {
+				return fmt.Errorf("vendor(%s) in CR %s not supported for vdpa", iface.Vendor, policy.GetName())
 			}
 		}
 	}

diff --git a/pkg/webhook/validate_test.go b/pkg/webhook/validate_test.go
@@ -538,7 +538,7 @@ func TestStaticValidateSriovNetworkNodePolicyWithConflictIsRdmaAndDeviceType(t *
 	g.Expect(ok).To(Equal(false))
 }
 
-func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVdpaType(t *testing.T) {
+func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVirtioVdpaType(t *testing.T) {
 	policy := &SriovNetworkNodePolicy{
 		Spec: SriovNetworkNodePolicySpec{
 			DeviceType: constants.DeviceTypeVfioPci,
@@ -558,11 +558,35 @@ func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVdpaType(t
 	}
 	g := NewGomegaWithT(t)
 	ok, err := staticValidateSriovNetworkNodePolicy(policy)
-	g.Expect(err).To(MatchError(ContainSubstring("'deviceType: vfio-pci' conflicts with 'vdpaType: virtio'")))
+	g.Expect(err).To(MatchError(ContainSubstring("'deviceType: vfio-pci' conflicts with 'virtio'")))
 	g.Expect(ok).To(Equal(false))
 }
 
-func TestStaticValidateSriovNetworkNodePolicyVdpaMustSpecifySwitchDev(t *testing.T) {
+func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVhostVdpaType(t *testing.T) {
+	policy := &SriovNetworkNodePolicy{
+		Spec: SriovNetworkNodePolicySpec{
+			DeviceType: constants.DeviceTypeVfioPci,
+			NicSelector: SriovNetworkNicSelector{
+				Vendor:   "15b3",
+				DeviceID: "101d",
+			},
+			NodeSelector: map[string]string{
+				"feature.node.kubernetes.io/network-sriov.capable": "true",
+			},
+			NumVfs:       1,
+			Priority:     99,
+			ResourceName: "p0",
+			VdpaType:     constants.VdpaTypeVhost,
+			EswitchMode:  "switchdev",
+		},
+	}
+	g := NewGomegaWithT(t)
+	ok, err := staticValidateSriovNetworkNodePolicy(policy)
+	g.Expect(err).To(MatchError(ContainSubstring("'deviceType: vfio-pci' conflicts with 'vhost'")))
+	g.Expect(ok).To(Equal(false))
+}
+
+func TestStaticValidateSriovNetworkNodePolicyVirtioVdpaMustSpecifySwitchDev(t *testing.T) {
 	policy := &SriovNetworkNodePolicy{
 		Spec: SriovNetworkNodePolicySpec{
 			DeviceType: "netdevice",
@@ -581,11 +605,34 @@ func TestStaticValidateSriovNetworkNodePolicyVdpaMustSpecifySwitchDev(t *testing
 	}
 	g := NewGomegaWithT(t)
 	ok, err := staticValidateSriovNetworkNodePolicy(policy)
-	g.Expect(err).To(MatchError(ContainSubstring("virtio/vdpa requires the device to be configured in switchdev mode")))
+	g.Expect(err).To(MatchError(ContainSubstring("vdpa requires the device to be configured in switchdev mode")))
 	g.Expect(ok).To(Equal(false))
 }
 
-func TestValidatePolicyForNodeStateVdpaWithNotSupportedVendor(t *testing.T) {
+func TestStaticValidateSriovNetworkNodePolicyVhostVdpaMustSpecifySwitchDev(t *testing.T) {
+	policy := &SriovNetworkNodePolicy{
+		Spec: SriovNetworkNodePolicySpec{
+			DeviceType: "netdevice",
+			NicSelector: SriovNetworkNicSelector{
+				Vendor:   "15b3",
+				DeviceID: "101d",
+			},
+			NodeSelector: map[string]string{
+				"feature.node.kubernetes.io/network-sriov.capable": "true",
+			},
+			NumVfs:       1,
+			Priority:     99,
+			ResourceName: "p0",
+			VdpaType:     constants.VdpaTypeVhost,
+		},
+	}
+	g := NewGomegaWithT(t)
+	ok, err := staticValidateSriovNetworkNodePolicy(policy)
+	g.Expect(err).To(MatchError(ContainSubstring("vdpa requires the device to be configured in switchdev mode")))
+	g.Expect(ok).To(Equal(false))
+}
+
+func TestValidatePolicyForNodeStateVirtioVdpaWithNotSupportedVendor(t *testing.T) {
 	state := newNodeState()
 	policy := &SriovNetworkNodePolicy{
 		ObjectMeta: metav1.ObjectMeta{
@@ -609,7 +656,34 @@ func TestValidatePolicyForNodeStateVdpaWithNotSupportedVendor(t *testing.T) {
 	}
 	g := NewGomegaWithT(t)
 	err := validatePolicyForNodeState(policy, state, NewNode())
-	g.Expect(err).To(MatchError(ContainSubstring(fmt.Sprintf("vendor(%s) in CR %s not supported for virtio-vdpa", state.Status.Interfaces[0].Vendor, policy.Name))))
+	g.Expect(err).To(MatchError(ContainSubstring(fmt.Sprintf("vendor(%s) in CR %s not supported for vdpa", state.Status.Interfaces[0].Vendor, policy.Name))))
+}
+
+func TestValidatePolicyForNodeStateVhostVdpaWithNotSupportedVendor(t *testing.T) {
+	state := newNodeState()
+	policy := &SriovNetworkNodePolicy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "p1",
+		},
+		Spec: SriovNetworkNodePolicySpec{
+			DeviceType: "netdevice",
+			VdpaType:   "vhost",
+			NicSelector: SriovNetworkNicSelector{
+				PfNames:     []string{"ens803f0"},
+				RootDevices: []string{"0000:86:00.0"},
+				Vendor:      "8086",
+			},
+			NodeSelector: map[string]string{
+				"feature.node.kubernetes.io/network-sriov.capable": "true",
+			},
+			NumVfs:       4,
+			Priority:     99,
+			ResourceName: "p0",
+		},
+	}
+	g := NewGomegaWithT(t)
+	err := validatePolicyForNodeState(policy, state, NewNode())
+	g.Expect(err).To(MatchError(ContainSubstring(fmt.Sprintf("vendor(%s) in CR %s not supported for vdpa", state.Status.Interfaces[0].Vendor, policy.Name))))
 }
 
 func TestValidatePolicyForNodeStateWithInvalidDevice(t *testing.T) {