From 5a4fa022df291f2dfe0ba6c11f413630743a9b1d Mon Sep 17 00:00:00 2001 From: Leonardo Milleri Date: Tue, 18 Jul 2023 12:59:56 +0200 Subject: [PATCH] vhost-vdpa support SriovNetworkNodePolicy API: VDPA device type can take now the following values: "virtio" and "vhost" Vhost/vdpa: Same behaviour of virtio/vdpa apart for the drivers that are loaded in the kernel (vhost-vdpa instead of virtio-vdpa) Signed-off-by: Leonardo Milleri --- api/v1/sriovnetworknodepolicy_types.go | 4 +- ...openshift.io_sriovnetworknodepolicies.yaml | 3 +- ...ork.openshift.io_sriovoperatorconfigs.yaml | 5 +- ...openshift.io_sriovnetworknodepolicies.yaml | 3 +- ...ork.openshift.io_sriovoperatorconfigs.yaml | 5 +- pkg/consts/constants.go | 1 + pkg/plugins/generic/generic_plugin.go | 9 ++ pkg/plugins/generic/generic_plugin_test.go | 57 ++++++++++++ pkg/webhook/validate.go | 12 +-- pkg/webhook/validate_test.go | 86 +++++++++++++++++-- 10 files changed, 165 insertions(+), 20 deletions(-) diff --git a/api/v1/sriovnetworknodepolicy_types.go b/api/v1/sriovnetworknodepolicy_types.go index 8955c300da..17e22d2fad 100644 --- a/api/v1/sriovnetworknodepolicy_types.go +++ b/api/v1/sriovnetworknodepolicy_types.go @@ -54,8 +54,8 @@ type SriovNetworkNodePolicySpec struct { // +kubebuilder:validation:Enum=legacy;switchdev // NIC Device Mode. Allowed value "legacy","switchdev". EswitchMode string `json:"eSwitchMode,omitempty"` - // +kubebuilder:validation:Enum=virtio - // VDPA device type. Allowed value "virtio" + // +kubebuilder:validation:Enum=virtio;vhost + // VDPA device type. Allowed value "virtio", "vhost" VdpaType string `json:"vdpaType,omitempty"` // Exclude device's NUMA node when advertising this resource by SRIOV network device plugin. Default to false. ExcludeTopology bool `json:"excludeTopology,omitempty"` diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml index 5bebf01e05..a81c6a2276 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml @@ -117,9 +117,10 @@ spec: description: SRIOV Network device plugin endpoint resource name type: string vdpaType: - description: VDPA device type. Allowed value "virtio" + description: VDPA device type. Allowed value "virtio", "vhost" enum: - virtio + - vhost type: string required: - nicSelector diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml index e6864e5d06..f1666e78e3 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml @@ -42,8 +42,9 @@ spec: description: NodeSelector selects the nodes to be configured type: object configurationMode: - description: Flag to enable the sriov-network-config-daemon to use - a systemd mode instead of the regular method + description: 'Flag to enable the sriov-network-config-daemon to use + a systemd service to configure SR-IOV devices on boot Default mode: + daemon' enum: - daemon - systemd diff --git a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml index 5bebf01e05..a81c6a2276 100644 --- a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml +++ b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml @@ -117,9 +117,10 @@ spec: description: SRIOV Network device plugin endpoint resource name type: string vdpaType: - description: VDPA device type. Allowed value "virtio" + description: VDPA device type. Allowed value "virtio", "vhost" enum: - virtio + - vhost type: string required: - nicSelector diff --git a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml index e6864e5d06..f1666e78e3 100644 --- a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml +++ b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml @@ -42,8 +42,9 @@ spec: description: NodeSelector selects the nodes to be configured type: object configurationMode: - description: Flag to enable the sriov-network-config-daemon to use - a systemd mode instead of the regular method + description: 'Flag to enable the sriov-network-config-daemon to use + a systemd service to configure SR-IOV devices on boot Default mode: + daemon' enum: - daemon - systemd diff --git a/pkg/consts/constants.go b/pkg/consts/constants.go index 72e847d820..1f37edfa42 100644 --- a/pkg/consts/constants.go +++ b/pkg/consts/constants.go @@ -32,4 +32,5 @@ const ( DeviceTypeVfioPci = "vfio-pci" DeviceTypeNetDevice = "netdevice" VdpaTypeVirtio = "virtio" + VdpaTypeVhost = "vhost" ) diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go index 8f64fa9fab..ba8c02f147 100644 --- a/pkg/plugins/generic/generic_plugin.go +++ b/pkg/plugins/generic/generic_plugin.go @@ -23,12 +23,14 @@ var PluginName = "generic_plugin" const ( Vfio = iota VirtioVdpa + VhostVdpa ) // driver name const ( vfioPciDriver = "vfio_pci" virtioVdpaDriver = "virtio_vdpa" + vhostVdpaDriver = "vhost_vdpa" ) // function type for determining if a given driver has to be loaded in the kernel @@ -73,6 +75,13 @@ func NewGenericPlugin(runningOnHost bool) (plugin.VendorPlugin, error) { NeedDriverFunc: needDriverCheckVdpaType, DriverLoaded: false, } + driverStateMap[VhostVdpa] = &DriverState{ + DriverName: vhostVdpaDriver, + DeviceType: constants.DeviceTypeNetDevice, + VdpaType: constants.VdpaTypeVhost, + NeedDriverFunc: needDriverCheckVdpaType, + DriverLoaded: false, + } return &GenericPlugin{ PluginName: PluginName, diff --git a/pkg/plugins/generic/generic_plugin_test.go b/pkg/plugins/generic/generic_plugin_test.go index 8a66ff73bf..75035e44af 100644 --- a/pkg/plugins/generic/generic_plugin_test.go +++ b/pkg/plugins/generic/generic_plugin_test.go @@ -239,6 +239,63 @@ var _ = Describe("Generic plugin", func() { concretePlugin.LoadDriverForTests(networkNodeState) Expect(driverState.DriverLoaded).To(BeTrue()) }) + + It("should load vhost_vdpa driver", func() { + networkNodeState := &sriovnetworkv1.SriovNetworkNodeState{ + Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{ + Interfaces: sriovnetworkv1.Interfaces{{ + PciAddress: "0000:00:00.0", + NumVfs: 2, + Mtu: 1500, + VfGroups: []sriovnetworkv1.VfGroup{{ + DeviceType: "netdevice", + VdpaType: "vhost", + PolicyName: "policy-1", + ResourceName: "resource-1", + VfRange: "0-1", + Mtu: 1500, + }}}}, + }, + Status: sriovnetworkv1.SriovNetworkNodeStateStatus{ + Interfaces: sriovnetworkv1.InterfaceExts{{ + PciAddress: "0000:00:00.0", + NumVfs: 2, + TotalVfs: 2, + DeviceID: "1015", + Vendor: "15b3", + Name: "sriovif1", + Mtu: 1500, + Mac: "0c:42:a1:55:ee:46", + Driver: "mlx5_core", + EswitchMode: "legacy", + LinkSpeed: "25000 Mb/s", + LinkType: "ETH", + VFs: []sriovnetworkv1.VirtualFunction{{ + PciAddress: "0000:00:00.1", + DeviceID: "1016", + Vendor: "15b3", + VfID: 0, + Driver: "mlx5_core", + Name: "sriovif1v0", + Mtu: 1500, + Mac: "8e:d6:2c:62:87:1b", + }, { + PciAddress: "0000:00:00.2", + DeviceID: "1016", + Vendor: "15b3", + VfID: 0, + Driver: "mlx5_core", + }}, + }}, + }, + } + + concretePlugin := genericPlugin.(*generic.GenericPlugin) + driverStateMap := concretePlugin.GetDriverStateMap() + driverState := driverStateMap[generic.VhostVdpa] + concretePlugin.LoadDriverForTests(networkNodeState) + Expect(driverState.DriverLoaded).To(BeTrue()) + }) }) }) diff --git a/pkg/webhook/validate.go b/pkg/webhook/validate.go index 91b3922fa4..571d93d37e 100644 --- a/pkg/webhook/validate.go +++ b/pkg/webhook/validate.go @@ -203,12 +203,12 @@ func staticValidateSriovNetworkNodePolicy(cr *sriovnetworkv1.SriovNetworkNodePol } // vdpa: deviceType must be set to 'netdevice' - if cr.Spec.DeviceType != constants.DeviceTypeNetDevice && cr.Spec.VdpaType == constants.VdpaTypeVirtio { - return false, fmt.Errorf("'deviceType: %s' conflicts with 'vdpaType: virtio'; Set 'deviceType' to (string)'netdevice' Or Remove 'vdpaType'", cr.Spec.DeviceType) + if cr.Spec.DeviceType != constants.DeviceTypeNetDevice && (cr.Spec.VdpaType == constants.VdpaTypeVirtio || cr.Spec.VdpaType == constants.VdpaTypeVhost) { + return false, fmt.Errorf("'deviceType: %s' conflicts with '%s'; Set 'deviceType' to (string)'netdevice' Or Remove 'vdpaType'", cr.Spec.DeviceType, cr.Spec.VdpaType) } // vdpa: device must be configured in switchdev mode - if cr.Spec.VdpaType == constants.VdpaTypeVirtio && cr.Spec.EswitchMode != sriovnetworkv1.ESwithModeSwitchDev { - return false, fmt.Errorf("virtio/vdpa requires the device to be configured in switchdev mode") + if (cr.Spec.VdpaType == constants.VdpaTypeVirtio || cr.Spec.VdpaType == constants.VdpaTypeVhost) && cr.Spec.EswitchMode != sriovnetworkv1.ESwithModeSwitchDev { + return false, fmt.Errorf("vdpa requires the device to be configured in switchdev mode") } return true, nil } @@ -286,8 +286,8 @@ func validatePolicyForNodeState(policy *sriovnetworkv1.SriovNetworkNodePolicy, s return fmt.Errorf("numVfs(%d) in CR %s exceed the maximum allowed value(%d)", policy.Spec.NumVfs, policy.GetName(), MlxMaxVFs) } // vdpa: only mellanox cards are supported - if policy.Spec.VdpaType == constants.VdpaTypeVirtio && iface.Vendor != MellanoxID { - return fmt.Errorf("vendor(%s) in CR %s not supported for virtio-vdpa", iface.Vendor, policy.GetName()) + if (policy.Spec.VdpaType == constants.VdpaTypeVirtio || policy.Spec.VdpaType == constants.VdpaTypeVhost) && iface.Vendor != MellanoxID { + return fmt.Errorf("vendor(%s) in CR %s not supported for vdpa", iface.Vendor, policy.GetName()) } } } diff --git a/pkg/webhook/validate_test.go b/pkg/webhook/validate_test.go index a82796312f..9655047705 100644 --- a/pkg/webhook/validate_test.go +++ b/pkg/webhook/validate_test.go @@ -538,7 +538,7 @@ func TestStaticValidateSriovNetworkNodePolicyWithConflictIsRdmaAndDeviceType(t * g.Expect(ok).To(Equal(false)) } -func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVdpaType(t *testing.T) { +func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVirtioVdpaType(t *testing.T) { policy := &SriovNetworkNodePolicy{ Spec: SriovNetworkNodePolicySpec{ DeviceType: constants.DeviceTypeVfioPci, @@ -558,11 +558,35 @@ func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVdpaType(t } g := NewGomegaWithT(t) ok, err := staticValidateSriovNetworkNodePolicy(policy) - g.Expect(err).To(MatchError(ContainSubstring("'deviceType: vfio-pci' conflicts with 'vdpaType: virtio'"))) + g.Expect(err).To(MatchError(ContainSubstring("'deviceType: vfio-pci' conflicts with 'virtio'"))) g.Expect(ok).To(Equal(false)) } -func TestStaticValidateSriovNetworkNodePolicyVdpaMustSpecifySwitchDev(t *testing.T) { +func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVhostVdpaType(t *testing.T) { + policy := &SriovNetworkNodePolicy{ + Spec: SriovNetworkNodePolicySpec{ + DeviceType: constants.DeviceTypeVfioPci, + NicSelector: SriovNetworkNicSelector{ + Vendor: "15b3", + DeviceID: "101d", + }, + NodeSelector: map[string]string{ + "feature.node.kubernetes.io/network-sriov.capable": "true", + }, + NumVfs: 1, + Priority: 99, + ResourceName: "p0", + VdpaType: constants.VdpaTypeVhost, + EswitchMode: "switchdev", + }, + } + g := NewGomegaWithT(t) + ok, err := staticValidateSriovNetworkNodePolicy(policy) + g.Expect(err).To(MatchError(ContainSubstring("'deviceType: vfio-pci' conflicts with 'vhost'"))) + g.Expect(ok).To(Equal(false)) +} + +func TestStaticValidateSriovNetworkNodePolicyVirtioVdpaMustSpecifySwitchDev(t *testing.T) { policy := &SriovNetworkNodePolicy{ Spec: SriovNetworkNodePolicySpec{ DeviceType: "netdevice", @@ -581,11 +605,34 @@ func TestStaticValidateSriovNetworkNodePolicyVdpaMustSpecifySwitchDev(t *testing } g := NewGomegaWithT(t) ok, err := staticValidateSriovNetworkNodePolicy(policy) - g.Expect(err).To(MatchError(ContainSubstring("virtio/vdpa requires the device to be configured in switchdev mode"))) + g.Expect(err).To(MatchError(ContainSubstring("vdpa requires the device to be configured in switchdev mode"))) g.Expect(ok).To(Equal(false)) } -func TestValidatePolicyForNodeStateVdpaWithNotSupportedVendor(t *testing.T) { +func TestStaticValidateSriovNetworkNodePolicyVhostVdpaMustSpecifySwitchDev(t *testing.T) { + policy := &SriovNetworkNodePolicy{ + Spec: SriovNetworkNodePolicySpec{ + DeviceType: "netdevice", + NicSelector: SriovNetworkNicSelector{ + Vendor: "15b3", + DeviceID: "101d", + }, + NodeSelector: map[string]string{ + "feature.node.kubernetes.io/network-sriov.capable": "true", + }, + NumVfs: 1, + Priority: 99, + ResourceName: "p0", + VdpaType: constants.VdpaTypeVhost, + }, + } + g := NewGomegaWithT(t) + ok, err := staticValidateSriovNetworkNodePolicy(policy) + g.Expect(err).To(MatchError(ContainSubstring("vdpa requires the device to be configured in switchdev mode"))) + g.Expect(ok).To(Equal(false)) +} + +func TestValidatePolicyForNodeStateVirtioVdpaWithNotSupportedVendor(t *testing.T) { state := newNodeState() policy := &SriovNetworkNodePolicy{ ObjectMeta: metav1.ObjectMeta{ @@ -609,7 +656,34 @@ func TestValidatePolicyForNodeStateVdpaWithNotSupportedVendor(t *testing.T) { } g := NewGomegaWithT(t) err := validatePolicyForNodeState(policy, state, NewNode()) - g.Expect(err).To(MatchError(ContainSubstring(fmt.Sprintf("vendor(%s) in CR %s not supported for virtio-vdpa", state.Status.Interfaces[0].Vendor, policy.Name)))) + g.Expect(err).To(MatchError(ContainSubstring(fmt.Sprintf("vendor(%s) in CR %s not supported for vdpa", state.Status.Interfaces[0].Vendor, policy.Name)))) +} + +func TestValidatePolicyForNodeStateVhostVdpaWithNotSupportedVendor(t *testing.T) { + state := newNodeState() + policy := &SriovNetworkNodePolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "p1", + }, + Spec: SriovNetworkNodePolicySpec{ + DeviceType: "netdevice", + VdpaType: "vhost", + NicSelector: SriovNetworkNicSelector{ + PfNames: []string{"ens803f0"}, + RootDevices: []string{"0000:86:00.0"}, + Vendor: "8086", + }, + NodeSelector: map[string]string{ + "feature.node.kubernetes.io/network-sriov.capable": "true", + }, + NumVfs: 4, + Priority: 99, + ResourceName: "p0", + }, + } + g := NewGomegaWithT(t) + err := validatePolicyForNodeState(policy, state, NewNode()) + g.Expect(err).To(MatchError(ContainSubstring(fmt.Sprintf("vendor(%s) in CR %s not supported for vdpa", state.Status.Interfaces[0].Vendor, policy.Name)))) } func TestValidatePolicyForNodeStateWithInvalidDevice(t *testing.T) {