diff --git a/api/v1/sriovnetworknodepolicy_types.go b/api/v1/sriovnetworknodepolicy_types.go index 8955c300da..17e22d2fad 100644 --- a/api/v1/sriovnetworknodepolicy_types.go +++ b/api/v1/sriovnetworknodepolicy_types.go @@ -54,8 +54,8 @@ type SriovNetworkNodePolicySpec struct { // +kubebuilder:validation:Enum=legacy;switchdev // NIC Device Mode. Allowed value "legacy","switchdev". EswitchMode string `json:"eSwitchMode,omitempty"` - // +kubebuilder:validation:Enum=virtio - // VDPA device type. Allowed value "virtio" + // +kubebuilder:validation:Enum=virtio;vhost + // VDPA device type. Allowed value "virtio", "vhost" VdpaType string `json:"vdpaType,omitempty"` // Exclude device's NUMA node when advertising this resource by SRIOV network device plugin. Default to false. ExcludeTopology bool `json:"excludeTopology,omitempty"` diff --git a/bindata/manifests/switchdev-config/files/switchdev-configuration-after-nm.sh.yaml b/bindata/manifests/switchdev-config/files/switchdev-configuration-after-nm.sh.yaml index 8d705a3659..af15dfb89e 100644 --- a/bindata/manifests/switchdev-config/files/switchdev-configuration-after-nm.sh.yaml +++ b/bindata/manifests/switchdev-config/files/switchdev-configuration-after-nm.sh.yaml @@ -58,7 +58,7 @@ contents: do extract_min_max_ids vdpaType=$(jq -c '.vdpaType' -r <<< $group) - if [ $vfid -le $maxId ] && [ $vfid -ge $minId ] && [ $vdpaType == "virtio" ]; then + if [ $vfid -le $maxId ] && [ $vfid -ge $minId ] && [ $vdpaType != "" ]; then vdpa_cmd="vdpa dev add name vdpa:"${VfPciAddr}" mgmtdev pci/"${VfPciAddr} eval $vdpa_cmd fi diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml index 5bebf01e05..a81c6a2276 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml @@ -117,9 +117,10 @@ spec: description: SRIOV Network device plugin endpoint resource name type: string vdpaType: - description: VDPA device type. Allowed value "virtio" + description: VDPA device type. Allowed value "virtio", "vhost" enum: - virtio + - vhost type: string required: - nicSelector diff --git a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml index 5bebf01e05..a81c6a2276 100644 --- a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml +++ b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml @@ -117,9 +117,10 @@ spec: description: SRIOV Network device plugin endpoint resource name type: string vdpaType: - description: VDPA device type. Allowed value "virtio" + description: VDPA device type. Allowed value "virtio", "vhost" enum: - virtio + - vhost type: string required: - nicSelector diff --git a/pkg/consts/constants.go b/pkg/consts/constants.go index 72e847d820..1f37edfa42 100644 --- a/pkg/consts/constants.go +++ b/pkg/consts/constants.go @@ -32,4 +32,5 @@ const ( DeviceTypeVfioPci = "vfio-pci" DeviceTypeNetDevice = "netdevice" VdpaTypeVirtio = "virtio" + VdpaTypeVhost = "vhost" ) diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go index c5b8c35963..49ee491db8 100644 --- a/pkg/plugins/generic/generic_plugin.go +++ b/pkg/plugins/generic/generic_plugin.go @@ -28,6 +28,7 @@ type GenericPlugin struct { LoadVirtioVdpaDriver uint RunningOnHost bool HostManager host.HostManagerInterface + LoadVhostVdpaDriver uint } const scriptsPath = "bindata/scripts/enable-kargs.sh" @@ -47,6 +48,7 @@ func NewGenericPlugin(runningOnHost bool) (plugin.VendorPlugin, error) { LoadVirtioVdpaDriver: unloaded, RunningOnHost: runningOnHost, HostManager: host.NewHostManager(runningOnHost), + LoadVhostVdpaDriver: unloaded, }, nil } @@ -69,7 +71,7 @@ func (p *GenericPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeSt p.DesireState = new needDrain = needDrainNode(new.Spec.Interfaces, new.Status.Interfaces) - needReboot = needRebootNode(new, &p.LoadVfioDriver, &p.LoadVirtioVdpaDriver) + needReboot = needRebootNode(new, &p.LoadVfioDriver, &p.LoadVirtioVdpaDriver, &p.LoadVhostVdpaDriver) if needReboot { needDrain = true @@ -96,6 +98,14 @@ func (p *GenericPlugin) Apply() error { p.LoadVirtioVdpaDriver = loaded } + if p.LoadVhostVdpaDriver == loading { + if err := p.HostManager.LoadKernelModule("vhost_vdpa"); err != nil { + glog.Errorf("generic-plugin Apply(): fail to load vhost_vdpa kmod: %v", err) + return err + } + p.LoadVhostVdpaDriver = loaded + } + if p.LastState != nil { glog.Infof("generic-plugin Apply(): lastStat=%v", p.LastState.Spec) if reflect.DeepEqual(p.LastState.Spec.Interfaces, p.DesireState.Spec.Interfaces) { @@ -151,6 +161,17 @@ func needVirtioVdpaDriver(state *sriovnetworkv1.SriovNetworkNodeState) bool { return false } +func needVhostVdpaDriver(state *sriovnetworkv1.SriovNetworkNodeState) bool { + for _, iface := range state.Spec.Interfaces { + for i := range iface.VfGroups { + if iface.VfGroups[i].VdpaType == constants.VdpaTypeVhost { + return true + } + } + } + return false +} + func tryEnableIommuInKernelArgs() (bool, error) { glog.Info("generic-plugin tryEnableIommuInKernelArgs()") args := [2]string{"intel_iommu=on", "iommu=pt"} @@ -215,7 +236,8 @@ func needDrainNode(desired sriovnetworkv1.Interfaces, current sriovnetworkv1.Int return } -func needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState, loadVfioDriver *uint, loadVirtioVdpaDriver *uint) (needReboot bool) { +func needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState, loadVfioDriver *uint, loadVirtioVdpaDriver *uint, + loadVhostVdpaDriver *uint) (needReboot bool) { needReboot = false if *loadVfioDriver != loaded { if needVfioDriver(state) { @@ -237,6 +259,12 @@ func needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState, loadVfioDriver } } + if *loadVhostVdpaDriver != loaded { + if needVhostVdpaDriver(state) { + *loadVhostVdpaDriver = loading + } + } + update, err := utils.WriteSwitchdevConfFile(state) if err != nil { glog.Errorf("generic-plugin needRebootNode(): fail to write switchdev device config file") diff --git a/pkg/webhook/validate.go b/pkg/webhook/validate.go index 91b3922fa4..571d93d37e 100644 --- a/pkg/webhook/validate.go +++ b/pkg/webhook/validate.go @@ -203,12 +203,12 @@ func staticValidateSriovNetworkNodePolicy(cr *sriovnetworkv1.SriovNetworkNodePol } // vdpa: deviceType must be set to 'netdevice' - if cr.Spec.DeviceType != constants.DeviceTypeNetDevice && cr.Spec.VdpaType == constants.VdpaTypeVirtio { - return false, fmt.Errorf("'deviceType: %s' conflicts with 'vdpaType: virtio'; Set 'deviceType' to (string)'netdevice' Or Remove 'vdpaType'", cr.Spec.DeviceType) + if cr.Spec.DeviceType != constants.DeviceTypeNetDevice && (cr.Spec.VdpaType == constants.VdpaTypeVirtio || cr.Spec.VdpaType == constants.VdpaTypeVhost) { + return false, fmt.Errorf("'deviceType: %s' conflicts with '%s'; Set 'deviceType' to (string)'netdevice' Or Remove 'vdpaType'", cr.Spec.DeviceType, cr.Spec.VdpaType) } // vdpa: device must be configured in switchdev mode - if cr.Spec.VdpaType == constants.VdpaTypeVirtio && cr.Spec.EswitchMode != sriovnetworkv1.ESwithModeSwitchDev { - return false, fmt.Errorf("virtio/vdpa requires the device to be configured in switchdev mode") + if (cr.Spec.VdpaType == constants.VdpaTypeVirtio || cr.Spec.VdpaType == constants.VdpaTypeVhost) && cr.Spec.EswitchMode != sriovnetworkv1.ESwithModeSwitchDev { + return false, fmt.Errorf("vdpa requires the device to be configured in switchdev mode") } return true, nil } @@ -286,8 +286,8 @@ func validatePolicyForNodeState(policy *sriovnetworkv1.SriovNetworkNodePolicy, s return fmt.Errorf("numVfs(%d) in CR %s exceed the maximum allowed value(%d)", policy.Spec.NumVfs, policy.GetName(), MlxMaxVFs) } // vdpa: only mellanox cards are supported - if policy.Spec.VdpaType == constants.VdpaTypeVirtio && iface.Vendor != MellanoxID { - return fmt.Errorf("vendor(%s) in CR %s not supported for virtio-vdpa", iface.Vendor, policy.GetName()) + if (policy.Spec.VdpaType == constants.VdpaTypeVirtio || policy.Spec.VdpaType == constants.VdpaTypeVhost) && iface.Vendor != MellanoxID { + return fmt.Errorf("vendor(%s) in CR %s not supported for vdpa", iface.Vendor, policy.GetName()) } } } diff --git a/pkg/webhook/validate_test.go b/pkg/webhook/validate_test.go index a82796312f..9655047705 100644 --- a/pkg/webhook/validate_test.go +++ b/pkg/webhook/validate_test.go @@ -538,7 +538,7 @@ func TestStaticValidateSriovNetworkNodePolicyWithConflictIsRdmaAndDeviceType(t * g.Expect(ok).To(Equal(false)) } -func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVdpaType(t *testing.T) { +func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVirtioVdpaType(t *testing.T) { policy := &SriovNetworkNodePolicy{ Spec: SriovNetworkNodePolicySpec{ DeviceType: constants.DeviceTypeVfioPci, @@ -558,11 +558,35 @@ func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVdpaType(t } g := NewGomegaWithT(t) ok, err := staticValidateSriovNetworkNodePolicy(policy) - g.Expect(err).To(MatchError(ContainSubstring("'deviceType: vfio-pci' conflicts with 'vdpaType: virtio'"))) + g.Expect(err).To(MatchError(ContainSubstring("'deviceType: vfio-pci' conflicts with 'virtio'"))) g.Expect(ok).To(Equal(false)) } -func TestStaticValidateSriovNetworkNodePolicyVdpaMustSpecifySwitchDev(t *testing.T) { +func TestStaticValidateSriovNetworkNodePolicyWithConflictDeviceTypeAndVhostVdpaType(t *testing.T) { + policy := &SriovNetworkNodePolicy{ + Spec: SriovNetworkNodePolicySpec{ + DeviceType: constants.DeviceTypeVfioPci, + NicSelector: SriovNetworkNicSelector{ + Vendor: "15b3", + DeviceID: "101d", + }, + NodeSelector: map[string]string{ + "feature.node.kubernetes.io/network-sriov.capable": "true", + }, + NumVfs: 1, + Priority: 99, + ResourceName: "p0", + VdpaType: constants.VdpaTypeVhost, + EswitchMode: "switchdev", + }, + } + g := NewGomegaWithT(t) + ok, err := staticValidateSriovNetworkNodePolicy(policy) + g.Expect(err).To(MatchError(ContainSubstring("'deviceType: vfio-pci' conflicts with 'vhost'"))) + g.Expect(ok).To(Equal(false)) +} + +func TestStaticValidateSriovNetworkNodePolicyVirtioVdpaMustSpecifySwitchDev(t *testing.T) { policy := &SriovNetworkNodePolicy{ Spec: SriovNetworkNodePolicySpec{ DeviceType: "netdevice", @@ -581,11 +605,34 @@ func TestStaticValidateSriovNetworkNodePolicyVdpaMustSpecifySwitchDev(t *testing } g := NewGomegaWithT(t) ok, err := staticValidateSriovNetworkNodePolicy(policy) - g.Expect(err).To(MatchError(ContainSubstring("virtio/vdpa requires the device to be configured in switchdev mode"))) + g.Expect(err).To(MatchError(ContainSubstring("vdpa requires the device to be configured in switchdev mode"))) g.Expect(ok).To(Equal(false)) } -func TestValidatePolicyForNodeStateVdpaWithNotSupportedVendor(t *testing.T) { +func TestStaticValidateSriovNetworkNodePolicyVhostVdpaMustSpecifySwitchDev(t *testing.T) { + policy := &SriovNetworkNodePolicy{ + Spec: SriovNetworkNodePolicySpec{ + DeviceType: "netdevice", + NicSelector: SriovNetworkNicSelector{ + Vendor: "15b3", + DeviceID: "101d", + }, + NodeSelector: map[string]string{ + "feature.node.kubernetes.io/network-sriov.capable": "true", + }, + NumVfs: 1, + Priority: 99, + ResourceName: "p0", + VdpaType: constants.VdpaTypeVhost, + }, + } + g := NewGomegaWithT(t) + ok, err := staticValidateSriovNetworkNodePolicy(policy) + g.Expect(err).To(MatchError(ContainSubstring("vdpa requires the device to be configured in switchdev mode"))) + g.Expect(ok).To(Equal(false)) +} + +func TestValidatePolicyForNodeStateVirtioVdpaWithNotSupportedVendor(t *testing.T) { state := newNodeState() policy := &SriovNetworkNodePolicy{ ObjectMeta: metav1.ObjectMeta{ @@ -609,7 +656,34 @@ func TestValidatePolicyForNodeStateVdpaWithNotSupportedVendor(t *testing.T) { } g := NewGomegaWithT(t) err := validatePolicyForNodeState(policy, state, NewNode()) - g.Expect(err).To(MatchError(ContainSubstring(fmt.Sprintf("vendor(%s) in CR %s not supported for virtio-vdpa", state.Status.Interfaces[0].Vendor, policy.Name)))) + g.Expect(err).To(MatchError(ContainSubstring(fmt.Sprintf("vendor(%s) in CR %s not supported for vdpa", state.Status.Interfaces[0].Vendor, policy.Name)))) +} + +func TestValidatePolicyForNodeStateVhostVdpaWithNotSupportedVendor(t *testing.T) { + state := newNodeState() + policy := &SriovNetworkNodePolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "p1", + }, + Spec: SriovNetworkNodePolicySpec{ + DeviceType: "netdevice", + VdpaType: "vhost", + NicSelector: SriovNetworkNicSelector{ + PfNames: []string{"ens803f0"}, + RootDevices: []string{"0000:86:00.0"}, + Vendor: "8086", + }, + NodeSelector: map[string]string{ + "feature.node.kubernetes.io/network-sriov.capable": "true", + }, + NumVfs: 4, + Priority: 99, + ResourceName: "p0", + }, + } + g := NewGomegaWithT(t) + err := validatePolicyForNodeState(policy, state, NewNode()) + g.Expect(err).To(MatchError(ContainSubstring(fmt.Sprintf("vendor(%s) in CR %s not supported for vdpa", state.Status.Interfaces[0].Vendor, policy.Name)))) } func TestValidatePolicyForNodeStateWithInvalidDevice(t *testing.T) {