Skip to content

Commit

Permalink
Merge pull request #467 from lmilleri/host-vdpa
Browse files Browse the repository at this point in the history 
vhost-vdpa support
  • Loading branch information
@zeeke
zeeke authored Aug 23, 2023
2 parents 4dd75bc + 688979d commit ee0e017
Show file tree
Hide file tree
Showing 11 changed files with 539 additions and 91 deletions.
133 changes: 126 additions & 7 deletions api/v1/helper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,14 +94,37 @@ func newNodePolicy() *v1.SriovNetworkNodePolicy {
}
}

func newVdpaNodePolicy() *v1.SriovNetworkNodePolicy {
func newVirtioVdpaNodePolicy() *v1.SriovNetworkNodePolicy {
return &v1.SriovNetworkNodePolicy{
ObjectMeta: metav1.ObjectMeta{
Name: "p1",
},
Spec: v1.SriovNetworkNodePolicySpec{
DeviceType: consts.DeviceTypeNetDevice,
VdpaType: consts.VdpaTypeVirtio,
NicSelector: v1.SriovNetworkNicSelector{
PfNames: []string{"ens803f1#2-3"},
RootDevices: []string{"0000:86:00.1"},
Vendor: "8086",
},
NodeSelector: map[string]string{
"feature.node.kubernetes.io/network-sriov.capable": "true",
},
NumVfs: 4,
Priority: 99,
ResourceName: "virtiovdpa",
},
}
}

func newVhostVdpaNodePolicy() *v1.SriovNetworkNodePolicy {
return &v1.SriovNetworkNodePolicy{
ObjectMeta: metav1.ObjectMeta{
Name: "p1",
},
Spec: v1.SriovNetworkNodePolicySpec{
DeviceType: consts.DeviceTypeNetDevice,
VdpaType: consts.VdpaTypeVhost,
NicSelector: v1.SriovNetworkNicSelector{
PfNames: []string{"ens803f1"},
RootDevices: []string{"0000:86:00.1"},
Expand All @@ -112,7 +135,7 @@ func newVdpaNodePolicy() *v1.SriovNetworkNodePolicy {
},
NumVfs: 2,
Priority: 99,
ResourceName: "p1res",
ResourceName: "vhostvdpa",
},
}
}
Expand Down Expand Up @@ -437,6 +460,55 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
},
},
},
{
// vdpa policy with same priority (both virtio and vhost), VfRange's do not overlap so all is merged
tname: "one vdpa policy present same pf same priority partitioning",
currentState: func() *v1.SriovNetworkNodeState {
st := newNodeState()
st.Spec.Interfaces = []v1.Interface{
{
Name: "ens803f1",
NumVfs: 4,
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: consts.DeviceTypeNetDevice,
VdpaType: consts.VdpaTypeVhost,
ResourceName: "vhostvdpa",
VfRange: "0-1",
PolicyName: "p2",
},
},
},
}
return st
}(),
policy: newVirtioVdpaNodePolicy(),
equalP: true,
expectedInterfaces: []v1.Interface{
{
Name: "ens803f1",
NumVfs: 4,
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: consts.DeviceTypeNetDevice,
VdpaType: consts.VdpaTypeVirtio,
ResourceName: "virtiovdpa",
VfRange: "2-3",
PolicyName: "p1",
},
{
DeviceType: consts.DeviceTypeNetDevice,
VdpaType: consts.VdpaTypeVhost,
ResourceName: "vhostvdpa",
VfRange: "0-1",
PolicyName: "p2",
},
},
},
},
},
{
// policy with same priority that overwrites the 2 present groups in
// SriovNetworkNodeState because they overlap VfRange
Expand Down Expand Up @@ -637,7 +709,7 @@ func TestSriovNetworkNodePolicyApply(t *testing.T) {
}
}

func TestVdpaNodePolicyApply(t *testing.T) {
func TestVirtioVdpaNodePolicyApply(t *testing.T) {
testtable := []struct {
tname string
currentState *v1.SriovNetworkNodeState
Expand All @@ -647,20 +719,67 @@ func TestVdpaNodePolicyApply(t *testing.T) {
expectedErr bool
}{
{
tname: "vdpa configuration",
tname: "virtio/vdpa configuration",
currentState: newNodeState(),
policy: newVdpaNodePolicy(),
policy: newVirtioVdpaNodePolicy(),
equalP: false,
expectedInterfaces: []v1.Interface{
{
Name: "ens803f1",
NumVfs: 2,
NumVfs: 4,
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: consts.DeviceTypeNetDevice,
VdpaType: consts.VdpaTypeVirtio,
ResourceName: "p1res",
ResourceName: "virtiovdpa",
VfRange: "2-3",
PolicyName: "p1",
},
},
},
},
},
}
for _, tc := range testtable {
t.Run(tc.tname, func(t *testing.T) {
err := tc.policy.Apply(tc.currentState, tc.equalP)
if tc.expectedErr && err == nil {
t.Errorf("Apply expecting error.")
} else if !tc.expectedErr && err != nil {
t.Errorf("Apply error:\n%s", err)
}
if diff := cmp.Diff(tc.expectedInterfaces, tc.currentState.Spec.Interfaces); diff != "" {
t.Errorf("SriovNetworkNodeState spec diff (-want +got):\n%s", diff)
}
})
}
}

func TestVhostVdpaNodePolicyApply(t *testing.T) {
testtable := []struct {
tname string
currentState *v1.SriovNetworkNodeState
policy *v1.SriovNetworkNodePolicy
expectedInterfaces v1.Interfaces
equalP bool
expectedErr bool
}{
{
tname: "vhost/vdpa configuration",
currentState: newNodeState(),
policy: newVhostVdpaNodePolicy(),
equalP: false,
expectedInterfaces: []v1.Interface{
{
Name: "ens803f1",
NumVfs: 2,
PciAddress: "0000:86:00.1",
VfGroups: []v1.VfGroup{
{
DeviceType: consts.DeviceTypeNetDevice,
VdpaType: consts.VdpaTypeVhost,
ResourceName: "vhostvdpa",
VfRange: "0-1",
PolicyName: "p1",
},
Expand Down
4 changes: 2 additions & 2 deletions api/v1/sriovnetworknodepolicy_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ type SriovNetworkNodePolicySpec struct {
// +kubebuilder:validation:Enum=legacy;switchdev
// NIC Device Mode. Allowed value "legacy","switchdev".
EswitchMode string `json:"eSwitchMode,omitempty"`
// +kubebuilder:validation:Enum=virtio
// VDPA device type. Allowed value "virtio"
// +kubebuilder:validation:Enum=virtio;vhost
// VDPA device type. Allowed value "virtio", "vhost"
VdpaType string `json:"vdpaType,omitempty"`
// Exclude device's NUMA node when advertising this resource by SRIOV network device plugin. Default to false.
ExcludeTopology bool `json:"excludeTopology,omitempty"`
Expand Down
12 changes: 10 additions & 2 deletions bindata/manifests/switchdev-config/files/switchdev-configuration-after-nm.sh.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,17 @@ contents:
do
extract_min_max_ids
vdpaType=$(jq -c '.vdpaType' -r <<< $group)
if [ $vfid -le $maxId ] && [ $vfid -ge $minId ] && [ $vdpaType == "virtio" ]; then
vdpa_cmd="vdpa dev add name vdpa:"${VfPciAddr}" mgmtdev pci/"${VfPciAddr}
if [ $vfid -le $maxId ] && [ $vfid -ge $minId ] && ([ $vdpaType == "virtio" ] || [ $vdpaType == "vhost" ]); then
vdpa_name=vdpa:${VfPciAddr}
vdpa_cmd="vdpa dev add name "${vdpa_name}" mgmtdev pci/"${VfPciAddr}" max_vqp 32"
eval $vdpa_cmd
# set the driver_override. When the specified driver will be loaded in the kernel
# it will be automatically bound to the vdpa device
driver_override=virtio_vdpa
if [ $vdpaType == "vhost" ]; then
driver_override=vhost_vdpa
fi
echo $driver_override > /sys/bus/vdpa/devices/$vdpa_name/driver_override
fi
done
done
Expand Down
3 changes: 2 additions & 1 deletion config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,10 @@ spec:
description: SRIOV Network device plugin endpoint resource name
type: string
vdpaType:
description: VDPA device type. Allowed value "virtio"
description: VDPA device type. Allowed value "virtio", "vhost"
enum:
- virtio
- vhost
type: string
required:
- nicSelector
Expand Down
21 changes: 20 additions & 1 deletion controllers/sriovnetworknodepolicy_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ func TestRenderDevicePluginConfigData(t *testing.T) {
expResource dptypes.ResourceConfList
}{
{
tname: "testVdpaVirtio",
tname: "testVirtioVdpaVirtio",
policy: sriovnetworkv1.SriovNetworkNodePolicy{
Spec: v1.SriovNetworkNodePolicySpec{
ResourceName: "resourceName",
Expand All @@ -170,6 +170,25 @@ func TestRenderDevicePluginConfigData(t *testing.T) {
},
},
},
}, {
tname: "testVhostVdpaVirtio",
policy: sriovnetworkv1.SriovNetworkNodePolicy{
Spec: v1.SriovNetworkNodePolicySpec{
ResourceName: "resourceName",
DeviceType: consts.DeviceTypeNetDevice,
VdpaType: consts.VdpaTypeVhost,
},
},
expResource: dptypes.ResourceConfList{
ResourceList: []dptypes.ResourceConfig{
{
ResourceName: "resourceName",
Selectors: mustMarshallSelector(t, &dptypes.NetDeviceSelectors{
VdpaType: dptypes.VdpaType(consts.VdpaTypeVhost),
}),
},
},
},
},
{
tname: "testExcludeTopology",
Expand Down
3 changes: 2 additions & 1 deletion ...yment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,10 @@ spec:
description: SRIOV Network device plugin endpoint resource name
type: string
vdpaType:
description: VDPA device type. Allowed value "virtio"
description: VDPA device type. Allowed value "virtio", "vhost"
enum:
- virtio
- vhost
type: string
required:
- nicSelector
Expand Down
1 change: 1 addition & 0 deletions pkg/consts/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,5 @@ const (
DeviceTypeVfioPci = "vfio-pci"
DeviceTypeNetDevice = "netdevice"
VdpaTypeVirtio = "virtio"
VdpaTypeVhost = "vhost"
)
Loading

0 comments on commit ee0e017

Please sign in to comment.