diff --git a/go.mod b/go.mod index c1aaf85ba..2ca75b2d9 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/hashicorp/go-retryablehttp v0.7.0 github.com/jaypipes/ghw v0.9.0 + github.com/k8snetworkplumbingwg/govdpa v0.1.4 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 github.com/k8snetworkplumbingwg/sriov-network-device-plugin v0.0.0-20221127172732-a5a7395122e3 github.com/onsi/ginkgo/v2 v2.11.0 @@ -91,7 +92,6 @@ require ( github.com/jaypipes/pcidb v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/k8snetworkplumbingwg/govdpa v0.1.4 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.17 // indirect diff --git a/pkg/consts/constants.go b/pkg/consts/constants.go index e7255368d..d3ddd3f6d 100644 --- a/pkg/consts/constants.go +++ b/pkg/consts/constants.go @@ -92,7 +92,18 @@ const ( UdevRulesFolder = UdevFolder + "/rules.d" HostUdevRulesFolder = Host + UdevRulesFolder UdevDisableNM = "/bindata/scripts/udev-find-sriov-pf.sh" - NMUdevRule = "SUBSYSTEM==\"net\", ACTION==\"add|change|move\", ATTRS{device}==\"%s\", IMPORT{program}=\"/etc/udev/disable-nm-sriov.sh $env{INTERFACE} %s\"" + // nolint:goconst + NMUdevRule = `SUBSYSTEM=="net", ` + + `ACTION=="add|change|move", ` + + `ATTRS{device}=="%s", ` + + `IMPORT{program}="/etc/udev/disable-nm-sriov.sh $env{INTERFACE} %s"` + // nolint:goconst + SwitchdevUdevRule = `SUBSYSTEM=="net", ` + + `ACTION=="add|move", ` + + `ATTRS{phys_switch_id}=="%s", ` + + `ATTR{phys_port_name}=="pf%svf*", ` + + `IMPORT{program}="/etc/udev/switchdev-vf-link-name.sh $attr{phys_port_name}", ` + + `NAME="%s_$env{NUMBER}"` KernelArgPciRealloc = "pci=realloc" KernelArgIntelIommu = "intel_iommu=on" diff --git a/pkg/helper/mock/mock_helper.go b/pkg/helper/mock/mock_helper.go index b96b0d4de..a8e417e49 100644 --- a/pkg/helper/mock/mock_helper.go +++ b/pkg/helper/mock/mock_helper.go @@ -54,6 +54,20 @@ func (mr *MockHostHelpersInterfaceMockRecorder) AddUdevRule(pfPciAddress interfa return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddUdevRule", reflect.TypeOf((*MockHostHelpersInterface)(nil).AddUdevRule), pfPciAddress) } +// AddVfRepresentorUdevRule mocks base method. +func (m *MockHostHelpersInterface) AddVfRepresentorUdevRule(pfPciAddress, pfName, pfSwitchID, pfSwitchPort string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddVfRepresentorUdevRule", pfPciAddress, pfName, pfSwitchID, pfSwitchPort) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddVfRepresentorUdevRule indicates an expected call of AddVfRepresentorUdevRule. +func (mr *MockHostHelpersInterfaceMockRecorder) AddVfRepresentorUdevRule(pfPciAddress, pfName, pfSwitchID, pfSwitchPort interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddVfRepresentorUdevRule", reflect.TypeOf((*MockHostHelpersInterface)(nil).AddVfRepresentorUdevRule), pfPciAddress, pfName, pfSwitchID, pfSwitchPort) +} + // BindDefaultDriver mocks base method. func (m *MockHostHelpersInterface) BindDefaultDriver(pciAddr string) error { m.ctrl.T.Helper() @@ -182,6 +196,34 @@ func (mr *MockHostHelpersInterfaceMockRecorder) ConfigSriovInterfaces(storeManag return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigSriovInterfaces", reflect.TypeOf((*MockHostHelpersInterface)(nil).ConfigSriovInterfaces), storeManager, interfaces, ifaceStatuses, pfsToConfig) } +// CreateVDPADevice mocks base method. +func (m *MockHostHelpersInterface) CreateVDPADevice(pciAddr, vdpaType string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateVDPADevice", pciAddr, vdpaType) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateVDPADevice indicates an expected call of CreateVDPADevice. +func (mr *MockHostHelpersInterfaceMockRecorder) CreateVDPADevice(pciAddr, vdpaType interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateVDPADevice", reflect.TypeOf((*MockHostHelpersInterface)(nil).CreateVDPADevice), pciAddr, vdpaType) +} + +// DeleteVDPADevice mocks base method. +func (m *MockHostHelpersInterface) DeleteVDPADevice(pciAddr string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteVDPADevice", pciAddr) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteVDPADevice indicates an expected call of DeleteVDPADevice. +func (mr *MockHostHelpersInterfaceMockRecorder) DeleteVDPADevice(pciAddr interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteVDPADevice", reflect.TypeOf((*MockHostHelpersInterface)(nil).DeleteVDPADevice), pciAddr) +} + // DiscoverSriovDevices mocks base method. func (m *MockHostHelpersInterface) DiscoverSriovDevices(storeManager store.ManagerInterface) ([]v1.InterfaceExt, error) { m.ctrl.T.Helper() @@ -197,6 +239,20 @@ func (mr *MockHostHelpersInterfaceMockRecorder) DiscoverSriovDevices(storeManage return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverSriovDevices", reflect.TypeOf((*MockHostHelpersInterface)(nil).DiscoverSriovDevices), storeManager) } +// DiscoverVDPAType mocks base method. +func (m *MockHostHelpersInterface) DiscoverVDPAType(pciAddr string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiscoverVDPAType", pciAddr) + ret0, _ := ret[0].(string) + return ret0 +} + +// DiscoverVDPAType indicates an expected call of DiscoverVDPAType. +func (mr *MockHostHelpersInterfaceMockRecorder) DiscoverVDPAType(pciAddr interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverVDPAType", reflect.TypeOf((*MockHostHelpersInterface)(nil).DiscoverVDPAType), pciAddr) +} + // EnableRDMA mocks base method. func (m *MockHostHelpersInterface) EnableRDMA(conditionFilePath, serviceName, packageManager string) (bool, error) { m.ctrl.T.Helper() @@ -809,6 +865,20 @@ func (mr *MockHostHelpersInterfaceMockRecorder) RemoveUdevRule(pfPciAddress inte return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveUdevRule", reflect.TypeOf((*MockHostHelpersInterface)(nil).RemoveUdevRule), pfPciAddress) } +// RemoveVfRepresentorUdevRule mocks base method. +func (m *MockHostHelpersInterface) RemoveVfRepresentorUdevRule(pfPciAddress string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveVfRepresentorUdevRule", pfPciAddress) + ret0, _ := ret[0].(error) + return ret0 +} + +// RemoveVfRepresentorUdevRule indicates an expected call of RemoveVfRepresentorUdevRule. +func (mr *MockHostHelpersInterfaceMockRecorder) RemoveVfRepresentorUdevRule(pfPciAddress interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveVfRepresentorUdevRule", reflect.TypeOf((*MockHostHelpersInterface)(nil).RemoveVfRepresentorUdevRule), pfPciAddress) +} + // ResetSriovDevice mocks base method. func (m *MockHostHelpersInterface) ResetSriovDevice(ifaceStatus v1.InterfaceExt) error { m.ctrl.T.Helper() @@ -872,6 +942,20 @@ func (mr *MockHostHelpersInterfaceMockRecorder) SetNetdevMTU(pciAddr, mtu interf return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetNetdevMTU", reflect.TypeOf((*MockHostHelpersInterface)(nil).SetNetdevMTU), pciAddr, mtu) } +// SetNicSriovMode mocks base method. +func (m *MockHostHelpersInterface) SetNicSriovMode(pciAddr, mode string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetNicSriovMode", pciAddr, mode) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetNicSriovMode indicates an expected call of SetNicSriovMode. +func (mr *MockHostHelpersInterfaceMockRecorder) SetNicSriovMode(pciAddr, mode interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetNicSriovMode", reflect.TypeOf((*MockHostHelpersInterface)(nil).SetNicSriovMode), pciAddr, mode) +} + // SetSriovNumVfs mocks base method. func (m *MockHostHelpersInterface) SetSriovNumVfs(pciAddr string, numVfs int) error { m.ctrl.T.Helper() diff --git a/pkg/host/internal/kernel/kernel_test.go b/pkg/host/internal/kernel/kernel_test.go index de6f22cec..b8eef8b07 100644 --- a/pkg/host/internal/kernel/kernel_test.go +++ b/pkg/host/internal/kernel/kernel_test.go @@ -1,38 +1,20 @@ package kernel import ( - "os" - "path/filepath" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/helpers" ) -func assertFileContentsEquals(path, expectedContent string) { - d, err := os.ReadFile(filepath.Join(vars.FilesystemRoot, path)) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - ExpectWithOffset(1, string(d)).To(Equal(expectedContent)) -} - var _ = Describe("Kernel", func() { Context("Drivers", func() { var ( k types.KernelInterface ) - configureFS := func(f *fakefilesystem.FS) { - var ( - cleanFakeFs func() - err error - ) - vars.FilesystemRoot, cleanFakeFs, err = f.Use() - Expect(err).ToNot(HaveOccurred()) - DeferCleanup(cleanFakeFs) - } BeforeEach(func() { k = New(nil) }) @@ -41,11 +23,11 @@ var _ = Describe("Kernel", func() { Expect(k.UnbindDriverByBusAndDevice(consts.BusPci, "unknown-dev")).NotTo(HaveOccurred()) }) It("known device, no driver", func() { - configureFS(&fakefilesystem.FS{Dirs: []string{"/sys/bus/pci/devices/0000:d8:00.0"}}) + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{Dirs: []string{"/sys/bus/pci/devices/0000:d8:00.0"}}) Expect(k.Unbind("0000:d8:00.0")).NotTo(HaveOccurred()) }) It("has driver, succeed", func() { - configureFS(&fakefilesystem.FS{ + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{ "/sys/bus/pci/devices/0000:d8:00.0", "/sys/bus/pci/drivers/test-driver"}, @@ -56,10 +38,10 @@ var _ = Describe("Kernel", func() { }) Expect(k.Unbind("0000:d8:00.0")).NotTo(HaveOccurred()) // check that echo to unbind path was done - assertFileContentsEquals("/sys/bus/pci/drivers/test-driver/unbind", "0000:d8:00.0") + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/drivers/test-driver/unbind", "0000:d8:00.0") }) It("has driver, failed to unbind", func() { - configureFS(&fakefilesystem.FS{ + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{ "/sys/bus/pci/devices/0000:d8:00.0"}, Symlinks: map[string]string{ @@ -75,13 +57,13 @@ var _ = Describe("Kernel", func() { Expect(driver).To(BeEmpty()) }) It("known device, no driver", func() { - configureFS(&fakefilesystem.FS{Dirs: []string{"/sys/bus/pci/devices/0000:d8:00.0"}}) + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{Dirs: []string{"/sys/bus/pci/devices/0000:d8:00.0"}}) has, driver := k.HasDriver("0000:d8:00.0") Expect(has).To(BeFalse()) Expect(driver).To(BeEmpty()) }) It("has driver", func() { - configureFS(&fakefilesystem.FS{ + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{ "/sys/bus/pci/devices/0000:d8:00.0", "/sys/bus/pci/drivers/test-driver"}, @@ -98,7 +80,7 @@ var _ = Describe("Kernel", func() { Expect(k.BindDefaultDriver("unknown-dev")).To(HaveOccurred()) }) It("no driver", func() { - configureFS(&fakefilesystem.FS{ + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{ "/sys/bus/pci/devices/0000:d8:00.0"}, Files: map[string][]byte{ @@ -106,10 +88,10 @@ var _ = Describe("Kernel", func() { }) Expect(k.BindDefaultDriver("0000:d8:00.0")).NotTo(HaveOccurred()) // should probe driver for dev - assertFileContentsEquals("/sys/bus/pci/drivers_probe", "0000:d8:00.0") + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/drivers_probe", "0000:d8:00.0") }) It("already bind to default driver", func() { - configureFS(&fakefilesystem.FS{ + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{ "/sys/bus/pci/devices/0000:d8:00.0"}, Symlinks: map[string]string{ @@ -118,7 +100,7 @@ var _ = Describe("Kernel", func() { Expect(k.BindDefaultDriver("0000:d8:00.0")).NotTo(HaveOccurred()) }) It("bind to dpdk driver", func() { - configureFS(&fakefilesystem.FS{ + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{ "/sys/bus/pci/devices/0000:d8:00.0", "/sys/bus/pci/drivers/vfio-pci"}, @@ -130,9 +112,9 @@ var _ = Describe("Kernel", func() { }) Expect(k.BindDefaultDriver("0000:d8:00.0")).NotTo(HaveOccurred()) // should unbind from dpdk driver - assertFileContentsEquals("/sys/bus/pci/drivers/vfio-pci/unbind", "0000:d8:00.0") + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/drivers/vfio-pci/unbind", "0000:d8:00.0") // should probe driver for dev - assertFileContentsEquals("/sys/bus/pci/drivers_probe", "0000:d8:00.0") + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/drivers_probe", "0000:d8:00.0") }) }) Context("BindDpdkDriver", func() { @@ -140,7 +122,7 @@ var _ = Describe("Kernel", func() { Expect(k.BindDpdkDriver("unknown-dev", "vfio-pci")).To(HaveOccurred()) }) It("no driver", func() { - configureFS(&fakefilesystem.FS{ + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{ "/sys/bus/pci/devices/0000:d8:00.0", "/sys/bus/pci/drivers/vfio-pci"}, @@ -149,10 +131,10 @@ var _ = Describe("Kernel", func() { }) Expect(k.BindDpdkDriver("0000:d8:00.0", "vfio-pci")).NotTo(HaveOccurred()) // should reset driver override - assertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/driver_override", "\x00") + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/driver_override", "\x00") }) It("already bind to required driver", func() { - configureFS(&fakefilesystem.FS{ + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{ "/sys/bus/pci/devices/0000:d8:00.0"}, Symlinks: map[string]string{ @@ -161,7 +143,7 @@ var _ = Describe("Kernel", func() { Expect(k.BindDpdkDriver("0000:d8:00.0", "vfio-pci")).NotTo(HaveOccurred()) }) It("bind to wrong driver", func() { - configureFS(&fakefilesystem.FS{ + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{ "/sys/bus/pci/devices/0000:d8:00.0", "/sys/bus/pci/drivers/test-driver", @@ -175,12 +157,12 @@ var _ = Describe("Kernel", func() { }) Expect(k.BindDpdkDriver("0000:d8:00.0", "vfio-pci")).NotTo(HaveOccurred()) // should unbind from driver1 - assertFileContentsEquals("/sys/bus/pci/drivers/test-driver/unbind", "0000:d8:00.0") + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/drivers/test-driver/unbind", "0000:d8:00.0") // should bind to driver2 - assertFileContentsEquals("/sys/bus/pci/drivers/vfio-pci/bind", "0000:d8:00.0") + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/drivers/vfio-pci/bind", "0000:d8:00.0") }) It("fail to bind", func() { - configureFS(&fakefilesystem.FS{ + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{ "/sys/bus/pci/devices/0000:d8:00.0", "/sys/bus/pci/drivers/test-driver"}, @@ -195,7 +177,7 @@ var _ = Describe("Kernel", func() { }) Context("BindDriverByBusAndDevice", func() { It("device doesn't support driver_override", func() { - configureFS(&fakefilesystem.FS{ + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ Dirs: []string{ "/sys/bus/pci/devices/0000:d8:00.0", "/sys/bus/pci/drivers/test-driver", @@ -208,9 +190,9 @@ var _ = Describe("Kernel", func() { }) Expect(k.BindDriverByBusAndDevice(consts.BusPci, "0000:d8:00.0", "vfio-pci")).NotTo(HaveOccurred()) // should unbind from driver1 - assertFileContentsEquals("/sys/bus/pci/drivers/test-driver/unbind", "0000:d8:00.0") + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/drivers/test-driver/unbind", "0000:d8:00.0") // should bind to driver2 - assertFileContentsEquals("/sys/bus/pci/drivers/vfio-pci/bind", "0000:d8:00.0") + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/drivers/vfio-pci/bind", "0000:d8:00.0") }) }) }) diff --git a/pkg/host/internal/lib/govdpa/govdpa.go b/pkg/host/internal/lib/govdpa/govdpa.go new file mode 100644 index 000000000..e85f89db1 --- /dev/null +++ b/pkg/host/internal/lib/govdpa/govdpa.go @@ -0,0 +1,40 @@ +package govdpa + +import ( + "github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa" +) + +func New() GoVdpaLib { + return &libWrapper{} +} + +type VdpaDevice interface { + kvdpa.VdpaDevice +} + +//go:generate ../../../../../bin/mockgen -destination mock/mock_govdpa.go -source govdpa.go +type GoVdpaLib interface { + // GetVdpaDevice returns the vdpa device information by a vdpa device name + GetVdpaDevice(vdpaDeviceName string) (VdpaDevice, error) + // AddVdpaDevice adds a new vdpa device to the given management device + AddVdpaDevice(mgmtDeviceName string, vdpaDeviceName string) error + // DeleteVdpaDevice deletes a vdpa device + DeleteVdpaDevice(vdpaDeviceName string) error +} + +type libWrapper struct{} + +// GetVdpaDevice returns the vdpa device information by a vdpa device name +func (w *libWrapper) GetVdpaDevice(name string) (VdpaDevice, error) { + return kvdpa.GetVdpaDevice(name) +} + +// AddVdpaDevice adds a new vdpa device to the given management device +func (w *libWrapper) AddVdpaDevice(mgmtDeviceName string, vdpaDeviceName string) error { + return kvdpa.AddVdpaDevice(mgmtDeviceName, vdpaDeviceName) +} + +// DeleteVdpaDevice deletes a vdpa device +func (w *libWrapper) DeleteVdpaDevice(name string) error { + return kvdpa.DeleteVdpaDevice(name) +} diff --git a/pkg/host/internal/lib/govdpa/mock/mock_govdpa.go b/pkg/host/internal/lib/govdpa/mock/mock_govdpa.go new file mode 100644 index 000000000..cdeeac743 --- /dev/null +++ b/pkg/host/internal/lib/govdpa/mock/mock_govdpa.go @@ -0,0 +1,187 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: govdpa.go + +// Package mock_govdpa is a generated GoMock package. +package mock_govdpa + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + kvdpa "github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa" + govdpa "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/govdpa" +) + +// MockVdpaDevice is a mock of VdpaDevice interface. +type MockVdpaDevice struct { + ctrl *gomock.Controller + recorder *MockVdpaDeviceMockRecorder +} + +// MockVdpaDeviceMockRecorder is the mock recorder for MockVdpaDevice. +type MockVdpaDeviceMockRecorder struct { + mock *MockVdpaDevice +} + +// NewMockVdpaDevice creates a new mock instance. +func NewMockVdpaDevice(ctrl *gomock.Controller) *MockVdpaDevice { + mock := &MockVdpaDevice{ctrl: ctrl} + mock.recorder = &MockVdpaDeviceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockVdpaDevice) EXPECT() *MockVdpaDeviceMockRecorder { + return m.recorder +} + +// Driver mocks base method. +func (m *MockVdpaDevice) Driver() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Driver") + ret0, _ := ret[0].(string) + return ret0 +} + +// Driver indicates an expected call of Driver. +func (mr *MockVdpaDeviceMockRecorder) Driver() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Driver", reflect.TypeOf((*MockVdpaDevice)(nil).Driver)) +} + +// MgmtDev mocks base method. +func (m *MockVdpaDevice) MgmtDev() kvdpa.MgmtDev { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MgmtDev") + ret0, _ := ret[0].(kvdpa.MgmtDev) + return ret0 +} + +// MgmtDev indicates an expected call of MgmtDev. +func (mr *MockVdpaDeviceMockRecorder) MgmtDev() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MgmtDev", reflect.TypeOf((*MockVdpaDevice)(nil).MgmtDev)) +} + +// Name mocks base method. +func (m *MockVdpaDevice) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockVdpaDeviceMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockVdpaDevice)(nil).Name)) +} + +// ParentDevicePath mocks base method. +func (m *MockVdpaDevice) ParentDevicePath() (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ParentDevicePath") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ParentDevicePath indicates an expected call of ParentDevicePath. +func (mr *MockVdpaDeviceMockRecorder) ParentDevicePath() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ParentDevicePath", reflect.TypeOf((*MockVdpaDevice)(nil).ParentDevicePath)) +} + +// VhostVdpa mocks base method. +func (m *MockVdpaDevice) VhostVdpa() kvdpa.VhostVdpa { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VhostVdpa") + ret0, _ := ret[0].(kvdpa.VhostVdpa) + return ret0 +} + +// VhostVdpa indicates an expected call of VhostVdpa. +func (mr *MockVdpaDeviceMockRecorder) VhostVdpa() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VhostVdpa", reflect.TypeOf((*MockVdpaDevice)(nil).VhostVdpa)) +} + +// VirtioNet mocks base method. +func (m *MockVdpaDevice) VirtioNet() kvdpa.VirtioNet { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VirtioNet") + ret0, _ := ret[0].(kvdpa.VirtioNet) + return ret0 +} + +// VirtioNet indicates an expected call of VirtioNet. +func (mr *MockVdpaDeviceMockRecorder) VirtioNet() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VirtioNet", reflect.TypeOf((*MockVdpaDevice)(nil).VirtioNet)) +} + +// MockGoVdpaLib is a mock of GoVdpaLib interface. +type MockGoVdpaLib struct { + ctrl *gomock.Controller + recorder *MockGoVdpaLibMockRecorder +} + +// MockGoVdpaLibMockRecorder is the mock recorder for MockGoVdpaLib. +type MockGoVdpaLibMockRecorder struct { + mock *MockGoVdpaLib +} + +// NewMockGoVdpaLib creates a new mock instance. +func NewMockGoVdpaLib(ctrl *gomock.Controller) *MockGoVdpaLib { + mock := &MockGoVdpaLib{ctrl: ctrl} + mock.recorder = &MockGoVdpaLibMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockGoVdpaLib) EXPECT() *MockGoVdpaLibMockRecorder { + return m.recorder +} + +// AddVdpaDevice mocks base method. +func (m *MockGoVdpaLib) AddVdpaDevice(mgmtDeviceName, vdpaDeviceName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddVdpaDevice", mgmtDeviceName, vdpaDeviceName) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddVdpaDevice indicates an expected call of AddVdpaDevice. +func (mr *MockGoVdpaLibMockRecorder) AddVdpaDevice(mgmtDeviceName, vdpaDeviceName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddVdpaDevice", reflect.TypeOf((*MockGoVdpaLib)(nil).AddVdpaDevice), mgmtDeviceName, vdpaDeviceName) +} + +// DeleteVdpaDevice mocks base method. +func (m *MockGoVdpaLib) DeleteVdpaDevice(vdpaDeviceName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteVdpaDevice", vdpaDeviceName) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteVdpaDevice indicates an expected call of DeleteVdpaDevice. +func (mr *MockGoVdpaLibMockRecorder) DeleteVdpaDevice(vdpaDeviceName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteVdpaDevice", reflect.TypeOf((*MockGoVdpaLib)(nil).DeleteVdpaDevice), vdpaDeviceName) +} + +// GetVdpaDevice mocks base method. +func (m *MockGoVdpaLib) GetVdpaDevice(vdpaDeviceName string) (govdpa.VdpaDevice, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVdpaDevice", vdpaDeviceName) + ret0, _ := ret[0].(govdpa.VdpaDevice) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetVdpaDevice indicates an expected call of GetVdpaDevice. +func (mr *MockGoVdpaLibMockRecorder) GetVdpaDevice(vdpaDeviceName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVdpaDevice", reflect.TypeOf((*MockGoVdpaLib)(nil).GetVdpaDevice), vdpaDeviceName) +} diff --git a/pkg/host/internal/lib/netlink/mock/mock_netlink.go b/pkg/host/internal/lib/netlink/mock/mock_netlink.go new file mode 100644 index 000000000..072ba8980 --- /dev/null +++ b/pkg/host/internal/lib/netlink/mock/mock_netlink.go @@ -0,0 +1,188 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: netlink.go + +// Package mock_netlink is a generated GoMock package. +package mock_netlink + +import ( + net "net" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + netlink "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" + netlink0 "github.com/vishvananda/netlink" +) + +// MockLink is a mock of Link interface. +type MockLink struct { + ctrl *gomock.Controller + recorder *MockLinkMockRecorder +} + +// MockLinkMockRecorder is the mock recorder for MockLink. +type MockLinkMockRecorder struct { + mock *MockLink +} + +// NewMockLink creates a new mock instance. +func NewMockLink(ctrl *gomock.Controller) *MockLink { + mock := &MockLink{ctrl: ctrl} + mock.recorder = &MockLinkMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockLink) EXPECT() *MockLinkMockRecorder { + return m.recorder +} + +// Attrs mocks base method. +func (m *MockLink) Attrs() *netlink0.LinkAttrs { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Attrs") + ret0, _ := ret[0].(*netlink0.LinkAttrs) + return ret0 +} + +// Attrs indicates an expected call of Attrs. +func (mr *MockLinkMockRecorder) Attrs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockLink)(nil).Attrs)) +} + +// Type mocks base method. +func (m *MockLink) Type() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Type") + ret0, _ := ret[0].(string) + return ret0 +} + +// Type indicates an expected call of Type. +func (mr *MockLinkMockRecorder) Type() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Type", reflect.TypeOf((*MockLink)(nil).Type)) +} + +// MockNetlinkLib is a mock of NetlinkLib interface. +type MockNetlinkLib struct { + ctrl *gomock.Controller + recorder *MockNetlinkLibMockRecorder +} + +// MockNetlinkLibMockRecorder is the mock recorder for MockNetlinkLib. +type MockNetlinkLibMockRecorder struct { + mock *MockNetlinkLib +} + +// NewMockNetlinkLib creates a new mock instance. +func NewMockNetlinkLib(ctrl *gomock.Controller) *MockNetlinkLib { + mock := &MockNetlinkLib{ctrl: ctrl} + mock.recorder = &MockNetlinkLibMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockNetlinkLib) EXPECT() *MockNetlinkLibMockRecorder { + return m.recorder +} + +// DevLinkGetDeviceByName mocks base method. +func (m *MockNetlinkLib) DevLinkGetDeviceByName(bus, device string) (*netlink0.DevlinkDevice, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DevLinkGetDeviceByName", bus, device) + ret0, _ := ret[0].(*netlink0.DevlinkDevice) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DevLinkGetDeviceByName indicates an expected call of DevLinkGetDeviceByName. +func (mr *MockNetlinkLibMockRecorder) DevLinkGetDeviceByName(bus, device interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DevLinkGetDeviceByName", reflect.TypeOf((*MockNetlinkLib)(nil).DevLinkGetDeviceByName), bus, device) +} + +// DevLinkSetEswitchMode mocks base method. +func (m *MockNetlinkLib) DevLinkSetEswitchMode(dev *netlink0.DevlinkDevice, newMode string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DevLinkSetEswitchMode", dev, newMode) + ret0, _ := ret[0].(error) + return ret0 +} + +// DevLinkSetEswitchMode indicates an expected call of DevLinkSetEswitchMode. +func (mr *MockNetlinkLibMockRecorder) DevLinkSetEswitchMode(dev, newMode interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DevLinkSetEswitchMode", reflect.TypeOf((*MockNetlinkLib)(nil).DevLinkSetEswitchMode), dev, newMode) +} + +// LinkByName mocks base method. +func (m *MockNetlinkLib) LinkByName(name string) (netlink.Link, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LinkByName", name) + ret0, _ := ret[0].(netlink.Link) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LinkByName indicates an expected call of LinkByName. +func (mr *MockNetlinkLibMockRecorder) LinkByName(name interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkByName", reflect.TypeOf((*MockNetlinkLib)(nil).LinkByName), name) +} + +// LinkSetUp mocks base method. +func (m *MockNetlinkLib) LinkSetUp(link netlink.Link) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LinkSetUp", link) + ret0, _ := ret[0].(error) + return ret0 +} + +// LinkSetUp indicates an expected call of LinkSetUp. +func (mr *MockNetlinkLibMockRecorder) LinkSetUp(link interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkSetUp", reflect.TypeOf((*MockNetlinkLib)(nil).LinkSetUp), link) +} + +// LinkSetVfHardwareAddr mocks base method. +func (m *MockNetlinkLib) LinkSetVfHardwareAddr(link netlink.Link, vf int, hwaddr net.HardwareAddr) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LinkSetVfHardwareAddr", link, vf, hwaddr) + ret0, _ := ret[0].(error) + return ret0 +} + +// LinkSetVfHardwareAddr indicates an expected call of LinkSetVfHardwareAddr. +func (mr *MockNetlinkLibMockRecorder) LinkSetVfHardwareAddr(link, vf, hwaddr interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkSetVfHardwareAddr", reflect.TypeOf((*MockNetlinkLib)(nil).LinkSetVfHardwareAddr), link, vf, hwaddr) +} + +// LinkSetVfNodeGUID mocks base method. +func (m *MockNetlinkLib) LinkSetVfNodeGUID(link netlink.Link, vf int, nodeguid net.HardwareAddr) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LinkSetVfNodeGUID", link, vf, nodeguid) + ret0, _ := ret[0].(error) + return ret0 +} + +// LinkSetVfNodeGUID indicates an expected call of LinkSetVfNodeGUID. +func (mr *MockNetlinkLibMockRecorder) LinkSetVfNodeGUID(link, vf, nodeguid interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkSetVfNodeGUID", reflect.TypeOf((*MockNetlinkLib)(nil).LinkSetVfNodeGUID), link, vf, nodeguid) +} + +// LinkSetVfPortGUID mocks base method. +func (m *MockNetlinkLib) LinkSetVfPortGUID(link netlink.Link, vf int, portguid net.HardwareAddr) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LinkSetVfPortGUID", link, vf, portguid) + ret0, _ := ret[0].(error) + return ret0 +} + +// LinkSetVfPortGUID indicates an expected call of LinkSetVfPortGUID. +func (mr *MockNetlinkLibMockRecorder) LinkSetVfPortGUID(link, vf, portguid interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkSetVfPortGUID", reflect.TypeOf((*MockNetlinkLib)(nil).LinkSetVfPortGUID), link, vf, portguid) +} diff --git a/pkg/host/internal/lib/netlink/netlink.go b/pkg/host/internal/lib/netlink/netlink.go new file mode 100644 index 000000000..4871abe95 --- /dev/null +++ b/pkg/host/internal/lib/netlink/netlink.go @@ -0,0 +1,86 @@ +package netlink + +import ( + "net" + + "github.com/vishvananda/netlink" +) + +func New() NetlinkLib { + return &libWrapper{} +} + +type Link interface { + netlink.Link +} + +//go:generate ../../../../../bin/mockgen -destination mock/mock_netlink.go -source netlink.go +type NetlinkLib interface { + // LinkSetVfNodeGUID sets the node GUID of a vf for the link. + // Equivalent to: `ip link set dev $link vf $vf node_guid $nodeguid` + LinkSetVfNodeGUID(link Link, vf int, nodeguid net.HardwareAddr) error + // LinkSetVfPortGUID sets the port GUID of a vf for the link. + // Equivalent to: `ip link set dev $link vf $vf port_guid $portguid` + LinkSetVfPortGUID(link Link, vf int, portguid net.HardwareAddr) error + // LinkByName finds a link by name and returns a pointer to the object. + LinkByName(name string) (Link, error) + // LinkSetVfHardwareAddr sets the hardware address of a vf for the link. + // Equivalent to: `ip link set $link vf $vf mac $hwaddr` + LinkSetVfHardwareAddr(link Link, vf int, hwaddr net.HardwareAddr) error + // LinkSetUp enables the link device. + // Equivalent to: `ip link set $link up` + LinkSetUp(link Link) error + // DevlinkGetDeviceByName provides a pointer to devlink device and nil error, + // otherwise returns an error code. + DevLinkGetDeviceByName(bus string, device string) (*netlink.DevlinkDevice, error) + // DevLinkSetEswitchMode sets eswitch mode if able to set successfully or + // returns an error code. + // Equivalent to: `devlink dev eswitch set $dev mode switchdev` + // Equivalent to: `devlink dev eswitch set $dev mode legacy` + DevLinkSetEswitchMode(dev *netlink.DevlinkDevice, newMode string) error +} + +type libWrapper struct{} + +// LinkSetVfNodeGUID sets the node GUID of a vf for the link. +// Equivalent to: `ip link set dev $link vf $vf node_guid $nodeguid` +func (w *libWrapper) LinkSetVfNodeGUID(link Link, vf int, nodeguid net.HardwareAddr) error { + return netlink.LinkSetVfNodeGUID(link, vf, nodeguid) +} + +// LinkSetVfPortGUID sets the port GUID of a vf for the link. +// Equivalent to: `ip link set dev $link vf $vf port_guid $portguid` +func (w *libWrapper) LinkSetVfPortGUID(link Link, vf int, portguid net.HardwareAddr) error { + return netlink.LinkSetVfPortGUID(link, vf, portguid) +} + +// LinkByName finds a link by name and returns a pointer to the object.// LinkByName finds a link by name and returns a pointer to the object. +func (w *libWrapper) LinkByName(name string) (Link, error) { + return netlink.LinkByName(name) +} + +// LinkSetVfHardwareAddr sets the hardware address of a vf for the link. +// Equivalent to: `ip link set $link vf $vf mac $hwaddr` +func (w *libWrapper) LinkSetVfHardwareAddr(link Link, vf int, hwaddr net.HardwareAddr) error { + return netlink.LinkSetVfHardwareAddr(link, vf, hwaddr) +} + +// LinkSetUp enables the link device. +// Equivalent to: `ip link set $link up` +func (w *libWrapper) LinkSetUp(link Link) error { + return netlink.LinkSetUp(link) +} + +// DevlinkGetDeviceByName provides a pointer to devlink device and nil error, +// otherwise returns an error code. +func (w *libWrapper) DevLinkGetDeviceByName(bus string, device string) (*netlink.DevlinkDevice, error) { + return netlink.DevLinkGetDeviceByName(bus, device) +} + +// DevLinkSetEswitchMode sets eswitch mode if able to set successfully or +// returns an error code. +// Equivalent to: `devlink dev eswitch set $dev mode switchdev` +// Equivalent to: `devlink dev eswitch set $dev mode legacy` +func (w *libWrapper) DevLinkSetEswitchMode(dev *netlink.DevlinkDevice, newMode string) error { + return netlink.DevLinkSetEswitchMode(dev, newMode) +} diff --git a/pkg/host/internal/sriov/sriov.go b/pkg/host/internal/sriov/sriov.go index 3baf6cbec..9541c8bcd 100644 --- a/pkg/host/internal/sriov/sriov.go +++ b/pkg/host/internal/sriov/sriov.go @@ -19,6 +19,7 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + netlinkPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/store" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" @@ -31,13 +32,20 @@ type sriov struct { kernelHelper types.KernelInterface networkHelper types.NetworkInterface udevHelper types.UdevInterface + netlinkLib netlinkPkg.NetlinkLib } func New(utilsHelper utils.CmdInterface, kernelHelper types.KernelInterface, networkHelper types.NetworkInterface, - udevHelper types.UdevInterface) types.SriovInterface { - return &sriov{utilsHelper: utilsHelper, kernelHelper: kernelHelper, networkHelper: networkHelper, udevHelper: udevHelper} + udevHelper types.UdevInterface, + netlinkLib netlinkPkg.NetlinkLib) types.SriovInterface { + return &sriov{utilsHelper: utilsHelper, + kernelHelper: kernelHelper, + networkHelper: networkHelper, + udevHelper: udevHelper, + netlinkLib: netlinkLib, + } } func (s *sriov) SetSriovNumVfs(pciAddr string, numVfs int) error { @@ -49,6 +57,9 @@ func (s *sriov) SetSriovNumVfs(pciAddr string, numVfs int) error { log.Log.Error(err, "SetSriovNumVfs(): fail to reset NumVfs file", "path", numVfsFilePath) return err } + if numVfs == 0 { + return nil + } err = os.WriteFile(numVfsFilePath, bs, os.ModeAppend) if err != nil { log.Log.Error(err, "SetSriovNumVfs(): fail to set NumVfs file", "path", numVfsFilePath) @@ -124,10 +135,10 @@ func (s *sriov) SetVfGUID(vfAddr string, pfLink netlink.Link) error { return err } guid := utils.GenerateRandomGUID() - if err := netlink.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil { + if err := s.netlinkLib.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil { return err } - if err := netlink.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil { + if err := s.netlinkLib.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil { return err } if err = s.kernelHelper.Unbind(vfAddr); err != nil { @@ -143,7 +154,7 @@ func (s *sriov) VFIsReady(pciAddr string) (netlink.Link, error) { var vfLink netlink.Link err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { vfName := s.networkHelper.TryGetInterfaceName(pciAddr) - vfLink, err = netlink.LinkByName(vfName) + vfLink, err = s.netlinkLib.LinkByName(vfName) if err != nil { log.Log.Error(err, "VFIsReady(): unable to get VF link", "device", pciAddr) } @@ -164,7 +175,7 @@ func (s *sriov) SetVfAdminMac(vfAddr string, pfLink, vfLink netlink.Link) error return err } - if err := netlink.LinkSetVfHardwareAddr(pfLink, vfID, vfLink.Attrs().HardwareAddr); err != nil { + if err := s.netlinkLib.LinkSetVfHardwareAddr(pfLink, vfID, vfLink.Attrs().HardwareAddr); err != nil { return err } @@ -333,7 +344,7 @@ func (s *sriov) ConfigSriovDevice(iface *sriovnetworkv1.Interface, ifaceStatus * if err != nil { log.Log.Error(err, "configSriovDevice(): unable to parse VFs for device", "device", iface.PciAddress) } - pfLink, err := netlink.LinkByName(iface.Name) + pfLink, err := s.netlinkLib.LinkByName(iface.Name) if err != nil { log.Log.Error(err, "configSriovDevice(): unable to get PF link for device", "device", iface) return err @@ -424,12 +435,12 @@ func (s *sriov) ConfigSriovDevice(iface *sriovnetworkv1.Interface, ifaceStatus * } } // Set PF link up - pfLink, err := netlink.LinkByName(ifaceStatus.Name) + pfLink, err := s.netlinkLib.LinkByName(ifaceStatus.Name) if err != nil { return err } if pfLink.Attrs().OperState != netlink.OperUp { - err = netlink.LinkSetUp(pfLink) + err = s.netlinkLib.LinkSetUp(pfLink) if err != nil { return err } @@ -576,7 +587,7 @@ func (s *sriov) ConfigSriovDeviceVirtual(iface *sriovnetworkv1.Interface) error func (s *sriov) GetNicSriovMode(pciAddress string) (string, error) { log.Log.V(2).Info("GetNicSriovMode()", "device", pciAddress) - devLink, err := netlink.DevLinkGetDeviceByName("pci", pciAddress) + devLink, err := s.netlinkLib.DevLinkGetDeviceByName("pci", pciAddress) if err != nil { if errors.Is(err, syscall.ENODEV) { // the device doesn't support devlink @@ -588,10 +599,20 @@ func (s *sriov) GetNicSriovMode(pciAddress string) (string, error) { return devLink.Attrs.Eswitch.Mode, nil } +func (s *sriov) SetNicSriovMode(pciAddress string, mode string) error { + log.Log.V(2).Info("SetNicSriovMode()", "device", pciAddress, "mode", mode) + + dev, err := s.netlinkLib.DevLinkGetDeviceByName("pci", pciAddress) + if err != nil { + return err + } + return s.netlinkLib.DevLinkSetEswitchMode(dev, mode) +} + func (s *sriov) GetLinkType(ifaceStatus sriovnetworkv1.InterfaceExt) string { log.Log.V(2).Info("GetLinkType()", "device", ifaceStatus.PciAddress) if ifaceStatus.Name != "" { - link, err := netlink.LinkByName(ifaceStatus.Name) + link, err := s.netlinkLib.LinkByName(ifaceStatus.Name) if err != nil { log.Log.Error(err, "GetLinkType(): failed to get link", "device", ifaceStatus.Name) return "" diff --git a/pkg/host/internal/sriov/sriov_test.go b/pkg/host/internal/sriov/sriov_test.go new file mode 100644 index 000000000..8a87a6de6 --- /dev/null +++ b/pkg/host/internal/sriov/sriov_test.go @@ -0,0 +1,96 @@ +package sriov + +import ( + "fmt" + "strconv" + "syscall" + + "github.com/golang/mock/gomock" + "github.com/vishvananda/netlink" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + netlinkMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink/mock" + hostMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/mock" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/helpers" +) + +var _ = Describe("SRIOV", func() { + var ( + s types.SriovInterface + libMock *netlinkMockPkg.MockNetlinkLib + hostMock *hostMockPkg.MockHostManagerInterface + + testCtrl *gomock.Controller + + testError = fmt.Errorf("test") + ) + BeforeEach(func() { + testCtrl = gomock.NewController(GinkgoT()) + libMock = netlinkMockPkg.NewMockNetlinkLib(testCtrl) + hostMock = hostMockPkg.NewMockHostManagerInterface(testCtrl) + s = New(nil, hostMock, hostMock, hostMock, libMock) + }) + + AfterEach(func() { + testCtrl.Finish() + }) + + Context("SetSriovNumVfs", func() { + It("set", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/sys/bus/pci/devices/0000:d8:00.0"}, + Files: map[string][]byte{"/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs": {}}, + }) + Expect(s.SetSriovNumVfs("0000:d8:00.0", 5)).NotTo(HaveOccurred()) + helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs", strconv.Itoa(5)) + }) + It("fail - no such device", func() { + Expect(s.SetSriovNumVfs("0000:d8:00.0", 5)).To(HaveOccurred()) + }) + }) + + Context("GetNicSriovMode", func() { + It("devlink returns info", func() { + libMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( + &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "switchdev"}}}, + nil) + mode, err := s.GetNicSriovMode("0000:d8:00.0") + Expect(err).NotTo(HaveOccurred()) + Expect(mode).To(Equal("switchdev")) + }) + It("devlink returns error", func() { + libMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(nil, testError) + _, err := s.GetNicSriovMode("0000:d8:00.0") + Expect(err).To(MatchError(testError)) + }) + It("devlink not supported - fail to get name", func() { + libMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(nil, syscall.ENODEV) + mode, err := s.GetNicSriovMode("0000:d8:00.0") + Expect(err).NotTo(HaveOccurred()) + Expect(mode).To(BeEmpty()) + }) + }) + + Context("SetNicSriovMode", func() { + It("set", func() { + testDev := &netlink.DevlinkDevice{} + libMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(&netlink.DevlinkDevice{}, nil) + libMock.EXPECT().DevLinkSetEswitchMode(testDev, "legacy").Return(nil) + Expect(s.SetNicSriovMode("0000:d8:00.0", "legacy")).NotTo(HaveOccurred()) + }) + It("fail to get dev", func() { + libMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(nil, testError) + Expect(s.SetNicSriovMode("0000:d8:00.0", "legacy")).To(MatchError(testError)) + }) + It("fail to set mode", func() { + testDev := &netlink.DevlinkDevice{} + libMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(&netlink.DevlinkDevice{}, nil) + libMock.EXPECT().DevLinkSetEswitchMode(testDev, "legacy").Return(testError) + Expect(s.SetNicSriovMode("0000:d8:00.0", "legacy")).To(MatchError(testError)) + }) + }) +}) diff --git a/pkg/host/internal/sriov/suite_test.go b/pkg/host/internal/sriov/suite_test.go new file mode 100644 index 000000000..8f76f0a85 --- /dev/null +++ b/pkg/host/internal/sriov/suite_test.go @@ -0,0 +1,21 @@ +package sriov + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "go.uber.org/zap/zapcore" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +func TestSriov(t *testing.T) { + log.SetLogger(zap.New( + zap.WriteTo(GinkgoWriter), + zap.Level(zapcore.Level(-2)), + zap.UseDevMode(true))) + RegisterFailHandler(Fail) + RunSpecs(t, "Package SRIOV Suite") +} diff --git a/pkg/host/internal/udev/suite_test.go b/pkg/host/internal/udev/suite_test.go new file mode 100644 index 000000000..34a8820cf --- /dev/null +++ b/pkg/host/internal/udev/suite_test.go @@ -0,0 +1,30 @@ +package udev + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "go.uber.org/zap/zapcore" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +func TestUdev(t *testing.T) { + log.SetLogger(zap.New( + zap.WriteTo(GinkgoWriter), + zap.Level(zapcore.Level(-2)), + zap.UseDevMode(true))) + RegisterFailHandler(Fail) + RunSpecs(t, "Package Udev Suite") +} + +var _ = BeforeSuite(func() { + vars.SupportedVfIds = []string{"0x1017", "0x1018"} + DeferCleanup(func() { + vars.SupportedVfIds = []string{} + }) +}) diff --git a/pkg/host/internal/udev/udev.go b/pkg/host/internal/udev/udev.go index dad88913e..2b4c4c6b3 100644 --- a/pkg/host/internal/udev/udev.go +++ b/pkg/host/internal/udev/udev.go @@ -146,34 +146,64 @@ func (u *udev) WriteSwitchdevConfFile(newState *sriovnetworkv1.SriovNetworkNodeS return true, nil } +// AddUdevRule adds a udev rule that disables network-manager for VFs on the concrete PF func (u *udev) AddUdevRule(pfPciAddress string) error { log.Log.V(2).Info("AddUdevRule()", "device", pfPciAddress) - pathFile := filepath.Join(vars.FilesystemRoot, consts.UdevRulesFolder) udevRuleContent := fmt.Sprintf(consts.NMUdevRule, strings.Join(vars.SupportedVfIds, "|"), pfPciAddress) + return u.addUdevRule(pfPciAddress, "10-nm-disable", udevRuleContent) +} + +// RemoveUdevRule removes a udev rule that disables network-manager for VFs on the concrete PF +func (u *udev) RemoveUdevRule(pfPciAddress string) error { + log.Log.V(2).Info("RemoveUdevRule()", "device", pfPciAddress) + return u.removeUdevRule(pfPciAddress, "10-nm-disable") +} + +// AddVfRepresentorUdevRule adds udev rule that renames VF representors on the concrete PF +func (u *udev) AddVfRepresentorUdevRule(pfPciAddress, pfName, pfSwitchID, pfSwitchPort string) error { + log.Log.V(2).Info("AddVfRepresentorUdevRule()", + "device", pfPciAddress, "name", pfName, "switch", pfSwitchID, "port", pfSwitchPort) + udevRuleContent := fmt.Sprintf(consts.SwitchdevUdevRule, pfSwitchID, strings.TrimPrefix(pfSwitchPort, "p"), pfName) + return u.addUdevRule(pfPciAddress, "20-switchdev", udevRuleContent) +} + +// RemoveVfRepresentorUdevRule removes udev rule that renames VF representors on the concrete PF +func (u *udev) RemoveVfRepresentorUdevRule(pfPciAddress string) error { + log.Log.V(2).Info("RemoveVfRepresentorUdevRule()", "device", pfPciAddress) + return u.removeUdevRule(pfPciAddress, "20-switchdev") +} - err := os.MkdirAll(pathFile, os.ModePerm) +func (u *udev) addUdevRule(pfPciAddress, ruleName, ruleContent string) error { + log.Log.V(2).Info("addUdevRule()", "device", pfPciAddress, "rule", ruleName) + rulePath := u.getRuleFolderPath() + err := os.MkdirAll(rulePath, os.ModePerm) if err != nil && !os.IsExist(err) { - log.Log.Error(err, "AddUdevRule(): failed to create dir", "path", pathFile) + log.Log.Error(err, "ensureUdevRulePathExist(): failed to create dir", "path", rulePath) return err } - - filePath := path.Join(pathFile, fmt.Sprintf("10-nm-disable-%s.rules", pfPciAddress)) - // if the file does not exist or if oldContent != newContent - // write to file and create it if it doesn't exist - err = os.WriteFile(filePath, []byte(udevRuleContent), 0666) - if err != nil { - log.Log.Error(err, "AddUdevRule(): fail to write file", "path", filePath) + filePath := u.getRulePathForPF(ruleName, pfPciAddress) + if err := os.WriteFile(filePath, []byte(ruleContent), 0666); err != nil { + log.Log.Error(err, "addUdevRule(): fail to write file", "path", filePath) return err } return nil } -func (u *udev) RemoveUdevRule(pfPciAddress string) error { - pathFile := filepath.Join(vars.FilesystemRoot, consts.UdevRulesFolder) - filePath := path.Join(pathFile, fmt.Sprintf("10-nm-disable-%s.rules", pfPciAddress)) - err := os.Remove(filePath) +func (u *udev) removeUdevRule(pfPciAddress, ruleName string) error { + log.Log.V(2).Info("removeUdevRule()", "device", pfPciAddress, "rule", ruleName) + rulePath := u.getRulePathForPF(ruleName, pfPciAddress) + err := os.Remove(rulePath) if err != nil && !os.IsNotExist(err) { + log.Log.Error(err, "removeUdevRule(): fail to remove rule file", "path", rulePath) return err } return nil } + +func (u *udev) getRuleFolderPath() string { + return filepath.Join(vars.FilesystemRoot, consts.UdevRulesFolder) +} + +func (u *udev) getRulePathForPF(ruleName, pfPciAddress string) string { + return path.Join(u.getRuleFolderPath(), fmt.Sprintf("%s-%s.rules", ruleName, pfPciAddress)) +} diff --git a/pkg/host/internal/udev/udev_test.go b/pkg/host/internal/udev/udev_test.go new file mode 100644 index 000000000..d5f222385 --- /dev/null +++ b/pkg/host/internal/udev/udev_test.go @@ -0,0 +1,117 @@ +package udev + +import ( + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/helpers" +) + +const ( + testExpectedNMUdevRule = `SUBSYSTEM=="net", ACTION=="add|change|move", ` + + `ATTRS{device}=="0x1017|0x1018", ` + + `IMPORT{program}="/etc/udev/disable-nm-sriov.sh $env{INTERFACE} 0000:d8:00.0"` + testExpectedSwitchdevUdevRule = `SUBSYSTEM=="net", ACTION=="add|move", ` + + `ATTRS{phys_switch_id}=="7cfe90ff2cc0", ` + + `ATTR{phys_port_name}=="pf0vf*", IMPORT{program}="/etc/udev/switchdev-vf-link-name.sh $attr{phys_port_name}", ` + + `NAME="enp216s0f0np0_$env{NUMBER}"` +) + +var _ = Describe("UDEV", func() { + var ( + s types.UdevInterface + ) + BeforeEach(func() { + s = New(nil) + }) + Context("AddUdevRule", func() { + It("Created", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{}) + Expect(s.AddUdevRule("0000:d8:00.0")).To(BeNil()) + helpers.GinkgoAssertFileContentsEquals( + "/etc/udev/rules.d/10-nm-disable-0000:d8:00.0.rules", + testExpectedNMUdevRule) + }) + It("Overwrite", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/etc/udev/rules.d"}, + Files: map[string][]byte{ + "/etc/udev/rules.d/10-nm-disable-0000:d8:00.0.rules": []byte("something"), + }, + }) + Expect(s.AddUdevRule("0000:d8:00.0")).To(BeNil()) + helpers.GinkgoAssertFileContentsEquals( + "/etc/udev/rules.d/10-nm-disable-0000:d8:00.0.rules", + testExpectedNMUdevRule) + }) + }) + Context("RemoveUdevRule", func() { + It("Exist", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/etc/udev/rules.d"}, + Files: map[string][]byte{ + "/etc/udev/rules.d/10-nm-disable-0000:d8:00.0.rules": []byte(testExpectedNMUdevRule), + }, + }) + Expect(s.RemoveUdevRule("0000:d8:00.0")).To(BeNil()) + _, err := os.Stat(filepath.Join(vars.FilesystemRoot, + "/etc/udev/rules.d/10-nm-disable-0000:d8:00.0.rules")) + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + It("Not found", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/etc/udev/rules.d"}, + }) + Expect(s.RemoveUdevRule("0000:d8:00.0")).To(BeNil()) + }) + }) + Context("AddVfRepresentorUdevRule", func() { + It("Created", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{}) + Expect(s.AddVfRepresentorUdevRule("0000:d8:00.0", + "enp216s0f0np0", "7cfe90ff2cc0", "p0")).To(BeNil()) + helpers.GinkgoAssertFileContentsEquals( + "/etc/udev/rules.d/20-switchdev-0000:d8:00.0.rules", + testExpectedSwitchdevUdevRule) + }) + It("Overwrite", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/etc/udev/rules.d"}, + Files: map[string][]byte{ + "/etc/udev/rules.d/20-switchdev-0000:d8:00.0.rules": []byte("something"), + }, + }) + Expect(s.AddVfRepresentorUdevRule("0000:d8:00.0", + "enp216s0f0np0", "7cfe90ff2cc0", "p0")).To(BeNil()) + helpers.GinkgoAssertFileContentsEquals( + "/etc/udev/rules.d/20-switchdev-0000:d8:00.0.rules", + testExpectedSwitchdevUdevRule) + }) + }) + Context("RemoveVfRepresentorUdevRule", func() { + It("Exist", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/etc/udev/rules.d"}, + Files: map[string][]byte{ + "/etc/udev/rules.d/20-switchdev-0000:d8:00.0.rules": []byte(testExpectedSwitchdevUdevRule), + }, + }) + Expect(s.RemoveVfRepresentorUdevRule("0000:d8:00.0")).To(BeNil()) + _, err := os.Stat(filepath.Join(vars.FilesystemRoot, + "/etc/udev/rules.d/20-switchdev-0000:d8:00.0.rules")) + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + It("Not found", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/etc/udev/rules.d"}, + }) + Expect(s.RemoveVfRepresentorUdevRule("0000:d8:00.0")).To(BeNil()) + }) + }) +}) diff --git a/pkg/host/internal/vdpa/suite_test.go b/pkg/host/internal/vdpa/suite_test.go new file mode 100644 index 000000000..5863f7f27 --- /dev/null +++ b/pkg/host/internal/vdpa/suite_test.go @@ -0,0 +1,21 @@ +package vdpa + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "go.uber.org/zap/zapcore" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +func TestVdpa(t *testing.T) { + log.SetLogger(zap.New( + zap.WriteTo(GinkgoWriter), + zap.Level(zapcore.Level(-2)), + zap.UseDevMode(true))) + RegisterFailHandler(Fail) + RunSpecs(t, "Package VDPA Suite") +} diff --git a/pkg/host/internal/vdpa/vdpa.go b/pkg/host/internal/vdpa/vdpa.go new file mode 100644 index 000000000..010098f8b --- /dev/null +++ b/pkg/host/internal/vdpa/vdpa.go @@ -0,0 +1,131 @@ +package vdpa + +import ( + "errors" + "fmt" + "syscall" + + "github.com/k8snetworkplumbingwg/govdpa/pkg/kvdpa" + "sigs.k8s.io/controller-runtime/pkg/log" + + constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/govdpa" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" +) + +type vdpa struct { + kernel types.KernelInterface + vdpaLib govdpa.GoVdpaLib +} + +func New(k types.KernelInterface, vdpaLib govdpa.GoVdpaLib) types.VdpaInterface { + return &vdpa{kernel: k, vdpaLib: vdpaLib} +} + +// CreateVDPADevice creates VDPA device for VF with required type, +// pciAddr - PCI address of the VF +// vdpaType - type of the VDPA device to create: virtio of vhost +func (v *vdpa) CreateVDPADevice(pciAddr, vdpaType string) error { + log.Log.V(2).Info("CreateVDPADevice(): create VDPA device for VF", + "device", pciAddr, "vdpaType", vdpaType) + expectedDriver := vdpaTypeToDriver(vdpaType) + if expectedDriver == "" { + return fmt.Errorf("unknown VDPA device type: %s", vdpaType) + } + expectedVDPAName := generateVDPADevName(pciAddr) + _, err := v.vdpaLib.GetVdpaDevice(expectedVDPAName) + if err != nil { + if !errors.Is(err, syscall.ENODEV) { + log.Log.Error(err, "CreateVDPADevice(): fail to check if VDPA device exist", + "device", pciAddr, "vdpaDev", expectedVDPAName) + return err + } + if err := v.vdpaLib.AddVdpaDevice("pci/"+pciAddr, expectedVDPAName); err != nil { + log.Log.Error(err, "CreateVDPADevice(): fail to create VDPA device", + "device", pciAddr, "vdpaDev", expectedVDPAName) + return err + } + } + err = v.kernel.BindDriverByBusAndDevice(constants.BusVdpa, expectedVDPAName, expectedDriver) + if err != nil { + log.Log.Error(err, "CreateVDPADevice(): fail to bind VDPA device to the driver", + "device", pciAddr, "vdpaDev", expectedVDPAName, "driver", expectedDriver) + return err + } + return nil +} + +// DeleteVDPADevice removes VDPA device for provided pci address +// pciAddr - PCI address of the VF +func (v *vdpa) DeleteVDPADevice(pciAddr string) error { + log.Log.V(2).Info("DeleteVDPADevice(): delete VDPA device for VF", + "device", pciAddr) + expectedVDPAName := generateVDPADevName(pciAddr) + if err := v.vdpaLib.DeleteVdpaDevice(expectedVDPAName); err != nil { + if errors.Is(err, syscall.ENODEV) { + log.Log.V(2).Info("DeleteVDPADevice(): VDPA device not found", + "device", pciAddr, "name", expectedVDPAName) + return nil + } + log.Log.Error(err, "DeleteVDPADevice(): fail to remove VDPA device", + "device", pciAddr, "name", expectedVDPAName) + return err + } + return nil +} + +// DiscoverVDPAType returns type of existing VDPA device for VF, +// returns empty string if VDPA device not found or unknown driver is in use +// pciAddr - PCI address of the VF +func (v *vdpa) DiscoverVDPAType(pciAddr string) string { + expectedVDPADevName := generateVDPADevName(pciAddr) + vdpaDev, err := v.vdpaLib.GetVdpaDevice(expectedVDPADevName) + if err != nil { + if errors.Is(err, syscall.ENODEV) { + log.Log.V(2).Info("discoverVDPAType(): VDPA device for VF not found", "device", pciAddr) + return "" + } + log.Log.Error(err, "getVfInfo(): unable to get VF VDPA devices", "device", pciAddr) + return "" + } + driverName := vdpaDev.Driver() + if driverName == "" { + log.Log.V(2).Info("discoverVDPAType(): VDPA device has no driver", "device", pciAddr) + return "" + } + vdpaType := vdpaDriverToType(driverName) + if vdpaType == "" { + log.Log.Error(nil, "getVfInfo(): WARNING: unknown VDPA device type for VF, ignore", + "device", pciAddr, "driver", driverName) + } + return vdpaType +} + +// generates predictable name for VDPA device, example: vpda:0000:03:00.1 +func generateVDPADevName(pciAddr string) string { + return "vdpa:" + pciAddr +} + +// vdpa type to driver name conversion +func vdpaTypeToDriver(vdpaType string) string { + switch vdpaType { + case constants.VdpaTypeVhost: + return kvdpa.VhostVdpaDriver + case constants.VdpaTypeVirtio: + return kvdpa.VirtioVdpaDriver + default: + return "" + } +} + +// vdpa driver name to type conversion +func vdpaDriverToType(driver string) string { + switch driver { + case kvdpa.VhostVdpaDriver: + return constants.VdpaTypeVhost + case kvdpa.VirtioVdpaDriver: + return constants.VdpaTypeVirtio + default: + return "" + } +} diff --git a/pkg/host/internal/vdpa/vdpa_test.go b/pkg/host/internal/vdpa/vdpa_test.go new file mode 100644 index 000000000..3c668aa44 --- /dev/null +++ b/pkg/host/internal/vdpa/vdpa_test.go @@ -0,0 +1,115 @@ +package vdpa + +import ( + "fmt" + "syscall" + + "github.com/golang/mock/gomock" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + govdpaMock "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/govdpa/mock" + hostMock "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/mock" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" +) + +var _ = Describe("VDPA", func() { + var ( + v types.VdpaInterface + libMock *govdpaMock.MockGoVdpaLib + vdpaDevMock *govdpaMock.MockVdpaDevice + kernelMock *hostMock.MockHostManagerInterface + + testCtrl *gomock.Controller + testErr = fmt.Errorf("test-error") + ) + BeforeEach(func() { + testCtrl = gomock.NewController(GinkgoT()) + libMock = govdpaMock.NewMockGoVdpaLib(testCtrl) + vdpaDevMock = govdpaMock.NewMockVdpaDevice(testCtrl) + kernelMock = hostMock.NewMockHostManagerInterface(testCtrl) + v = New(kernelMock, libMock) + }) + AfterEach(func() { + testCtrl.Finish() + }) + Context("CreateVDPADevice", func() { + callFunc := func() error { + return v.CreateVDPADevice("0000:d8:00.2", constants.VdpaTypeVhost) + } + It("Created", func() { + libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(nil, syscall.ENODEV) + libMock.EXPECT().AddVdpaDevice("pci/"+"0000:d8:00.2", "vdpa:0000:d8:00.2").Return(nil) + kernelMock.EXPECT().BindDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.2", "vhost_vdpa").Return(nil) + Expect(callFunc()).NotTo(HaveOccurred()) + }) + It("Already exist", func() { + libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(vdpaDevMock, nil) + kernelMock.EXPECT().BindDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.2", "vhost_vdpa").Return(nil) + Expect(callFunc()).NotTo(HaveOccurred()) + }) + It("Fail to Get device", func() { + libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(nil, testErr) + Expect(callFunc()).To(MatchError(testErr)) + }) + It("Fail to Create device", func() { + libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(nil, syscall.ENODEV) + libMock.EXPECT().AddVdpaDevice("pci/"+"0000:d8:00.2", "vdpa:0000:d8:00.2").Return(testErr) + Expect(callFunc()).To(MatchError(testErr)) + }) + It("Fail to Bind device", func() { + libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(vdpaDevMock, nil) + kernelMock.EXPECT().BindDriverByBusAndDevice(consts.BusVdpa, "vdpa:0000:d8:00.2", "vhost_vdpa").Return(testErr) + Expect(callFunc()).To(MatchError(testErr)) + }) + }) + Context("DeleteVDPADevice", func() { + callFunc := func() error { + return v.DeleteVDPADevice("0000:d8:00.2") + } + It("Removed", func() { + libMock.EXPECT().DeleteVdpaDevice("vdpa:0000:d8:00.2").Return(nil) + Expect(callFunc()).NotTo(HaveOccurred()) + }) + It("Not found", func() { + libMock.EXPECT().DeleteVdpaDevice("vdpa:0000:d8:00.2").Return(syscall.ENODEV) + Expect(callFunc()).NotTo(HaveOccurred()) + }) + It("Fail to delete device", func() { + libMock.EXPECT().DeleteVdpaDevice("vdpa:0000:d8:00.2").Return(testErr) + Expect(callFunc()).To(MatchError(testErr)) + }) + }) + Context("DiscoverVDPAType", func() { + callFunc := func() string { + return v.DiscoverVDPAType("0000:d8:00.2") + } + It("No device", func() { + libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(nil, syscall.ENODEV) + Expect(callFunc()).To(BeEmpty()) + }) + It("No driver", func() { + vdpaDevMock.EXPECT().Driver().Return("") + libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(vdpaDevMock, nil) + Expect(callFunc()).To(BeEmpty()) + }) + It("Unknown driver", func() { + vdpaDevMock.EXPECT().Driver().Return("something") + libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(vdpaDevMock, nil) + Expect(callFunc()).To(BeEmpty()) + }) + It("Vhost driver", func() { + vdpaDevMock.EXPECT().Driver().Return("vhost_vdpa") + libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(vdpaDevMock, nil) + Expect(callFunc()).To(Equal("vhost")) + }) + It("Virtio driver", func() { + vdpaDevMock.EXPECT().Driver().Return("virtio_vdpa") + libMock.EXPECT().GetVdpaDevice("vdpa:0000:d8:00.2").Return(vdpaDevMock, nil) + Expect(callFunc()).To(Equal("virtio")) + }) + }) +}) diff --git a/pkg/host/manager.go b/pkg/host/manager.go index 3983fa9b6..f883ffe06 100644 --- a/pkg/host/manager.go +++ b/pkg/host/manager.go @@ -2,10 +2,13 @@ package host import ( "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/kernel" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/govdpa" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/network" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/service" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/sriov" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/udev" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/vdpa" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" ) @@ -19,6 +22,7 @@ type HostManagerInterface interface { types.ServiceInterface types.UdevInterface types.SriovInterface + types.VdpaInterface } type hostManager struct { @@ -28,6 +32,7 @@ type hostManager struct { types.ServiceInterface types.UdevInterface types.SriovInterface + types.VdpaInterface } func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { @@ -35,7 +40,8 @@ func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { n := network.New(utilsInterface) sv := service.New(utilsInterface) u := udev.New(utilsInterface) - sr := sriov.New(utilsInterface, k, n, u) + sr := sriov.New(utilsInterface, k, n, u, netlink.New()) + v := vdpa.New(k, govdpa.New()) return &hostManager{ utilsInterface, @@ -44,5 +50,6 @@ func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { sv, u, sr, + v, } } diff --git a/pkg/host/mock/mock_host.go b/pkg/host/mock/mock_host.go index 7c52fd28b..ddba8626e 100644 --- a/pkg/host/mock/mock_host.go +++ b/pkg/host/mock/mock_host.go @@ -53,6 +53,20 @@ func (mr *MockHostManagerInterfaceMockRecorder) AddUdevRule(pfPciAddress interfa return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddUdevRule", reflect.TypeOf((*MockHostManagerInterface)(nil).AddUdevRule), pfPciAddress) } +// AddVfRepresentorUdevRule mocks base method. +func (m *MockHostManagerInterface) AddVfRepresentorUdevRule(pfPciAddress, pfName, pfSwitchID, pfSwitchPort string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddVfRepresentorUdevRule", pfPciAddress, pfName, pfSwitchID, pfSwitchPort) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddVfRepresentorUdevRule indicates an expected call of AddVfRepresentorUdevRule. +func (mr *MockHostManagerInterfaceMockRecorder) AddVfRepresentorUdevRule(pfPciAddress, pfName, pfSwitchID, pfSwitchPort interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddVfRepresentorUdevRule", reflect.TypeOf((*MockHostManagerInterface)(nil).AddVfRepresentorUdevRule), pfPciAddress, pfName, pfSwitchID, pfSwitchPort) +} + // BindDefaultDriver mocks base method. func (m *MockHostManagerInterface) BindDefaultDriver(pciAddr string) error { m.ctrl.T.Helper() @@ -152,6 +166,34 @@ func (mr *MockHostManagerInterfaceMockRecorder) ConfigSriovInterfaces(storeManag return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigSriovInterfaces", reflect.TypeOf((*MockHostManagerInterface)(nil).ConfigSriovInterfaces), storeManager, interfaces, ifaceStatuses, pfsToConfig) } +// CreateVDPADevice mocks base method. +func (m *MockHostManagerInterface) CreateVDPADevice(pciAddr, vdpaType string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateVDPADevice", pciAddr, vdpaType) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateVDPADevice indicates an expected call of CreateVDPADevice. +func (mr *MockHostManagerInterfaceMockRecorder) CreateVDPADevice(pciAddr, vdpaType interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateVDPADevice", reflect.TypeOf((*MockHostManagerInterface)(nil).CreateVDPADevice), pciAddr, vdpaType) +} + +// DeleteVDPADevice mocks base method. +func (m *MockHostManagerInterface) DeleteVDPADevice(pciAddr string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteVDPADevice", pciAddr) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteVDPADevice indicates an expected call of DeleteVDPADevice. +func (mr *MockHostManagerInterfaceMockRecorder) DeleteVDPADevice(pciAddr interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteVDPADevice", reflect.TypeOf((*MockHostManagerInterface)(nil).DeleteVDPADevice), pciAddr) +} + // DiscoverSriovDevices mocks base method. func (m *MockHostManagerInterface) DiscoverSriovDevices(storeManager store.ManagerInterface) ([]v1.InterfaceExt, error) { m.ctrl.T.Helper() @@ -167,6 +209,20 @@ func (mr *MockHostManagerInterfaceMockRecorder) DiscoverSriovDevices(storeManage return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverSriovDevices", reflect.TypeOf((*MockHostManagerInterface)(nil).DiscoverSriovDevices), storeManager) } +// DiscoverVDPAType mocks base method. +func (m *MockHostManagerInterface) DiscoverVDPAType(pciAddr string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DiscoverVDPAType", pciAddr) + ret0, _ := ret[0].(string) + return ret0 +} + +// DiscoverVDPAType indicates an expected call of DiscoverVDPAType. +func (mr *MockHostManagerInterfaceMockRecorder) DiscoverVDPAType(pciAddr interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverVDPAType", reflect.TypeOf((*MockHostManagerInterface)(nil).DiscoverVDPAType), pciAddr) +} + // EnableRDMA mocks base method. func (m *MockHostManagerInterface) EnableRDMA(conditionFilePath, serviceName, packageManager string) (bool, error) { m.ctrl.T.Helper() @@ -687,6 +743,20 @@ func (mr *MockHostManagerInterfaceMockRecorder) RemoveUdevRule(pfPciAddress inte return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveUdevRule", reflect.TypeOf((*MockHostManagerInterface)(nil).RemoveUdevRule), pfPciAddress) } +// RemoveVfRepresentorUdevRule mocks base method. +func (m *MockHostManagerInterface) RemoveVfRepresentorUdevRule(pfPciAddress string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveVfRepresentorUdevRule", pfPciAddress) + ret0, _ := ret[0].(error) + return ret0 +} + +// RemoveVfRepresentorUdevRule indicates an expected call of RemoveVfRepresentorUdevRule. +func (mr *MockHostManagerInterfaceMockRecorder) RemoveVfRepresentorUdevRule(pfPciAddress interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveVfRepresentorUdevRule", reflect.TypeOf((*MockHostManagerInterface)(nil).RemoveVfRepresentorUdevRule), pfPciAddress) +} + // ResetSriovDevice mocks base method. func (m *MockHostManagerInterface) ResetSriovDevice(ifaceStatus v1.InterfaceExt) error { m.ctrl.T.Helper() @@ -715,6 +785,20 @@ func (mr *MockHostManagerInterfaceMockRecorder) SetNetdevMTU(pciAddr, mtu interf return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetNetdevMTU", reflect.TypeOf((*MockHostManagerInterface)(nil).SetNetdevMTU), pciAddr, mtu) } +// SetNicSriovMode mocks base method. +func (m *MockHostManagerInterface) SetNicSriovMode(pciAddr, mode string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetNicSriovMode", pciAddr, mode) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetNicSriovMode indicates an expected call of SetNicSriovMode. +func (mr *MockHostManagerInterfaceMockRecorder) SetNicSriovMode(pciAddr, mode interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetNicSriovMode", reflect.TypeOf((*MockHostManagerInterface)(nil).SetNicSriovMode), pciAddr, mode) +} + // SetSriovNumVfs mocks base method. func (m *MockHostManagerInterface) SetSriovNumVfs(pciAddr string, numVfs int) error { m.ctrl.T.Helper() diff --git a/pkg/host/types/interfaces.go b/pkg/host/types/interfaces.go index d1613cd44..6a12f29cd 100644 --- a/pkg/host/types/interfaces.go +++ b/pkg/host/types/interfaces.go @@ -132,6 +132,9 @@ type SriovInterface interface { // GetNicSriovMode returns the interface mode // supported modes SR-IOV legacy and switchdev GetNicSriovMode(pciAddr string) (string, error) + // SetNicSriovMode configure the interface mode + // supported modes SR-IOV legacy and switchdev + SetNicSriovMode(pciAddr, mode string) error // GetLinkType return the link type // supported types are ethernet and infiniband GetLinkType(ifaceStatus sriovnetworkv1.InterfaceExt) string @@ -154,8 +157,22 @@ type UdevInterface interface { // PrepareNMUdevRule creates the needed udev rules to disable NetworkManager from // our managed SR-IOV virtual functions PrepareNMUdevRule(supportedVfIds []string) error - // AddUdevRule adds a specific udev rule to the system + // AddUdevRule adds a udev rule that disables network-manager for VFs on the concrete PF AddUdevRule(pfPciAddress string) error - // RemoveUdevRule removes a udev rule from the system + // RemoveUdevRule removes a udev rule that disables network-manager for VFs on the concrete PF RemoveUdevRule(pfPciAddress string) error + // AddVfRepresentorUdevRule adds udev rule that renames VF representors on the concrete PF + AddVfRepresentorUdevRule(pfPciAddress, pfName, pfSwitchID, pfSwitchPort string) error + // RemoveVfRepresentorUdevRule removes udev rule that renames VF representors on the concrete PF + RemoveVfRepresentorUdevRule(pfPciAddress string) error +} + +type VdpaInterface interface { + // CreateVDPADevice creates VDPA device for VF with required type + CreateVDPADevice(pciAddr, vdpaType string) error + // DeleteVDPADevice removes VDPA device for provided pci address + DeleteVDPADevice(pciAddr string) error + // DiscoverVDPAType returns type of existing VDPA device for VF, + // returns empty string if VDPA device not found or unknown driver is in use + DiscoverVDPAType(pciAddr string) string } diff --git a/test/util/helpers/helpers.go b/test/util/helpers/helpers.go new file mode 100644 index 000000000..35f1bf22f --- /dev/null +++ b/test/util/helpers/helpers.go @@ -0,0 +1,34 @@ +package helpers + +import ( + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" +) + +// GinkgoConfigureFakeFS configure fake filesystem by setting vars.FilesystemRoot +// and register Ginkgo DeferCleanup handler to clean the fs when test completed +func GinkgoConfigureFakeFS(f *fakefilesystem.FS) { + var ( + cleanFakeFs func() + err error + ) + vars.FilesystemRoot, cleanFakeFs, err = f.Use() + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(cleanFakeFs) +} + +// GinkgoAssertFileContentsEquals check that content of the file +// match the expected value. +// prepends vars.FilesystemRoot to the file path to be compatible with +// GinkgoConfigureFakeFS function +func GinkgoAssertFileContentsEquals(path, expectedContent string) { + d, err := os.ReadFile(filepath.Join(vars.FilesystemRoot, path)) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, string(d)).To(Equal(expectedContent)) +}