diff --git a/pkg/consts/constants.go b/pkg/consts/constants.go index 9ab23e098c..d8f006ef6c 100644 --- a/pkg/consts/constants.go +++ b/pkg/consts/constants.go @@ -104,6 +104,8 @@ const ( HostUdevRulesFolder = Host + UdevRulesFolder UdevDisableNM = "/bindata/scripts/udev-find-sriov-pf.sh" UdevRepName = "/bindata/scripts/switchdev-vf-link-name.sh" + // The path to the file on the host filesystem that contains the IB GUID distribution for IB VFs + InfinibandGUIDConfigFilePath = SriovConfBasePath + "/infiniband/guids" // nolint:goconst PFNameUdevRule = `SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", KERNELS=="%s", NAME="%s"` // nolint:goconst @@ -130,9 +132,6 @@ const ( // ResourceInjectorMatchConditionFeatureGate: switch injector to fail policy and add mactch condition // this will make the mutating webhook to be called only when a pod has 'k8s.v1.cni.cncf.io/networks' annotation ResourceInjectorMatchConditionFeatureGate = "resourceInjectorMatchCondition" - - // The path to the file on the host filesystem that contains the IB GUID distribution for IB VFs - InfinibandGUIDConfigFilePath = SriovConfBasePath + "/infiniband/guids" ) const ( diff --git a/pkg/helper/host.go b/pkg/helper/host.go index 5829d161a8..2e93a30d3d 100644 --- a/pkg/helper/host.go +++ b/pkg/helper/host.go @@ -35,7 +35,11 @@ func NewHostHelpers(utilsHelper utils.CmdInterface, func NewDefaultHostHelpers() (HostHelpersInterface, error) { utilsHelper := utils.New() mlxHelper := mlx.New(utilsHelper) - hostManager := host.NewHostManager(utilsHelper) + hostManager, err := host.NewHostManager(utilsHelper) + if err != nil { + log.Log.Error(err, "failed to create host manager") + return nil, err + } storeManager, err := store.NewManager() if err != nil { log.Log.Error(err, "failed to create store manager") diff --git a/pkg/helper/mock/mock_helper.go b/pkg/helper/mock/mock_helper.go index 22b2294c76..5170e29125 100644 --- a/pkg/helper/mock/mock_helper.go +++ b/pkg/helper/mock/mock_helper.go @@ -195,6 +195,20 @@ func (mr *MockHostHelpersInterfaceMockRecorder) ConfigSriovInterfaces(storeManag return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigSriovInterfaces", reflect.TypeOf((*MockHostHelpersInterface)(nil).ConfigSriovInterfaces), storeManager, interfaces, ifaceStatuses, skipVFConfiguration) } +// ConfigureVfGUID mocks base method. +func (m *MockHostHelpersInterface) ConfigureVfGUID(vfAddr, pfAddr string, vfID int, pfLink netlink.Link) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConfigureVfGUID", vfAddr, pfAddr, vfID, pfLink) + ret0, _ := ret[0].(error) + return ret0 +} + +// ConfigureVfGUID indicates an expected call of ConfigureVfGUID. +func (mr *MockHostHelpersInterfaceMockRecorder) ConfigureVfGUID(vfAddr, pfAddr, vfID, pfLink interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigureVfGUID", reflect.TypeOf((*MockHostHelpersInterface)(nil).ConfigureVfGUID), vfAddr, pfAddr, vfID, pfLink) +} + // CreateVDPADevice mocks base method. func (m *MockHostHelpersInterface) CreateVDPADevice(pciAddr, vdpaType string) error { m.ctrl.T.Helper() diff --git a/pkg/host/internal/sriov/sriov.go b/pkg/host/internal/sriov/sriov.go index fb7c09287f..43cd4ea746 100644 --- a/pkg/host/internal/sriov/sriov.go +++ b/pkg/host/internal/sriov/sriov.go @@ -476,6 +476,9 @@ func (s *sriov) configSriovVFDevices(iface *sriovnetworkv1.Interface) error { if err := s.infinibandHelper.ConfigureVfGUID(addr, iface.PciAddress, vfID, pfLink); err != nil { return err } + if err := s.kernelHelper.Unbind(iface.PciAddress); err != nil { + return err + } } else { vfLink, err := s.VFIsReady(addr) if err != nil { diff --git a/pkg/host/internal/sriov/sriov_test.go b/pkg/host/internal/sriov/sriov_test.go index c79ce75db8..fea389cdd4 100644 --- a/pkg/host/internal/sriov/sriov_test.go +++ b/pkg/host/internal/sriov/sriov_test.go @@ -29,8 +29,7 @@ var _ = Describe("SRIOV", func() { dputilsLibMock *dputilsMockPkg.MockDPUtilsLib hostMock *hostMockPkg.MockHostManagerInterface storeManagerMode *hostStoreMockPkg.MockManagerInterface - - testCtrl *gomock.Controller + testCtrl *gomock.Controller testError = fmt.Errorf("test") ) @@ -41,7 +40,7 @@ var _ = Describe("SRIOV", func() { hostMock = hostMockPkg.NewMockHostManagerInterface(testCtrl) storeManagerMode = hostStoreMockPkg.NewMockManagerInterface(testCtrl) - s = New(nil, hostMock, hostMock, hostMock, hostMock, nil, netlinkLibMock, dputilsLibMock) + s = New(nil, hostMock, hostMock, hostMock, hostMock, hostMock, netlinkLibMock, dputilsLibMock) }) AfterEach(func() { @@ -123,7 +122,6 @@ var _ = Describe("SRIOV", func() { pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Flags: 0, EncapType: "ether"}) netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false) netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).Times(0) dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2) hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(false, "") @@ -192,16 +190,12 @@ var _ = Describe("SRIOV", func() { netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false) netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil) - dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2) - hostMock.EXPECT().Unbind("0000:d8:00.2").Return(nil) + dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(1) hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(true, "test").Times(2) hostMock.EXPECT().UnbindDriverIfNeeded("0000:d8:00.2", true).Return(nil) hostMock.EXPECT().BindDefaultDriver("0000:d8:00.2").Return(nil) hostMock.EXPECT().SetNetdevMTU("0000:d8:00.2", 2000).Return(nil) - vf0LinkMock := netlinkMockPkg.NewMockLink(testCtrl) - netlinkLibMock.EXPECT().LinkSetVfNodeGUID(vf0LinkMock, 0, gomock.Any()).Return(nil) - netlinkLibMock.EXPECT().LinkSetVfPortGUID(vf0LinkMock, 0, gomock.Any()).Return(nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() + hostMock.EXPECT().ConfigureVfGUID(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil) @@ -268,7 +262,6 @@ var _ = Describe("SRIOV", func() { hostMock.EXPECT().AddVfRepresentorUdevRule("0000:d8:00.0", "enp216s0f0np0", "7cfe90ff2cc0", "p0").Return(nil) hostMock.EXPECT().CreateVDPADevice("0000:d8:00.2", "vhost_vdpa") hostMock.EXPECT().LoadUdevRules().Return(nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil) @@ -296,7 +289,6 @@ var _ = Describe("SRIOV", func() { It("externally managed - wrong VF count", func() { dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.0").Return(0) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() Expect(s.ConfigSriovInterfaces(storeManagerMode, []sriovnetworkv1.Interface{{ @@ -322,7 +314,6 @@ var _ = Describe("SRIOV", func() { netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() hostMock.EXPECT().GetNetdevMTU("0000:d8:00.0") Expect(s.ConfigSriovInterfaces(storeManagerMode, @@ -358,7 +349,6 @@ var _ = Describe("SRIOV", func() { netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil) @@ -377,7 +367,6 @@ var _ = Describe("SRIOV", func() { helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs", "0") }) It("reset device - skip external", func() { - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() storeManagerMode.EXPECT().LoadPfsStatus("0000:d8:00.0").Return(&sriovnetworkv1.Interface{ Name: "enp216s0f0np0", PciAddress: "0000:d8:00.0", @@ -405,7 +394,6 @@ var _ = Describe("SRIOV", func() { netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil) diff --git a/pkg/host/manager.go b/pkg/host/manager.go index a79331b533..661434548d 100644 --- a/pkg/host/manager.go +++ b/pkg/host/manager.go @@ -1,6 +1,7 @@ package host import ( + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/infiniband" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/kernel" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ethtool" @@ -24,6 +25,7 @@ type HostManagerInterface interface { types.UdevInterface types.SriovInterface types.VdpaInterface + types.InfinibandInterface } type hostManager struct { @@ -34,9 +36,10 @@ type hostManager struct { types.UdevInterface types.SriovInterface types.VdpaInterface + types.InfinibandInterface } -func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { +func NewHostManager(utilsInterface utils.CmdInterface) (HostManagerInterface, error) { dpUtils := dputils.New() netlinkLib := netlink.New() ethtoolLib := ethtool.New() @@ -45,7 +48,11 @@ func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { sv := service.New(utilsInterface) u := udev.New(utilsInterface) v := vdpa.New(k, netlinkLib) - sr := sriov.New(utilsInterface, k, n, u, v, netlinkLib, dpUtils) + ib, err := infiniband.New(netlinkLib, k, n) + sr := sriov.New(utilsInterface, k, n, u, v, ib, netlinkLib, dpUtils) + if err != nil { + return nil, err + } return &hostManager{ utilsInterface, @@ -55,5 +62,6 @@ func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { u, sr, v, - } + ib, + }, nil } diff --git a/pkg/host/mock/mock_host.go b/pkg/host/mock/mock_host.go index 2cec7fd6b0..b001b12e4d 100644 --- a/pkg/host/mock/mock_host.go +++ b/pkg/host/mock/mock_host.go @@ -165,6 +165,20 @@ func (mr *MockHostManagerInterfaceMockRecorder) ConfigSriovInterfaces(storeManag return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigSriovInterfaces", reflect.TypeOf((*MockHostManagerInterface)(nil).ConfigSriovInterfaces), storeManager, interfaces, ifaceStatuses, skipVFConfiguration) } +// ConfigureVfGUID mocks base method. +func (m *MockHostManagerInterface) ConfigureVfGUID(vfAddr, pfAddr string, vfID int, pfLink netlink.Link) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConfigureVfGUID", vfAddr, pfAddr, vfID, pfLink) + ret0, _ := ret[0].(error) + return ret0 +} + +// ConfigureVfGUID indicates an expected call of ConfigureVfGUID. +func (mr *MockHostManagerInterfaceMockRecorder) ConfigureVfGUID(vfAddr, pfAddr, vfID, pfLink interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigureVfGUID", reflect.TypeOf((*MockHostManagerInterface)(nil).ConfigureVfGUID), vfAddr, pfAddr, vfID, pfLink) +} + // CreateVDPADevice mocks base method. func (m *MockHostManagerInterface) CreateVDPADevice(pciAddr, vdpaType string) error { m.ctrl.T.Helper() diff --git a/pkg/platforms/platforms.go b/pkg/platforms/platforms.go index 19c74cef30..3320dbaaca 100644 --- a/pkg/platforms/platforms.go +++ b/pkg/platforms/platforms.go @@ -1,6 +1,8 @@ package platforms import ( + "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openstack" @@ -24,7 +26,11 @@ func NewDefaultPlatformHelper() (Interface, error) { return nil, err } utilsHelper := utils.New() - hostManager := host.NewHostManager(utilsHelper) + hostManager, err := host.NewHostManager(utilsHelper) + if err != nil { + log.Log.Error(err, "failed to create host manager") + return nil, err + } openstackContext := openstack.New(hostManager) return &platformHelper{ diff --git a/pkg/plugins/k8s/k8s_plugin_test.go b/pkg/plugins/k8s/k8s_plugin_test.go index 28a2e85c1f..b4344b95fc 100644 --- a/pkg/plugins/k8s/k8s_plugin_test.go +++ b/pkg/plugins/k8s/k8s_plugin_test.go @@ -79,7 +79,7 @@ var _ = Describe("K8s plugin", func() { testCtrl = gomock.NewController(GinkgoT()) hostHelper = mock_helper.NewMockHostHelpersInterface(testCtrl) - realHostMgr := host.NewHostManager(hostHelper) + realHostMgr, _ := host.NewHostManager(hostHelper) // proxy some functions to real host manager to simplify testing and to additionally validate manifests for _, f := range []string{