Create instance of IB interface in the host object

Signed-off-by: amaslennikov <[email protected]>

pkg/consts/constants.go

@@ -104,6 +104,8 @@ const (
 	HostUdevRulesFolder = Host + UdevRulesFolder
 	UdevDisableNM       = "/bindata/scripts/udev-find-sriov-pf.sh"
 	UdevRepName         = "/bindata/scripts/switchdev-vf-link-name.sh"
+	// The path to the file on the host filesystem that contains the IB GUID distribution for IB VFs
+	InfinibandGUIDConfigFilePath = SriovConfBasePath + "/infiniband/guids"
 	// nolint:goconst
 	PFNameUdevRule = `SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", KERNELS=="%s", NAME="%s"`
 	// nolint:goconst
@@ -130,9 +132,6 @@ const (
 	// ResourceInjectorMatchConditionFeatureGate: switch injector to fail policy and add mactch condition
 	// this will make the mutating webhook to be called only when a pod has 'k8s.v1.cni.cncf.io/networks' annotation
 	ResourceInjectorMatchConditionFeatureGate = "resourceInjectorMatchCondition"
-
-	// The path to the file on the host filesystem that contains the IB GUID distribution for IB VFs
-	InfinibandGUIDConfigFilePath = SriovConfBasePath + "/infiniband/guids"
 )
 
 const (

pkg/helper/host.go

@@ -35,7 +35,11 @@ func NewHostHelpers(utilsHelper utils.CmdInterface,
 func NewDefaultHostHelpers() (HostHelpersInterface, error) {
 	utilsHelper := utils.New()
 	mlxHelper := mlx.New(utilsHelper)
-	hostManager := host.NewHostManager(utilsHelper)
+	hostManager, err := host.NewHostManager(utilsHelper)
+	if err != nil {
+		log.Log.Error(err, "failed to create host manager")
+		return nil, err
+	}
 	storeManager, err := store.NewManager()
 	if err != nil {
 		log.Log.Error(err, "failed to create store manager")

pkg/host/internal/sriov/sriov.go

@@ -476,6 +476,9 @@ func (s *sriov) configSriovVFDevices(iface *sriovnetworkv1.Interface) error {
 					if err := s.infinibandHelper.ConfigureVfGUID(addr, iface.PciAddress, vfID, pfLink); err != nil {
 						return err
 					}
+					if err := s.kernelHelper.Unbind(iface.PciAddress); err != nil {
+						return err
+					}
 				} else {
 					vfLink, err := s.VFIsReady(addr)
 					if err != nil {

pkg/host/internal/sriov/sriov_test.go

@@ -29,8 +29,7 @@ var _ = Describe("SRIOV", func() {
 		dputilsLibMock   *dputilsMockPkg.MockDPUtilsLib
 		hostMock         *hostMockPkg.MockHostManagerInterface
 		storeManagerMode *hostStoreMockPkg.MockManagerInterface
-
-		testCtrl *gomock.Controller
+		testCtrl         *gomock.Controller
 
 		testError = fmt.Errorf("test")
 	)
@@ -41,7 +40,7 @@ var _ = Describe("SRIOV", func() {
 		hostMock = hostMockPkg.NewMockHostManagerInterface(testCtrl)
 		storeManagerMode = hostStoreMockPkg.NewMockManagerInterface(testCtrl)
 
-		s = New(nil, hostMock, hostMock, hostMock, hostMock, nil, netlinkLibMock, dputilsLibMock)
+		s = New(nil, hostMock, hostMock, hostMock, hostMock, hostMock, netlinkLibMock, dputilsLibMock)
 	})
 
 	AfterEach(func() {
@@ -123,7 +122,6 @@ var _ = Describe("SRIOV", func() {
 			pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Flags: 0, EncapType: "ether"})
 			netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false)
 			netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil)
-			netlinkLibMock.EXPECT().LinkList().Return(nil, nil).Times(0)
 
 			dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2)
 			hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(false, "")
@@ -192,16 +190,12 @@ var _ = Describe("SRIOV", func() {
 			netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false)
 			netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil)
 
-			dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2)
-			hostMock.EXPECT().Unbind("0000:d8:00.2").Return(nil)
+			dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(1)
 			hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(true, "test").Times(2)
 			hostMock.EXPECT().UnbindDriverIfNeeded("0000:d8:00.2", true).Return(nil)
 			hostMock.EXPECT().BindDefaultDriver("0000:d8:00.2").Return(nil)
 			hostMock.EXPECT().SetNetdevMTU("0000:d8:00.2", 2000).Return(nil)
-			vf0LinkMock := netlinkMockPkg.NewMockLink(testCtrl)
-			netlinkLibMock.EXPECT().LinkSetVfNodeGUID(vf0LinkMock, 0, gomock.Any()).Return(nil)
-			netlinkLibMock.EXPECT().LinkSetVfPortGUID(vf0LinkMock, 0, gomock.Any()).Return(nil)
-			netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes()
+			hostMock.EXPECT().ConfigureVfGUID(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
 
 			storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil)
 
@@ -268,7 +262,6 @@ var _ = Describe("SRIOV", func() {
 			hostMock.EXPECT().AddVfRepresentorUdevRule("0000:d8:00.0", "enp216s0f0np0", "7cfe90ff2cc0", "p0").Return(nil)
 			hostMock.EXPECT().CreateVDPADevice("0000:d8:00.2", "vhost_vdpa")
 			hostMock.EXPECT().LoadUdevRules().Return(nil)
-			netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes()
 
 			storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil)
 
@@ -296,7 +289,6 @@ var _ = Describe("SRIOV", func() {
 
 		It("externally managed - wrong VF count", func() {
 			dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.0").Return(0)
-			netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes()
 
 			Expect(s.ConfigSriovInterfaces(storeManagerMode,
 				[]sriovnetworkv1.Interface{{
@@ -322,7 +314,6 @@ var _ = Describe("SRIOV", func() {
 			netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(
 				&netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}},
 				nil)
-			netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes()
 
 			hostMock.EXPECT().GetNetdevMTU("0000:d8:00.0")
 			Expect(s.ConfigSriovInterfaces(storeManagerMode,
@@ -358,7 +349,6 @@ var _ = Describe("SRIOV", func() {
 			netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(
 				&netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}},
 				nil)
-			netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes()
 			hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil)
 			hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil)
 			hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil)
@@ -377,7 +367,6 @@ var _ = Describe("SRIOV", func() {
 			helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs", "0")
 		})
 		It("reset device - skip external", func() {
-			netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes()
 			storeManagerMode.EXPECT().LoadPfsStatus("0000:d8:00.0").Return(&sriovnetworkv1.Interface{
 				Name:              "enp216s0f0np0",
 				PciAddress:        "0000:d8:00.0",
@@ -405,7 +394,6 @@ var _ = Describe("SRIOV", func() {
 			netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return(
 				&netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}},
 				nil)
-			netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes()
 			hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil)
 			hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil)
 			hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil)

pkg/host/manager.go

@@ -1,6 +1,7 @@
 package host
 
 import (
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/infiniband"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/kernel"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ethtool"
@@ -24,6 +25,7 @@ type HostManagerInterface interface {
 	types.UdevInterface
 	types.SriovInterface
 	types.VdpaInterface
+	types.InfinibandInterface
 }
 
 type hostManager struct {
@@ -34,9 +36,10 @@ type hostManager struct {
 	types.UdevInterface
 	types.SriovInterface
 	types.VdpaInterface
+	types.InfinibandInterface
 }
 
-func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface {
+func NewHostManager(utilsInterface utils.CmdInterface) (HostManagerInterface, error) {
 	dpUtils := dputils.New()
 	netlinkLib := netlink.New()
 	ethtoolLib := ethtool.New()
@@ -45,7 +48,11 @@ func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface {
 	sv := service.New(utilsInterface)
 	u := udev.New(utilsInterface)
 	v := vdpa.New(k, netlinkLib)
-	sr := sriov.New(utilsInterface, k, n, u, v, netlinkLib, dpUtils)
+	ib, err := infiniband.New(netlinkLib, k, n)
+	sr := sriov.New(utilsInterface, k, n, u, v, ib, netlinkLib, dpUtils)
+	if err != nil {
+		return nil, err
+	}
 
 	return &hostManager{
 		utilsInterface,
@@ -55,5 +62,6 @@ func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface {
 		u,
 		sr,
 		v,
-	}
+		ib,
+	}, nil
 }

pkg/platforms/platforms.go

@@ -1,6 +1,8 @@
 package platforms
 
 import (
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openstack"
@@ -24,7 +26,11 @@ func NewDefaultPlatformHelper() (Interface, error) {
 		return nil, err
 	}
 	utilsHelper := utils.New()
-	hostManager := host.NewHostManager(utilsHelper)
+	hostManager, err := host.NewHostManager(utilsHelper)
+	if err != nil {
+		log.Log.Error(err, "failed to create host manager")
+		return nil, err
+	}
 	openstackContext := openstack.New(hostManager)
 
 	return &platformHelper{

pkg/plugins/k8s/k8s_plugin_test.go

@@ -79,7 +79,7 @@ var _ = Describe("K8s plugin", func() {
 		testCtrl = gomock.NewController(GinkgoT())
 
 		hostHelper = mock_helper.NewMockHostHelpersInterface(testCtrl)
-		realHostMgr := host.NewHostManager(hostHelper)
+		realHostMgr, _ := host.NewHostManager(hostHelper)
 
 		// proxy some functions to real host manager to simplify testing and to additionally validate manifests
 		for _, f := range []string{