From b51ba3bc61ae0c7fddd7ac1a87438b499a849d63 Mon Sep 17 00:00:00 2001 From: amaslennikov Date: Mon, 18 Mar 2024 12:01:03 +0300 Subject: [PATCH 1/5] Add LinkList() method to netlink library wrapper Signed-off-by: amaslennikov --- .../internal/lib/netlink/mock/mock_netlink.go | 15 ++++++++++++++ pkg/host/internal/lib/netlink/netlink.go | 20 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/pkg/host/internal/lib/netlink/mock/mock_netlink.go b/pkg/host/internal/lib/netlink/mock/mock_netlink.go index 367f57018..5b3bcc790 100644 --- a/pkg/host/internal/lib/netlink/mock/mock_netlink.go +++ b/pkg/host/internal/lib/netlink/mock/mock_netlink.go @@ -189,6 +189,21 @@ func (mr *MockNetlinkLibMockRecorder) LinkByName(name interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkByName", reflect.TypeOf((*MockNetlinkLib)(nil).LinkByName), name) } +// LinkList mocks base method. +func (m *MockNetlinkLib) LinkList() ([]netlink.Link, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LinkList") + ret0, _ := ret[0].([]netlink.Link) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LinkList indicates an expected call of LinkList. +func (mr *MockNetlinkLibMockRecorder) LinkList() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkList", reflect.TypeOf((*MockNetlinkLib)(nil).LinkList)) +} + // LinkSetMTU mocks base method. func (m *MockNetlinkLib) LinkSetMTU(link netlink.Link, mtu int) error { m.ctrl.T.Helper() diff --git a/pkg/host/internal/lib/netlink/netlink.go b/pkg/host/internal/lib/netlink/netlink.go index 4994c4bf1..ed063834e 100644 --- a/pkg/host/internal/lib/netlink/netlink.go +++ b/pkg/host/internal/lib/netlink/netlink.go @@ -26,6 +26,9 @@ type NetlinkLib interface { LinkByName(name string) (Link, error) // LinkByIndex finds a link by index and returns a pointer to the object. LinkByIndex(index int) (Link, error) + // LinkList gets a list of link devices. + // Equivalent to: `ip link show` + LinkList() ([]Link, error) // LinkSetVfHardwareAddr sets the hardware address of a vf for the link. // Equivalent to: `ip link set $link vf $vf mac $hwaddr` LinkSetVfHardwareAddr(link Link, vf int, hwaddr net.HardwareAddr) error @@ -91,6 +94,23 @@ func (w *libWrapper) LinkByIndex(index int) (Link, error) { return netlink.LinkByIndex(index) } +// LinkList gets a list of link devices. +// Equivalent to: `ip link show` +func (w *libWrapper) LinkList() ([]Link, error) { + links, err := netlink.LinkList() + if err != nil { + return nil, err + } + + // Convert each netlink.Link to the custom Link interface + customLinks := make([]Link, len(links)) + for i, link := range links { + customLinks[i] = link + } + + return customLinks, nil +} + // LinkSetVfHardwareAddr sets the hardware address of a vf for the link. // Equivalent to: `ip link set $link vf $vf mac $hwaddr` func (w *libWrapper) LinkSetVfHardwareAddr(link Link, vf int, hwaddr net.HardwareAddr) error { From 7116dc01c8b0d6addbd59c8a24bb4ebf2901ec3a Mon Sep 17 00:00:00 2001 From: amaslennikov Date: Tue, 26 Mar 2024 10:46:47 +0300 Subject: [PATCH 2/5] Move IB VF configuration to the IB interface switch sriov module to use the new interface Signed-off-by: amaslennikov --- pkg/helper/mock/mock_helper.go | 29 ++++++++++++++------------- pkg/host/internal/sriov/sriov.go | 26 ++++-------------------- pkg/host/internal/sriov/sriov_test.go | 12 ++++++++++- pkg/host/mock/mock_host.go | 29 ++++++++++++++------------- pkg/host/types/interfaces.go | 9 +++++++-- 5 files changed, 52 insertions(+), 53 deletions(-) diff --git a/pkg/helper/mock/mock_helper.go b/pkg/helper/mock/mock_helper.go index 5ef289ac4..a45e6e9e4 100644 --- a/pkg/helper/mock/mock_helper.go +++ b/pkg/helper/mock/mock_helper.go @@ -528,6 +528,21 @@ func (mr *MockHostHelpersInterfaceMockRecorder) GetOSPrettyName() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOSPrettyName", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetOSPrettyName)) } +// GetPciAddressFromInterfaceName mocks base method. +func (m *MockHostHelpersInterface) GetPciAddressFromInterfaceName(interfaceName string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPciAddressFromInterfaceName", interfaceName) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPciAddressFromInterfaceName indicates an expected call of GetPciAddressFromInterfaceName. +func (mr *MockHostHelpersInterfaceMockRecorder) GetPciAddressFromInterfaceName(interfaceName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPciAddressFromInterfaceName", reflect.TypeOf((*MockHostHelpersInterface)(nil).GetPciAddressFromInterfaceName), interfaceName) +} + // GetPhysPortName mocks base method. func (m *MockHostHelpersInterface) GetPhysPortName(name string) (string, error) { m.ctrl.T.Helper() @@ -1089,20 +1104,6 @@ func (mr *MockHostHelpersInterfaceMockRecorder) SetVfAdminMac(vfAddr, pfLink, vf return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetVfAdminMac", reflect.TypeOf((*MockHostHelpersInterface)(nil).SetVfAdminMac), vfAddr, pfLink, vfLink) } -// SetVfGUID mocks base method. -func (m *MockHostHelpersInterface) SetVfGUID(vfAddr string, pfLink netlink.Link) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetVfGUID", vfAddr, pfLink) - ret0, _ := ret[0].(error) - return ret0 -} - -// SetVfGUID indicates an expected call of SetVfGUID. -func (mr *MockHostHelpersInterfaceMockRecorder) SetVfGUID(vfAddr, pfLink interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetVfGUID", reflect.TypeOf((*MockHostHelpersInterface)(nil).SetVfGUID), vfAddr, pfLink) -} - // TriggerUdevEvent mocks base method. func (m *MockHostHelpersInterface) TriggerUdevEvent() error { m.ctrl.T.Helper() diff --git a/pkg/host/internal/sriov/sriov.go b/pkg/host/internal/sriov/sriov.go index 5a110e2ff..0d7b4c262 100644 --- a/pkg/host/internal/sriov/sriov.go +++ b/pkg/host/internal/sriov/sriov.go @@ -38,6 +38,7 @@ type sriov struct { networkHelper types.NetworkInterface udevHelper types.UdevInterface vdpaHelper types.VdpaInterface + infinibandHelper types.InfinibandInterface netlinkLib netlinkPkg.NetlinkLib dputilsLib dputilsPkg.DPUtilsLib sriovnetLib sriovnetPkg.SriovnetLib @@ -49,6 +50,7 @@ func New(utilsHelper utils.CmdInterface, networkHelper types.NetworkInterface, udevHelper types.UdevInterface, vdpaHelper types.VdpaInterface, + infinibandHelper types.InfinibandInterface, netlinkLib netlinkPkg.NetlinkLib, dputilsLib dputilsPkg.DPUtilsLib, sriovnetLib sriovnetPkg.SriovnetLib, @@ -58,6 +60,7 @@ func New(utilsHelper utils.CmdInterface, networkHelper: networkHelper, udevHelper: udevHelper, vdpaHelper: vdpaHelper, + infinibandHelper: infinibandHelper, netlinkLib: netlinkLib, dputilsLib: dputilsLib, sriovnetLib: sriovnetLib, @@ -163,27 +166,6 @@ func (s *sriov) getVfInfo(vfAddr string, pfName string, eswitchMode string, devi return vf } -func (s *sriov) SetVfGUID(vfAddr string, pfLink netlink.Link) error { - log.Log.Info("SetVfGUID()", "vf", vfAddr) - vfID, err := s.dputilsLib.GetVFID(vfAddr) - if err != nil { - log.Log.Error(err, "SetVfGUID(): unable to get VF id", "address", vfAddr) - return err - } - guid := utils.GenerateRandomGUID() - if err := s.netlinkLib.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil { - return err - } - if err := s.netlinkLib.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil { - return err - } - if err = s.kernelHelper.Unbind(vfAddr); err != nil { - return err - } - - return nil -} - func (s *sriov) VFIsReady(pciAddr string) (netlink.Link, error) { log.Log.Info("VFIsReady()", "device", pciAddr) var err error @@ -491,7 +473,7 @@ func (s *sriov) configSriovVFDevices(iface *sriovnetworkv1.Interface) error { linkType = s.GetLinkType(iface.Name) } if strings.EqualFold(linkType, consts.LinkTypeIB) { - if err = s.SetVfGUID(addr, pfLink); err != nil { + if err := s.infinibandHelper.ConfigureVfGUID(addr, iface.PciAddress, vfID, pfLink); err != nil { return err } } else { diff --git a/pkg/host/internal/sriov/sriov_test.go b/pkg/host/internal/sriov/sriov_test.go index 4a772bb52..6b81f9957 100644 --- a/pkg/host/internal/sriov/sriov_test.go +++ b/pkg/host/internal/sriov/sriov_test.go @@ -50,7 +50,7 @@ var _ = Describe("SRIOV", func() { hostMock = hostMockPkg.NewMockHostManagerInterface(testCtrl) storeManagerMode = hostStoreMockPkg.NewMockManagerInterface(testCtrl) - s = New(nil, hostMock, hostMock, hostMock, hostMock, netlinkLibMock, dputilsLibMock, sriovnetLibMock, ghwLibMock) + s = New(nil, hostMock, hostMock, hostMock, hostMock, nil, netlinkLibMock, dputilsLibMock, sriovnetLibMock, ghwLibMock) }) AfterEach(func() { @@ -227,6 +227,7 @@ var _ = Describe("SRIOV", func() { pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Flags: 0, EncapType: "ether"}) netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false) netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil) + netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2) hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(false, "") @@ -304,6 +305,7 @@ var _ = Describe("SRIOV", func() { vf0LinkMock := netlinkMockPkg.NewMockLink(testCtrl) netlinkLibMock.EXPECT().LinkSetVfNodeGUID(vf0LinkMock, 0, gomock.Any()).Return(nil) netlinkLibMock.EXPECT().LinkSetVfPortGUID(vf0LinkMock, 0, gomock.Any()).Return(nil) + netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil) @@ -370,6 +372,7 @@ var _ = Describe("SRIOV", func() { hostMock.EXPECT().AddVfRepresentorUdevRule("0000:d8:00.0", "enp216s0f0np0", "7cfe90ff2cc0", "p0").Return(nil) hostMock.EXPECT().CreateVDPADevice("0000:d8:00.2", "vhost_vdpa") hostMock.EXPECT().LoadUdevRules().Return(nil) + netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil) @@ -397,6 +400,8 @@ var _ = Describe("SRIOV", func() { It("externally managed - wrong VF count", func() { dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.0").Return(0) + netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() + Expect(s.ConfigSriovInterfaces(storeManagerMode, []sriovnetworkv1.Interface{{ Name: "enp216s0f0np0", @@ -421,6 +426,8 @@ var _ = Describe("SRIOV", func() { netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil) + netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() + hostMock.EXPECT().GetNetdevMTU("0000:d8:00.0") Expect(s.ConfigSriovInterfaces(storeManagerMode, []sriovnetworkv1.Interface{{ @@ -456,6 +463,7 @@ var _ = Describe("SRIOV", func() { netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil) + netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil) @@ -474,6 +482,7 @@ var _ = Describe("SRIOV", func() { helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs", "0") }) It("reset device - skip external", func() { + netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() storeManagerMode.EXPECT().LoadPfsStatus("0000:d8:00.0").Return(&sriovnetworkv1.Interface{ Name: "enp216s0f0np0", PciAddress: "0000:d8:00.0", @@ -502,6 +511,7 @@ var _ = Describe("SRIOV", func() { netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil) + netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil) diff --git a/pkg/host/mock/mock_host.go b/pkg/host/mock/mock_host.go index 45d5bc70f..d787a4640 100644 --- a/pkg/host/mock/mock_host.go +++ b/pkg/host/mock/mock_host.go @@ -452,6 +452,21 @@ func (mr *MockHostManagerInterfaceMockRecorder) GetOSPrettyName() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOSPrettyName", reflect.TypeOf((*MockHostManagerInterface)(nil).GetOSPrettyName)) } +// GetPciAddressFromInterfaceName mocks base method. +func (m *MockHostManagerInterface) GetPciAddressFromInterfaceName(interfaceName string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPciAddressFromInterfaceName", interfaceName) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPciAddressFromInterfaceName indicates an expected call of GetPciAddressFromInterfaceName. +func (mr *MockHostManagerInterfaceMockRecorder) GetPciAddressFromInterfaceName(interfaceName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPciAddressFromInterfaceName", reflect.TypeOf((*MockHostManagerInterface)(nil).GetPciAddressFromInterfaceName), interfaceName) +} + // GetPhysPortName mocks base method. func (m *MockHostManagerInterface) GetPhysPortName(name string) (string, error) { m.ctrl.T.Helper() @@ -918,20 +933,6 @@ func (mr *MockHostManagerInterfaceMockRecorder) SetVfAdminMac(vfAddr, pfLink, vf return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetVfAdminMac", reflect.TypeOf((*MockHostManagerInterface)(nil).SetVfAdminMac), vfAddr, pfLink, vfLink) } -// SetVfGUID mocks base method. -func (m *MockHostManagerInterface) SetVfGUID(vfAddr string, pfLink netlink.Link) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetVfGUID", vfAddr, pfLink) - ret0, _ := ret[0].(error) - return ret0 -} - -// SetVfGUID indicates an expected call of SetVfGUID. -func (mr *MockHostManagerInterfaceMockRecorder) SetVfGUID(vfAddr, pfLink interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetVfGUID", reflect.TypeOf((*MockHostManagerInterface)(nil).SetVfGUID), vfAddr, pfLink) -} - // TriggerUdevEvent mocks base method. func (m *MockHostManagerInterface) TriggerUdevEvent() error { m.ctrl.T.Helper() diff --git a/pkg/host/types/interfaces.go b/pkg/host/types/interfaces.go index 55d2fe620..ea2944938 100644 --- a/pkg/host/types/interfaces.go +++ b/pkg/host/types/interfaces.go @@ -110,6 +110,8 @@ type NetworkInterface interface { EnableHwTcOffload(ifaceName string) error // GetNetDevLinkAdminState returns the admin state of the interface. GetNetDevLinkAdminState(ifaceName string) string + // GetPciAddressFromInterfaceName parses sysfs to get pci address of an interface by name + GetPciAddressFromInterfaceName(interfaceName string) (string, error) } type ServiceInterface interface { @@ -135,8 +137,6 @@ type SriovInterface interface { // SetSriovNumVfs changes the number of virtual functions allocated for a specific // physical function base on pci address SetSriovNumVfs(pciAddr string, numVfs int) error - // SetVfGUID sets the GUID for a virtual function - SetVfGUID(vfAddr string, pfLink netlink.Link) error // VFIsReady returns the interface virtual function if the device is ready VFIsReady(pciAddr string) (netlink.Link, error) // SetVfAdminMac sets the virtual function administrative mac address via the physical function @@ -204,3 +204,8 @@ type BridgeInterface interface { // The function detach interface from managed bridges only. DetachInterfaceFromManagedBridge(pciAddr string) error } + +type InfinibandInterface interface { + // ConfigureVfGUID configures and sets a GUID for an IB VF device + ConfigureVfGUID(vfAddr string, pfAddr string, vfID int, pfLink netlink.Link) error +} From 0022c10c66204b7cb149592a6737af47464dc616 Mon Sep 17 00:00:00 2001 From: amaslennikov Date: Tue, 26 Mar 2024 10:48:06 +0300 Subject: [PATCH 3/5] Add implementation of Infiniband interface new package incapsulates the guid config file validation, guid pool implementation and host manipulations to set IB VF GUIDs --- pkg/consts/constants.go | 3 + pkg/host/internal/infiniband/guid.go | 62 ++++ pkg/host/internal/infiniband/guid_test.go | 29 ++ .../internal/infiniband/ib_guid_config.go | 126 ++++++++ .../infiniband/ib_guid_config_test.go | 285 ++++++++++++++++++ pkg/host/internal/infiniband/ib_guid_pool.go | 63 ++++ .../internal/infiniband/ib_guid_pool_test.go | 79 +++++ pkg/host/internal/infiniband/infiniband.go | 67 ++++ .../internal/infiniband/infiniband_test.go | 106 +++++++ pkg/host/internal/infiniband/suite_test.go | 21 ++ pkg/host/internal/network/network.go | 17 ++ pkg/host/internal/network/network_test.go | 12 + pkg/host/internal/sriov/sriov_test.go | 2 +- pkg/utils/utils.go | 15 - 14 files changed, 871 insertions(+), 16 deletions(-) create mode 100644 pkg/host/internal/infiniband/guid.go create mode 100644 pkg/host/internal/infiniband/guid_test.go create mode 100644 pkg/host/internal/infiniband/ib_guid_config.go create mode 100644 pkg/host/internal/infiniband/ib_guid_config_test.go create mode 100644 pkg/host/internal/infiniband/ib_guid_pool.go create mode 100644 pkg/host/internal/infiniband/ib_guid_pool_test.go create mode 100644 pkg/host/internal/infiniband/infiniband.go create mode 100644 pkg/host/internal/infiniband/infiniband_test.go create mode 100644 pkg/host/internal/infiniband/suite_test.go diff --git a/pkg/consts/constants.go b/pkg/consts/constants.go index af217cb01..cbfe9ad98 100644 --- a/pkg/consts/constants.go +++ b/pkg/consts/constants.go @@ -138,6 +138,9 @@ const ( // ManageSoftwareBridgesFeatureGate: enables management of software bridges by the operator ManageSoftwareBridgesFeatureGate = "manageSoftwareBridges" + + // The path to the file on the host filesystem that contains the IB GUID distribution for IB VFs + InfinibandGUIDConfigFilePath = SriovConfBasePath + "/infiniband/guids" ) const ( diff --git a/pkg/host/internal/infiniband/guid.go b/pkg/host/internal/infiniband/guid.go new file mode 100644 index 000000000..9cb2bffb6 --- /dev/null +++ b/pkg/host/internal/infiniband/guid.go @@ -0,0 +1,62 @@ +package infiniband + +import ( + "fmt" + "math/rand" + "net" +) + +// GUID address is an uint64 encapsulation for network hardware address +type GUID uint64 + +const ( + guidLength = 8 + byteBitLen = 8 + byteMask = 0xff +) + +// ParseGUID parses string only as GUID 64 bit +func ParseGUID(s string) (GUID, error) { + ha, err := net.ParseMAC(s) + if err != nil { + return 0, err + } + if len(ha) != guidLength { + return 0, fmt.Errorf("invalid GUID address %s", s) + } + var guid uint64 + for idx, octet := range ha { + guid |= uint64(octet) << uint(byteBitLen*(guidLength-1-idx)) + } + return GUID(guid), nil +} + +// String returns the string representation of GUID +func (g GUID) String() string { + return g.HardwareAddr().String() +} + +// HardwareAddr returns GUID representation as net.HardwareAddr +func (g GUID) HardwareAddr() net.HardwareAddr { + value := uint64(g) + ha := make(net.HardwareAddr, guidLength) + for idx := guidLength - 1; idx >= 0; idx-- { + ha[idx] = byte(value & byteMask) + value >>= byteBitLen + } + + return ha +} + +func generateRandomGUID() net.HardwareAddr { + guid := make(net.HardwareAddr, 8) + + // First field is 0x01 - xfe to avoid all zero and all F invalid guids + guid[0] = byte(1 + rand.Intn(0xfe)) + + for i := 1; i < len(guid); i++ { + guid[i] = byte(rand.Intn(0x100)) + } + + return guid +} diff --git a/pkg/host/internal/infiniband/guid_test.go b/pkg/host/internal/infiniband/guid_test.go new file mode 100644 index 000000000..f31d5d3d0 --- /dev/null +++ b/pkg/host/internal/infiniband/guid_test.go @@ -0,0 +1,29 @@ +package infiniband + +import ( + "net" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("GUID", func() { + It("should parse and process GUIDs correctly", func() { + guidStr := "00:01:02:03:04:05:06:08" + nextGuidStr := "00:01:02:03:04:05:06:09" + + guid, err := ParseGUID(guidStr) + Expect(err).NotTo(HaveOccurred()) + + Expect(guid.String()).To(Equal(guidStr)) + Expect((guid + 1).String()).To(Equal(nextGuidStr)) + }) + It("should represent GUID as HW address", func() { + guidStr := "00:01:02:03:04:05:06:08" + + guid, err := ParseGUID(guidStr) + Expect(err).NotTo(HaveOccurred()) + + Expect(guid.HardwareAddr()).To(Equal(net.HardwareAddr{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08})) + }) +}) diff --git a/pkg/host/internal/infiniband/ib_guid_config.go b/pkg/host/internal/infiniband/ib_guid_config.go new file mode 100644 index 000000000..261b40b5e --- /dev/null +++ b/pkg/host/internal/infiniband/ib_guid_config.go @@ -0,0 +1,126 @@ +package infiniband + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" +) + +type ibPfGUIDJSONConfig struct { + PciAddress string `json:"pciAddress,omitempty"` + PfGUID string `json:"pfGuid,omitempty"` + GUIDs []string `json:"guids,omitempty"` + GUIDsRange *GUIDRangeJSON `json:"guidsRange,omitempty"` +} + +type GUIDRangeJSON struct { + Start string `json:"start,omitempty"` + End string `json:"end,omitempty"` +} + +type ibPfGUIDConfig struct { + GUIDs []GUID + GUIDRange *GUIDRange +} + +type GUIDRange struct { + Start GUID + End GUID +} + +func getIbGUIDConfig(configPath string, netlinkLib netlink.NetlinkLib, networkHelper types.NetworkInterface) (map[string]ibPfGUIDConfig, error) { + links, err := netlinkLib.LinkList() + if err != nil { + return nil, err + } + rawConfigs, err := readJSONConfig(configPath) + if err != nil { + return nil, err + } + resultConfigs := map[string]ibPfGUIDConfig{} + // Parse JSON config into an internal struct + for _, rawConfig := range rawConfigs { + pciAddress, err := getPfPciAddressFromRawConfig(rawConfig, links, networkHelper) + if err != nil { + return nil, fmt.Errorf("failed to extract pci address from ib guid config: %w", err) + } + if len(rawConfig.GUIDs) == 0 && (rawConfig.GUIDsRange == nil || (rawConfig.GUIDsRange.Start == "" || rawConfig.GUIDsRange.End == "")) { + return nil, fmt.Errorf("either guid list or guid range should be provided, got none") + } + if len(rawConfig.GUIDs) != 0 && rawConfig.GUIDsRange != nil { + return nil, fmt.Errorf("either guid list or guid range should be provided, got both") + } + if rawConfig.GUIDsRange != nil && ((rawConfig.GUIDsRange.Start != "" && rawConfig.GUIDsRange.End == "") || (rawConfig.GUIDsRange.Start == "" && rawConfig.GUIDsRange.End != "")) { + return nil, fmt.Errorf("both guid rangeStart and rangeEnd should be provided, got one") + } + if len(rawConfig.GUIDs) != 0 { + var guids []GUID + for _, guidStr := range rawConfig.GUIDs { + guid, err := ParseGUID(guidStr) + if err != nil { + return nil, fmt.Errorf("failed to parse ib guid %s: %w", guidStr, err) + } + guids = append(guids, guid) + } + resultConfigs[pciAddress] = ibPfGUIDConfig{ + GUIDs: guids, + } + continue + } + + rangeStart, err := ParseGUID(rawConfig.GUIDsRange.Start) + if err != nil { + return nil, fmt.Errorf("failed to parse ib guid range start: %w", err) + } + rangeEnd, err := ParseGUID(rawConfig.GUIDsRange.End) + if err != nil { + return nil, fmt.Errorf("failed to parse ib guid range end: %w", err) + } + if rangeEnd < rangeStart { + return nil, fmt.Errorf("range end cannot be less then range start") + } + resultConfigs[pciAddress] = ibPfGUIDConfig{ + GUIDRange: &GUIDRange{ + Start: rangeStart, + End: rangeEnd, + }, + } + } + return resultConfigs, nil +} + +// readJSONConfig reads the file at the given path and unmarshals the contents into an array of ibPfGUIDJSONConfig structs +func readJSONConfig(configPath string) ([]ibPfGUIDJSONConfig, error) { + data, err := os.ReadFile(utils.GetHostExtensionPath(configPath)) + if err != nil { + return nil, fmt.Errorf("failed to read ib guid config: %w", err) + } + var configs []ibPfGUIDJSONConfig + if err := json.Unmarshal(data, &configs); err != nil { + return nil, fmt.Errorf("failed to unmarshal content of ib guid config: %w", err) + } + return configs, nil +} + +func getPfPciAddressFromRawConfig(pfRawConfig ibPfGUIDJSONConfig, links []netlink.Link, networkHelper types.NetworkInterface) (string, error) { + if pfRawConfig.PciAddress != "" && pfRawConfig.PfGUID != "" { + return "", fmt.Errorf("either PCI address or PF GUID required to describe an interface, both provided") + } + if pfRawConfig.PciAddress == "" && pfRawConfig.PfGUID == "" { + return "", fmt.Errorf("either PCI address or PF GUID required to describe an interface, none provided") + } + if pfRawConfig.PciAddress != "" { + return pfRawConfig.PciAddress, nil + } + // PfGUID is provided, need to resolve the pci address + for _, link := range links { + if link.Attrs().HardwareAddr.String() == pfRawConfig.PfGUID { + return networkHelper.GetPciAddressFromInterfaceName(link.Attrs().Name) + } + } + return "", fmt.Errorf("no matching link found for pf guid: %s", pfRawConfig.PfGUID) +} diff --git a/pkg/host/internal/infiniband/ib_guid_config_test.go b/pkg/host/internal/infiniband/ib_guid_config_test.go new file mode 100644 index 000000000..63f6b1683 --- /dev/null +++ b/pkg/host/internal/infiniband/ib_guid_config_test.go @@ -0,0 +1,285 @@ +package infiniband + +import ( + "fmt" + + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/vishvananda/netlink" + + netlinkLibPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" + netlinkMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink/mock" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/network" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/helpers" +) + +var _ = Describe("IbGuidConfig", func() { + Describe("readJSONConfig", Ordered, func() { + var ( + createJsonConfig func(string) string + ) + + BeforeEach(func() { + createJsonConfig = func(content string) string { + configPath := "/config.json" + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/host"}, + Files: map[string][]byte{"/host" + configPath: []byte(content)}, + }) + + return configPath + } + }) + + It("should correctly decode a JSON configuration file with all fields present", func() { + mockJsonConfig := `[{"pciAddress":"0000:00:00.0","pfGuid":"00:00:00:00:00:00:00:00","guids":["00:01:02:03:04:05:06:07", "00:01:02:03:04:05:06:08"],"guidsRange":{"start":"00:01:02:03:04:05:06:08","end":"00:01:02:03:04:05:06:FF"}}]` + + configPath := createJsonConfig(mockJsonConfig) + + configs, err := readJSONConfig(configPath) + Expect(err).NotTo(HaveOccurred()) + Expect(configs).To(HaveLen(1)) + Expect(configs[0].PciAddress).To(Equal("0000:00:00.0")) + Expect(configs[0].GUIDs).To(ContainElement("00:01:02:03:04:05:06:07")) + Expect(configs[0].GUIDs).To(ContainElement("00:01:02:03:04:05:06:08")) + Expect(configs[0].GUIDsRange.Start).To(Equal("00:01:02:03:04:05:06:08")) + Expect(configs[0].GUIDsRange.End).To(Equal("00:01:02:03:04:05:06:FF")) + + }) + It("should correctly decode a JSON configuration file with one field present", func() { + mockJsonConfig := `[{"pciAddress":"0000:00:00.0"}]` + + configPath := createJsonConfig(mockJsonConfig) + + configs, err := readJSONConfig(configPath) + Expect(err).NotTo(HaveOccurred()) + Expect(configs).To(HaveLen(1)) + Expect(configs[0].PciAddress).To(Equal("0000:00:00.0")) + }) + It("should correctly decode a JSON array with several elements", func() { + mockJsonConfig := `[{"pciAddress":"0000:00:00.0","guids":["00:01:02:03:04:05:06:07"]},{"pfGuid":"00:00:00:00:00:00:00:00","guidsRange":{"start":"00:01:02:03:04:05:06:08","end":"00:01:02:03:04:05:06:FF"}}]` + + configPath := createJsonConfig(mockJsonConfig) + + configs, err := readJSONConfig(configPath) + Expect(err).NotTo(HaveOccurred()) + Expect(configs).To(HaveLen(2)) + Expect(configs[0].PciAddress).To(Equal("0000:00:00.0")) + Expect(configs[1].PfGUID).To(Equal("00:00:00:00:00:00:00:00")) + Expect(configs[1].GUIDsRange.Start).To(Equal("00:01:02:03:04:05:06:08")) + Expect(configs[1].GUIDsRange.End).To(Equal("00:01:02:03:04:05:06:FF")) + }) + It("should fail on a non-array JSON", func() { + mockJsonConfig := `{"pciAddress":"0000:00:00.0", "newField": "newValue"}` + + configPath := createJsonConfig(mockJsonConfig) + + _, err := readJSONConfig(configPath) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("getPfPciAddressFromRawConfig", func() { + var ( + networkHelper types.NetworkInterface + ) + BeforeEach(func() { + networkHelper = network.New(nil, nil, nil, nil) + }) + It("should return same pci address when pci address is provided", func() { + pci, err := getPfPciAddressFromRawConfig(ibPfGUIDJSONConfig{PciAddress: "pciAddress"}, nil, networkHelper) + Expect(err).NotTo(HaveOccurred()) + Expect(pci).To(Equal("pciAddress")) + }) + It("should find correct pci address when pf guid is given", func() { + pfGuid := generateRandomGUID() + + testCtrl := gomock.NewController(GinkgoT()) + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Name: "ib216s0f0", HardwareAddr: pfGuid}).Times(2) + + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/sys/bus/pci/0000:3b:00.0", "/sys/class/net/ib216s0f0"}, + Symlinks: map[string]string{"/sys/class/net/ib216s0f0/device": "/sys/bus/pci/0000:3b:00.0"}, + }) + + pci, err := getPfPciAddressFromRawConfig(ibPfGUIDJSONConfig{PfGUID: pfGuid.String()}, []netlinkLibPkg.Link{pfLinkMock}, networkHelper) + Expect(err).NotTo(HaveOccurred()) + Expect(pci).To(Equal("0000:3b:00.0")) + + testCtrl.Finish() + }) + It("should return an error when no matching link is found", func() { + pfGuidDesired := generateRandomGUID() + pfGuidActual := generateRandomGUID() + + testCtrl := gomock.NewController(GinkgoT()) + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Name: "ib216s0f0", HardwareAddr: pfGuidActual}).Times(1) + + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/sys/bus/pci/0000:3b:00.0", "/sys/class/net/ib216s0f0"}, + Symlinks: map[string]string{"/sys/class/net/ib216s0f0/device": "/sys/bus/pci/0000:3b:00.0"}, + }) + + _, err := getPfPciAddressFromRawConfig(ibPfGUIDJSONConfig{PfGUID: pfGuidDesired.String()}, []netlinkLibPkg.Link{pfLinkMock}, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp(`no matching link found for pf guid:.*`))) + + testCtrl.Finish() + }) + It("should return an error when too many parameters are provided", func() { + _, err := getPfPciAddressFromRawConfig(ibPfGUIDJSONConfig{PfGUID: "pfGuid", PciAddress: "pciAddress"}, nil, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("either PCI address or PF GUID required to describe an interface, both provided")) + }) + It("should return an error when too few parameters are provided", func() { + _, err := getPfPciAddressFromRawConfig(ibPfGUIDJSONConfig{}, nil, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("either PCI address or PF GUID required to describe an interface, none provided")) + }) + }) + + Describe("getIbGUIDConfig", func() { + Describe("Tests without common mocks", func() { + It("should parse correct json config and return a map", func() { + pfGuid, _ := ParseGUID(generateRandomGUID().String()) + vfGuid1, _ := ParseGUID(generateRandomGUID().String()) + vfGuid2, _ := ParseGUID(generateRandomGUID().String()) + rangeStart, err := ParseGUID("00:01:02:03:04:05:06:08") + Expect(err).NotTo(HaveOccurred()) + rangeEnd, err := ParseGUID("00:01:02:03:04:05:06:FF") + Expect(err).NotTo(HaveOccurred()) + + configPath := "/config.json" + configStr := fmt.Sprintf(`[{"pciAddress":"0000:3b:00.1","guids":["%s", "%s"]},{"pfGuid":"%s","guidsRange":{"start":"%s","end":"%s"}}]`, vfGuid1.String(), vfGuid2.String(), pfGuid.String(), rangeStart.String(), rangeEnd.String()) + + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/sys/bus/pci/0000:3b:00.0", "/sys/class/net/ib216s0f0", "/host"}, + Symlinks: map[string]string{"/sys/class/net/ib216s0f0/device": "/sys/bus/pci/0000:3b:00.0"}, + Files: map[string][]byte{"/host" + configPath: []byte(configStr)}, + }) + + testCtrl := gomock.NewController(GinkgoT()) + + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Name: "ib216s0f0", HardwareAddr: pfGuid.HardwareAddr()}).Times(2) + + netlinkLibMock := netlinkMockPkg.NewMockNetlinkLib(testCtrl) + netlinkLibMock.EXPECT().LinkList().Return([]netlinkLibPkg.Link{pfLinkMock}, nil).Times(1) + + networkHelper := network.New(nil, nil, nil, nil) + + config, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).NotTo(HaveOccurred()) + Expect(config["0000:3b:00.1"].GUIDs[0]).To(Equal(vfGuid1)) + Expect(config["0000:3b:00.1"].GUIDs[1]).To(Equal(vfGuid2)) + Expect(config["0000:3b:00.0"].GUIDRange).To(Not(BeNil())) + Expect(config["0000:3b:00.0"].GUIDRange.Start).To(Equal(rangeStart)) + Expect(config["0000:3b:00.0"].GUIDRange.End).To(Equal(rangeEnd)) + + testCtrl.Finish() + }) + }) + + Describe("Tests with common mocks", func() { + var ( + netlinkLibMock *netlinkMockPkg.MockNetlinkLib + testCtrl *gomock.Controller + + createJsonConfig func(string) string + + networkHelper types.NetworkInterface + ) + + BeforeEach(func() { + createJsonConfig = func(content string) string { + configPath := "/config.json" + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/host"}, + Files: map[string][]byte{"/host" + configPath: []byte(content)}, + }) + + return configPath + } + + testCtrl = gomock.NewController(GinkgoT()) + netlinkLibMock = netlinkMockPkg.NewMockNetlinkLib(testCtrl) + netlinkLibMock.EXPECT().LinkList().Return([]netlinkLibPkg.Link{}, nil).Times(1) + + networkHelper = network.New(nil, nil, nil, nil) + }) + + AfterEach(func() { + testCtrl.Finish() + }) + + It("should return an error when invalid json config is provided", func() { + configPath := createJsonConfig(`[invalid file]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("failed to unmarshal content of ib guid config.*"))) + }) + It("should return an error when failed to determine pf's pci address", func() { + configPath := createJsonConfig(`[{"guids":["00:01:02:03:04:05:06:07"]}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("failed to extract pci address from ib guid config.*"))) + }) + It("should return an error when both guids and rangeStart are provided", func() { + configPath := createJsonConfig(`[{"pciAddress": "someaddress", "guids":["00:01:02:03:04:05:06:07"], "guidsRange":{"start": "00:01:02:03:04:05:06:AA"}}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("either guid list or guid range should be provided, got both.*"))) + }) + It("should return an error when both guids and rangeEnd are provided", func() { + configPath := createJsonConfig(`[{"pciAddress": "someaddress", "guids":["00:01:02:03:04:05:06:07"], "guidsRange":{"end": "00:01:02:03:04:05:06:AA"}}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("either guid list or guid range should be provided, got both.*"))) + }) + It("should return an error when neither guids nor range are provided", func() { + configPath := createJsonConfig(`[{"pciAddress": "someaddress"}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("either guid list or guid range should be provided, got none.*"))) + }) + It("should return an error when invalid guid list is provided", func() { + configPath := createJsonConfig(`[{"pciAddress": "someaddress", "guids":["invalid_guid"]}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("failed to parse ib guid invalid_guid.*"))) + }) + It("should return an error when invalid guid range start is provided", func() { + configPath := createJsonConfig(`[{"pciAddress": "someaddress", "guidsRange": {"start":"invalid range start", "end":"00:01:02:03:04:05:06:FF"}}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("failed to parse ib guid range start.*"))) + }) + It("should return an error when invalid guid range end is provided", func() { + configPath := createJsonConfig(`[{"pciAddress": "someaddress", "guidsRange": {"start":"00:01:02:03:04:05:06:08", "end":"invalid range end"}}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("failed to parse ib guid range end.*"))) + }) + It("should return an error when guid range end is less than range start", func() { + configPath := createJsonConfig(`[{"pciAddress": "someaddress", "guidsRange": {"start":"00:01:02:03:04:05:06:FF", "end":"00:01:02:03:04:05:06:AA"}}]`) + + _, err := getIbGUIDConfig(configPath, netlinkLibMock, networkHelper) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(MatchRegexp("range end cannot be less then range start.*"))) + }) + }) + }) +}) diff --git a/pkg/host/internal/infiniband/ib_guid_pool.go b/pkg/host/internal/infiniband/ib_guid_pool.go new file mode 100644 index 000000000..fa0b52528 --- /dev/null +++ b/pkg/host/internal/infiniband/ib_guid_pool.go @@ -0,0 +1,63 @@ +package infiniband + +import ( + "fmt" + "net" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" +) + +// ibGUIDPool is an interface that returns the GUID, allocated for a specific VF id of the specific PF +type ibGUIDPool interface { + // GetVFGUID returns the GUID, allocated for a specific VF id of the specific PF + // If no guid pool exists for the given pfPciAddr, returns an error + // If no guids are available for the given VF id, returns an error + GetVFGUID(pfPciAddr string, vfID int) (net.HardwareAddr, error) +} + +type ibGUIDPoolImpl struct { + guidConfigs map[string]ibPfGUIDConfig +} + +// newIbGUIDPool returns an instance of ibGUIDPool +func newIbGUIDPool(configPath string, netlinkLib netlink.NetlinkLib, networkHelper types.NetworkInterface) (ibGUIDPool, error) { + // All validation for the config file is done in the getIbGUIDConfig function + configs, err := getIbGUIDConfig(configPath, netlinkLib, networkHelper) + if err != nil { + return nil, fmt.Errorf("failed to create ib guid pool: %w", err) + } + + return &ibGUIDPoolImpl{guidConfigs: configs}, nil +} + +// GetVFGUID returns the GUID, allocated for a specific VF id of the specific PF +// If no guid pool exists for the given pfPciAddr, returns an error +// If no guids are available for the given VF id, returns an error +func (p *ibGUIDPoolImpl) GetVFGUID(pfPciAddr string, vfID int) (net.HardwareAddr, error) { + config, exists := p.guidConfigs[pfPciAddr] + if !exists { + return nil, fmt.Errorf("no guid pool for pci address: %s", pfPciAddr) + } + + if len(config.GUIDs) != 0 { + if vfID >= len(config.GUIDs) { + return nil, fmt.Errorf("no guid allocation found for VF id: %d on pf %s", vfID, pfPciAddr) + } + + guid := config.GUIDs[vfID] + + return guid.HardwareAddr(), nil + } + + if config.GUIDRange != nil { + nextGUID := config.GUIDRange.Start + GUID(vfID) + if nextGUID > config.GUIDRange.End { + return nil, fmt.Errorf("no guid allocation found for VF id: %d on pf %s", vfID, pfPciAddr) + } + + return nextGUID.HardwareAddr(), nil + } + + return nil, fmt.Errorf("no guid pool for pci address: %s", pfPciAddr) +} diff --git a/pkg/host/internal/infiniband/ib_guid_pool_test.go b/pkg/host/internal/infiniband/ib_guid_pool_test.go new file mode 100644 index 000000000..86878d21a --- /dev/null +++ b/pkg/host/internal/infiniband/ib_guid_pool_test.go @@ -0,0 +1,79 @@ +package infiniband + +import ( + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + netlinkLibPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" + netlinkMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink/mock" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/network" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/helpers" +) + +var _ = Describe("ibGUIDPool", Ordered, func() { + var ( + netlinkLibMock *netlinkMockPkg.MockNetlinkLib + testCtrl *gomock.Controller + + createJsonConfig func(string) string + + guidPool ibGUIDPool + ) + + BeforeAll(func() { + var err error + + createJsonConfig = func(content string) string { + configPath := "/config.json" + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/host"}, + Files: map[string][]byte{"/host" + configPath: []byte(content)}, + }) + + return configPath + } + + testCtrl = gomock.NewController(GinkgoT()) + netlinkLibMock = netlinkMockPkg.NewMockNetlinkLib(testCtrl) + netlinkLibMock.EXPECT().LinkList().Return([]netlinkLibPkg.Link{}, nil).Times(1) + + configPath := createJsonConfig( + `[{"pciAddress":"0000:3b:00.0","guids":["00:00:00:00:00:00:00:00", "00:00:00:00:00:00:00:01"]}, + {"pciAddress":"0000:3b:00.1","guidsRange":{"start":"00:00:00:00:00:00:01:00","end":"00:00:00:00:00:00:01:02"}}]`) + + guidPool, err = newIbGUIDPool(configPath, netlinkLibMock, network.New(nil, nil, nil, nil)) + Expect(err).NotTo(HaveOccurred()) + }) + + DescribeTable("check GetVFGUID function", + func(pfAddr string, vfID int, expectedError string, expectedGUID string) { + guid, err := guidPool.GetVFGUID(pfAddr, vfID) + if expectedError != "" { + Expect(err).To(MatchError(expectedError)) + } else { + Expect(err).NotTo(HaveOccurred()) + } + if expectedGUID != "" { + Expect(guid.String()).To(Equal(expectedGUID)) + } + }, + + Entry("Should get the first GUID out of the array", "0000:3b:00.0", 0, "", "00:00:00:00:00:00:00:00"), + Entry("Should get the last GUID out of the array", "0000:3b:00.0", 1, "", "00:00:00:00:00:00:00:01"), + Entry("Should get the same result when called again for an array GUID", "0000:3b:00.0", 0, "", "00:00:00:00:00:00:00:00"), + Entry("Should return a pool exhausted error when given a VF ID immediately out of bounds of a GUID array", "0000:3b:00.0", 2, "no guid allocation found for VF id: 2 on pf 0000:3b:00.0", ""), + Entry("Should return a pool exhausted error when given a VF ID, grossly out of bounds of a GUID array", "0000:3b:00.0", 5, "no guid allocation found for VF id: 5 on pf 0000:3b:00.0", ""), + Entry("Should get a correct GUID from the GUID range #1", "0000:3b:00.1", 0, "", "00:00:00:00:00:00:01:00"), + Entry("Should get a correct GUID from the GUID range #2", "0000:3b:00.1", 1, "", "00:00:00:00:00:00:01:01"), + Entry("Should get a correct GUID from the GUID range #3", "0000:3b:00.1", 2, "", "00:00:00:00:00:00:01:02"), + Entry("Should return a pool exhausted error when given a VF ID immediately out of bounds of a GUID range", "0000:3b:00.1", 3, "no guid allocation found for VF id: 3 on pf 0000:3b:00.1", ""), + Entry("Should return a pool exhausted error when given a VF ID, grossly out of bounds of a GUID range", "0000:3b:00.1", 5, "no guid allocation found for VF id: 5 on pf 0000:3b:00.1", ""), + Entry("Should get the same result when called again for an array GUID", "0000:3b:00.1", 1, "", "00:00:00:00:00:00:01:01"), + ) + + AfterAll(func() { + testCtrl.Finish() + }) +}) diff --git a/pkg/host/internal/infiniband/infiniband.go b/pkg/host/internal/infiniband/infiniband.go new file mode 100644 index 000000000..f54957bc3 --- /dev/null +++ b/pkg/host/internal/infiniband/infiniband.go @@ -0,0 +1,67 @@ +package infiniband + +import ( + "errors" + "fmt" + "io/fs" + "net" + + "github.com/vishvananda/netlink" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + netlinkLibPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/types" +) + +// New creates and returns an InfinibandInterface object, that handles IB VF GUID configuration +func New(netlinkLib netlinkLibPkg.NetlinkLib, kernelHelper types.KernelInterface, networkHelper types.NetworkInterface) (types.InfinibandInterface, error) { + guidPool, err := newIbGUIDPool(consts.InfinibandGUIDConfigFilePath, netlinkLib, networkHelper) + if err != nil { + // if config file doesn't exist, fallback to the random GUID generation + if errors.Is(err, fs.ErrNotExist) { + log.Log.Info("infiniband.New(): ib guid config doesn't exist, continuing without it", "config path", consts.InfinibandGUIDConfigFilePath) + return &infiniband{guidPool: nil, netlinkLib: netlinkLib, kernelHelper: kernelHelper}, nil + } + + return nil, fmt.Errorf("failed to create the ib guid pool: %w", err) + } + + return &infiniband{guidPool: guidPool, netlinkLib: netlinkLib, kernelHelper: kernelHelper}, nil +} + +type infiniband struct { + guidPool ibGUIDPool + netlinkLib netlinkLibPkg.NetlinkLib + kernelHelper types.KernelInterface +} + +// ConfigureVfGUID configures and sets a GUID for an IB VF device +func (i *infiniband) ConfigureVfGUID(vfAddr string, pfAddr string, vfID int, pfLink netlink.Link) error { + log.Log.Info("ConfigureVfGUID(): configure vf guid", "vfAddr", vfAddr, "pfAddr", pfAddr, "vfID", vfID) + + guid := generateRandomGUID() + + if i.guidPool != nil { + guidFromPool, err := i.guidPool.GetVFGUID(pfAddr, vfID) + if err != nil { + log.Log.Info("ConfigureVfGUID(): failed to get GUID from IB GUID pool", "address", vfAddr, "error", err) + return err + } + guid = guidFromPool + } + log.Log.Info("ConfigureVfGUID(): set vf guid", "address", vfAddr, "guid", guid) + + return i.applyVfGUIDToInterface(guid, vfAddr, vfID, pfLink) +} + +func (i *infiniband) applyVfGUIDToInterface(guid net.HardwareAddr, vfAddr string, vfID int, pfLink netlink.Link) error { + if err := i.netlinkLib.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil { + return err + } + if err := i.netlinkLib.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil { + return err + } + + return nil +} diff --git a/pkg/host/internal/infiniband/infiniband_test.go b/pkg/host/internal/infiniband/infiniband_test.go new file mode 100644 index 000000000..58b3a8fec --- /dev/null +++ b/pkg/host/internal/infiniband/infiniband_test.go @@ -0,0 +1,106 @@ +package infiniband + +import ( + "net" + + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" + netlinkLibPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink" + netlinkMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink/mock" + hostMockPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/mock" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/fakefilesystem" + "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/helpers" +) + +var _ = Describe("infiniband interface implementation", func() { + var ( + testCtrl *gomock.Controller + netlinkLibMock *netlinkMockPkg.MockNetlinkLib + hostMock *hostMockPkg.MockHostManagerInterface + ) + BeforeEach(func() { + testCtrl = gomock.NewController(GinkgoT()) + netlinkLibMock = netlinkMockPkg.NewMockNetlinkLib(testCtrl) + hostMock = hostMockPkg.NewMockHostManagerInterface(testCtrl) + }) + AfterEach(func() { + testCtrl.Finish() + }) + It("should create infiniband helper if guid config path is empty", func() { + netlinkLibMock.EXPECT().LinkList().Return([]netlinkLibPkg.Link{}, nil) + _, err := New(netlinkLibMock, hostMock, hostMock) + Expect(err).NotTo(HaveOccurred()) + }) + It("should assign guids if guid pool is nil", func() { + netlinkLibMock.EXPECT().LinkList().Return([]netlinkLibPkg.Link{}, nil) + var generatedGUID string + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + netlinkLibMock.EXPECT().LinkSetVfNodeGUID(pfLinkMock, 0, gomock.Any()).DoAndReturn( + func(link netlinkLibPkg.Link, vf int, nodeguid net.HardwareAddr) error { + // save generated GUID to validate that it is valid + generatedGUID = nodeguid.String() + return nil + }) + netlinkLibMock.EXPECT().LinkSetVfPortGUID(pfLinkMock, 0, gomock.Any()).Return(nil) + ib, err := New(netlinkLibMock, hostMock, hostMock) + Expect(err).NotTo(HaveOccurred()) + err = ib.ConfigureVfGUID("0000:d8:00.2", "0000:d8:00.0", 0, pfLinkMock) + Expect(err).NotTo(HaveOccurred()) + // validate that generated GUID is valid + _, err = ParseGUID(generatedGUID) + Expect(err).NotTo(HaveOccurred()) + }) + It("should assign guids if guid pool is not nil", func() { + var assignedGUID string + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + netlinkLibMock.EXPECT().LinkSetVfNodeGUID(pfLinkMock, 0, gomock.Any()).DoAndReturn( + func(link netlinkLibPkg.Link, vf int, nodeguid net.HardwareAddr) error { + // save generated GUID to validate that it is valid + assignedGUID = nodeguid.String() + return nil + }) + netlinkLibMock.EXPECT().LinkSetVfPortGUID(pfLinkMock, 0, gomock.Any()).Return(nil) + + guid, _ := ParseGUID("00:00:00:00:00:00:00:01") + pool := &ibGUIDPoolImpl{guidConfigs: map[string]ibPfGUIDConfig{"0000:d8:00.0": {GUIDs: []GUID{guid}}}} + + ib := &infiniband{guidPool: pool, netlinkLib: netlinkLibMock, kernelHelper: hostMock} + + err := ib.ConfigureVfGUID("0000:d8:00.2", "0000:d8:00.0", 0, pfLinkMock) + Expect(err).NotTo(HaveOccurred()) + // validate that generated GUID is valid + resultGUID, err := ParseGUID(assignedGUID) + Expect(err).NotTo(HaveOccurred()) + Expect(resultGUID).To(Equal(guid)) + }) + It("should read guids from the file", func() { + var assignedGUID string + netlinkLibMock.EXPECT().LinkList().Return([]netlinkLibPkg.Link{}, nil) + pfLinkMock := netlinkMockPkg.NewMockLink(testCtrl) + netlinkLibMock.EXPECT().LinkSetVfNodeGUID(pfLinkMock, 0, gomock.Any()).DoAndReturn( + func(link netlinkLibPkg.Link, vf int, nodeguid net.HardwareAddr) error { + // save generated GUID to validate that it is valid + assignedGUID = nodeguid.String() + return nil + }) + netlinkLibMock.EXPECT().LinkSetVfPortGUID(pfLinkMock, 0, gomock.Any()).Return(nil) + + mockJsonConfig := `[{"pciAddress":"0000:d8:00.0","guids":["00:01:02:03:04:05:06:07"]}]` + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/host" + consts.SriovConfBasePath + "/infiniband"}, + Files: map[string][]byte{"/host" + consts.InfinibandGUIDConfigFilePath: []byte(mockJsonConfig)}, + }) + + ib, err := New(netlinkLibMock, hostMock, hostMock) + Expect(err).NotTo(HaveOccurred()) + err = ib.ConfigureVfGUID("0000:d8:00.2", "0000:d8:00.0", 0, pfLinkMock) + Expect(err).NotTo(HaveOccurred()) + // validate that generated GUID is valid + resultGUID, err := ParseGUID(assignedGUID) + Expect(err).NotTo(HaveOccurred()) + Expect(resultGUID.String()).To(Equal("00:01:02:03:04:05:06:07")) + }) +}) diff --git a/pkg/host/internal/infiniband/suite_test.go b/pkg/host/internal/infiniband/suite_test.go new file mode 100644 index 000000000..fb584a6e0 --- /dev/null +++ b/pkg/host/internal/infiniband/suite_test.go @@ -0,0 +1,21 @@ +package infiniband + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "go.uber.org/zap/zapcore" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +func TestInfiniband(t *testing.T) { + log.SetLogger(zap.New( + zap.WriteTo(GinkgoWriter), + zap.Level(zapcore.Level(-2)), + zap.UseDevMode(true))) + RegisterFailHandler(Fail) + RunSpecs(t, "Package Infiniband Suite") +} diff --git a/pkg/host/internal/network/network.go b/pkg/host/internal/network/network.go index d04980eb8..b3014f9e9 100644 --- a/pkg/host/internal/network/network.go +++ b/pkg/host/internal/network/network.go @@ -404,3 +404,20 @@ func (n *network) GetNetDevLinkAdminState(ifaceName string) string { return consts.LinkAdminStateDown } + +// GetPciAddressFromInterfaceName parses sysfs to get pci address of an interface by name +func (n *network) GetPciAddressFromInterfaceName(interfaceName string) (string, error) { + log.Log.V(2).Info("GetPciAddressFromInterfaceName(): get pci address", "interface", interfaceName) + sysfsPath := filepath.Join(vars.FilesystemRoot, consts.SysClassNet, interfaceName, "device") + + pciDevDir, err := os.Readlink(sysfsPath) + + if err != nil { + log.Log.Error(err, "GetPciAddressFromInterfaceName(): failed to get pci device dir", "interface", interfaceName) + return "", err + } + + pciAddress := filepath.Base(pciDevDir) + log.Log.V(2).Info("GetPciAddressFromInterfaceName(): result", "interface", interfaceName, "pci address", pciAddress) + return pciAddress, nil +} diff --git a/pkg/host/internal/network/network_test.go b/pkg/host/internal/network/network_test.go index 350423745..19eb3f438 100644 --- a/pkg/host/internal/network/network_test.go +++ b/pkg/host/internal/network/network_test.go @@ -271,4 +271,16 @@ var _ = Describe("Network", func() { Expect(index).To(Equal(-1)) }) }) + Context("GetPciAddressFromInterfaceName", func() { + It("Should get PCI address from sys fs", func() { + helpers.GinkgoConfigureFakeFS(&fakefilesystem.FS{ + Dirs: []string{"/sys/bus/pci/0000:3b:00.0", "/sys/class/net/ib216s0f0"}, + Symlinks: map[string]string{"/sys/class/net/ib216s0f0/device": "/sys/bus/pci/0000:3b:00.0"}, + }) + + pci, err := n.GetPciAddressFromInterfaceName("ib216s0f0") + Expect(err).NotTo(HaveOccurred()) + Expect(pci).To(Equal("0000:3b:00.0")) + }) + }) }) diff --git a/pkg/host/internal/sriov/sriov_test.go b/pkg/host/internal/sriov/sriov_test.go index 6b81f9957..6d2e43a71 100644 --- a/pkg/host/internal/sriov/sriov_test.go +++ b/pkg/host/internal/sriov/sriov_test.go @@ -227,7 +227,7 @@ var _ = Describe("SRIOV", func() { pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Flags: 0, EncapType: "ether"}) netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false) netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() + netlinkLibMock.EXPECT().LinkList().Return(nil, nil).Times(0) dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2) hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(false, "") diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index e475bcff6..73d3ca995 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -3,8 +3,6 @@ package utils import ( "bytes" "fmt" - "math/rand" - "net" "os" "os/exec" "path/filepath" @@ -65,19 +63,6 @@ func (u *utilsHelper) RunCommand(command string, args ...string) (string, string return stdout.String(), stderr.String(), err } -func GenerateRandomGUID() net.HardwareAddr { - guid := make(net.HardwareAddr, 8) - - // First field is 0x01 - xfe to avoid all zero and all F invalid guids - guid[0] = byte(1 + rand.Intn(0xfe)) - - for i := 1; i < len(guid); i++ { - guid[i] = byte(rand.Intn(0x100)) - } - - return guid -} - func IsCommandNotFound(err error) bool { if exitErr, ok := err.(*exec.ExitError); ok { if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.ExitStatus() == 127 { From 2886ca62cb7ec3efb3cb3e6ad42055c84e478820 Mon Sep 17 00:00:00 2001 From: amaslennikov Date: Tue, 26 Mar 2024 10:49:23 +0300 Subject: [PATCH 4/5] Create instance of IB interface in the host object Signed-off-by: amaslennikov --- pkg/helper/host.go | 6 ++++- pkg/helper/mock/mock_helper.go | 14 ++++++++++ pkg/host/internal/sriov/sriov.go | 37 +++++++++++++++------------ pkg/host/internal/sriov/sriov_test.go | 22 +++++----------- pkg/host/manager.go | 14 +++++++--- pkg/host/mock/mock_host.go | 14 ++++++++++ pkg/platforms/platforms.go | 8 +++++- pkg/plugins/k8s/k8s_plugin_test.go | 2 +- 8 files changed, 78 insertions(+), 39 deletions(-) diff --git a/pkg/helper/host.go b/pkg/helper/host.go index 5829d161a..2e93a30d3 100644 --- a/pkg/helper/host.go +++ b/pkg/helper/host.go @@ -35,7 +35,11 @@ func NewHostHelpers(utilsHelper utils.CmdInterface, func NewDefaultHostHelpers() (HostHelpersInterface, error) { utilsHelper := utils.New() mlxHelper := mlx.New(utilsHelper) - hostManager := host.NewHostManager(utilsHelper) + hostManager, err := host.NewHostManager(utilsHelper) + if err != nil { + log.Log.Error(err, "failed to create host manager") + return nil, err + } storeManager, err := store.NewManager() if err != nil { log.Log.Error(err, "failed to create store manager") diff --git a/pkg/helper/mock/mock_helper.go b/pkg/helper/mock/mock_helper.go index a45e6e9e4..23b32a1d4 100644 --- a/pkg/helper/mock/mock_helper.go +++ b/pkg/helper/mock/mock_helper.go @@ -194,6 +194,20 @@ func (mr *MockHostHelpersInterfaceMockRecorder) ConfigSriovInterfaces(storeManag return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigSriovInterfaces", reflect.TypeOf((*MockHostHelpersInterface)(nil).ConfigSriovInterfaces), storeManager, interfaces, ifaceStatuses, skipVFConfiguration) } +// ConfigureVfGUID mocks base method. +func (m *MockHostHelpersInterface) ConfigureVfGUID(vfAddr, pfAddr string, vfID int, pfLink netlink.Link) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConfigureVfGUID", vfAddr, pfAddr, vfID, pfLink) + ret0, _ := ret[0].(error) + return ret0 +} + +// ConfigureVfGUID indicates an expected call of ConfigureVfGUID. +func (mr *MockHostHelpersInterfaceMockRecorder) ConfigureVfGUID(vfAddr, pfAddr, vfID, pfLink interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigureVfGUID", reflect.TypeOf((*MockHostHelpersInterface)(nil).ConfigureVfGUID), vfAddr, pfAddr, vfID, pfLink) +} + // CreateVDPADevice mocks base method. func (m *MockHostHelpersInterface) CreateVDPADevice(pciAddr, vdpaType string) error { m.ctrl.T.Helper() diff --git a/pkg/host/internal/sriov/sriov.go b/pkg/host/internal/sriov/sriov.go index 0d7b4c262..64f1fa537 100644 --- a/pkg/host/internal/sriov/sriov.go +++ b/pkg/host/internal/sriov/sriov.go @@ -33,16 +33,16 @@ type interfaceToConfigure struct { } type sriov struct { - utilsHelper utils.CmdInterface - kernelHelper types.KernelInterface - networkHelper types.NetworkInterface - udevHelper types.UdevInterface - vdpaHelper types.VdpaInterface + utilsHelper utils.CmdInterface + kernelHelper types.KernelInterface + networkHelper types.NetworkInterface + udevHelper types.UdevInterface + vdpaHelper types.VdpaInterface infinibandHelper types.InfinibandInterface - netlinkLib netlinkPkg.NetlinkLib - dputilsLib dputilsPkg.DPUtilsLib - sriovnetLib sriovnetPkg.SriovnetLib - ghwLib ghwPkg.GHWLib + netlinkLib netlinkPkg.NetlinkLib + dputilsLib dputilsPkg.DPUtilsLib + sriovnetLib sriovnetPkg.SriovnetLib + ghwLib ghwPkg.GHWLib } func New(utilsHelper utils.CmdInterface, @@ -56,15 +56,15 @@ func New(utilsHelper utils.CmdInterface, sriovnetLib sriovnetPkg.SriovnetLib, ghwLib ghwPkg.GHWLib) types.SriovInterface { return &sriov{utilsHelper: utilsHelper, - kernelHelper: kernelHelper, - networkHelper: networkHelper, - udevHelper: udevHelper, - vdpaHelper: vdpaHelper, + kernelHelper: kernelHelper, + networkHelper: networkHelper, + udevHelper: udevHelper, + vdpaHelper: vdpaHelper, infinibandHelper: infinibandHelper, - netlinkLib: netlinkLib, - dputilsLib: dputilsLib, - sriovnetLib: sriovnetLib, - ghwLib: ghwLib, + netlinkLib: netlinkLib, + dputilsLib: dputilsLib, + sriovnetLib: sriovnetLib, + ghwLib: ghwLib, } } @@ -476,6 +476,9 @@ func (s *sriov) configSriovVFDevices(iface *sriovnetworkv1.Interface) error { if err := s.infinibandHelper.ConfigureVfGUID(addr, iface.PciAddress, vfID, pfLink); err != nil { return err } + if err := s.kernelHelper.Unbind(iface.PciAddress); err != nil { + return err + } } else { vfLink, err := s.VFIsReady(addr) if err != nil { diff --git a/pkg/host/internal/sriov/sriov_test.go b/pkg/host/internal/sriov/sriov_test.go index 6d2e43a71..f89ecd3c5 100644 --- a/pkg/host/internal/sriov/sriov_test.go +++ b/pkg/host/internal/sriov/sriov_test.go @@ -35,8 +35,7 @@ var _ = Describe("SRIOV", func() { ghwLibMock *ghwMockPkg.MockGHWLib hostMock *hostMockPkg.MockHostManagerInterface storeManagerMode *hostStoreMockPkg.MockManagerInterface - - testCtrl *gomock.Controller + testCtrl *gomock.Controller testError = fmt.Errorf("test") ) @@ -50,7 +49,7 @@ var _ = Describe("SRIOV", func() { hostMock = hostMockPkg.NewMockHostManagerInterface(testCtrl) storeManagerMode = hostStoreMockPkg.NewMockManagerInterface(testCtrl) - s = New(nil, hostMock, hostMock, hostMock, hostMock, nil, netlinkLibMock, dputilsLibMock, sriovnetLibMock, ghwLibMock) + s = New(nil, hostMock, hostMock, hostMock, hostMock, hostMock, netlinkLibMock, dputilsLibMock, sriovnetLibMock, ghwLibMock) }) AfterEach(func() { @@ -227,7 +226,6 @@ var _ = Describe("SRIOV", func() { pfLinkMock.EXPECT().Attrs().Return(&netlink.LinkAttrs{Flags: 0, EncapType: "ether"}) netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false) netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).Times(0) dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2) hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(false, "") @@ -296,16 +294,14 @@ var _ = Describe("SRIOV", func() { netlinkLibMock.EXPECT().IsLinkAdminStateUp(pfLinkMock).Return(false) netlinkLibMock.EXPECT().LinkSetUp(pfLinkMock).Return(nil) - dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(2) - hostMock.EXPECT().Unbind("0000:d8:00.2").Return(nil) + dputilsLibMock.EXPECT().GetVFID("0000:d8:00.2").Return(0, nil).Times(1) hostMock.EXPECT().HasDriver("0000:d8:00.2").Return(true, "test").Times(2) hostMock.EXPECT().UnbindDriverIfNeeded("0000:d8:00.2", true).Return(nil) hostMock.EXPECT().BindDefaultDriver("0000:d8:00.2").Return(nil) hostMock.EXPECT().SetNetdevMTU("0000:d8:00.2", 2000).Return(nil) - vf0LinkMock := netlinkMockPkg.NewMockLink(testCtrl) - netlinkLibMock.EXPECT().LinkSetVfNodeGUID(vf0LinkMock, 0, gomock.Any()).Return(nil) - netlinkLibMock.EXPECT().LinkSetVfPortGUID(vf0LinkMock, 0, gomock.Any()).Return(nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() + hostMock.EXPECT().ConfigureVfGUID(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) + + hostMock.EXPECT().Unbind(gomock.Any()).Return(nil).Times(1) storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil) @@ -372,7 +368,6 @@ var _ = Describe("SRIOV", func() { hostMock.EXPECT().AddVfRepresentorUdevRule("0000:d8:00.0", "enp216s0f0np0", "7cfe90ff2cc0", "p0").Return(nil) hostMock.EXPECT().CreateVDPADevice("0000:d8:00.2", "vhost_vdpa") hostMock.EXPECT().LoadUdevRules().Return(nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() storeManagerMode.EXPECT().SaveLastPfAppliedStatus(gomock.Any()).Return(nil) @@ -400,7 +395,6 @@ var _ = Describe("SRIOV", func() { It("externally managed - wrong VF count", func() { dputilsLibMock.EXPECT().GetVFconfigured("0000:d8:00.0").Return(0) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() Expect(s.ConfigSriovInterfaces(storeManagerMode, []sriovnetworkv1.Interface{{ @@ -426,7 +420,6 @@ var _ = Describe("SRIOV", func() { netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() hostMock.EXPECT().GetNetdevMTU("0000:d8:00.0") Expect(s.ConfigSriovInterfaces(storeManagerMode, @@ -463,7 +456,6 @@ var _ = Describe("SRIOV", func() { netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil) @@ -482,7 +474,6 @@ var _ = Describe("SRIOV", func() { helpers.GinkgoAssertFileContentsEquals("/sys/bus/pci/devices/0000:d8:00.0/sriov_numvfs", "0") }) It("reset device - skip external", func() { - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() storeManagerMode.EXPECT().LoadPfsStatus("0000:d8:00.0").Return(&sriovnetworkv1.Interface{ Name: "enp216s0f0np0", PciAddress: "0000:d8:00.0", @@ -511,7 +502,6 @@ var _ = Describe("SRIOV", func() { netlinkLibMock.EXPECT().DevLinkGetDeviceByName("pci", "0000:d8:00.0").Return( &netlink.DevlinkDevice{Attrs: netlink.DevlinkDevAttrs{Eswitch: netlink.DevlinkDevEswitchAttr{Mode: "legacy"}}}, nil) - netlinkLibMock.EXPECT().LinkList().Return(nil, nil).AnyTimes() hostMock.EXPECT().RemoveDisableNMUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemovePersistPFNameUdevRule("0000:d8:00.0").Return(nil) hostMock.EXPECT().RemoveVfRepresentorUdevRule("0000:d8:00.0").Return(nil) diff --git a/pkg/host/manager.go b/pkg/host/manager.go index de7f6e3b6..b70f6b387 100644 --- a/pkg/host/manager.go +++ b/pkg/host/manager.go @@ -1,6 +1,7 @@ package host import ( + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/infiniband" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/kernel" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/ethtool" @@ -26,6 +27,7 @@ type HostManagerInterface interface { types.UdevInterface types.SriovInterface types.VdpaInterface + types.InfinibandInterface } type hostManager struct { @@ -36,9 +38,10 @@ type hostManager struct { types.UdevInterface types.SriovInterface types.VdpaInterface + types.InfinibandInterface } -func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { +func NewHostManager(utilsInterface utils.CmdInterface) (HostManagerInterface, error) { dpUtils := dputils.New() netlinkLib := netlink.New() ethtoolLib := ethtool.New() @@ -49,7 +52,11 @@ func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { sv := service.New(utilsInterface) u := udev.New(utilsInterface) v := vdpa.New(k, netlinkLib) - sr := sriov.New(utilsInterface, k, n, u, v, netlinkLib, dpUtils, sriovnetLib, ghwLib) + ib, err := infiniband.New(netlinkLib, k, n) + if err != nil { + return nil, err + } + sr := sriov.New(utilsInterface, k, n, u, v, ib, netlinkLib, dpUtils, sriovnetLib, ghwLib) return &hostManager{ utilsInterface, @@ -59,5 +66,6 @@ func NewHostManager(utilsInterface utils.CmdInterface) HostManagerInterface { u, sr, v, - } + ib, + }, nil } diff --git a/pkg/host/mock/mock_host.go b/pkg/host/mock/mock_host.go index d787a4640..72155f557 100644 --- a/pkg/host/mock/mock_host.go +++ b/pkg/host/mock/mock_host.go @@ -164,6 +164,20 @@ func (mr *MockHostManagerInterfaceMockRecorder) ConfigSriovInterfaces(storeManag return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigSriovInterfaces", reflect.TypeOf((*MockHostManagerInterface)(nil).ConfigSriovInterfaces), storeManager, interfaces, ifaceStatuses, skipVFConfiguration) } +// ConfigureVfGUID mocks base method. +func (m *MockHostManagerInterface) ConfigureVfGUID(vfAddr, pfAddr string, vfID int, pfLink netlink.Link) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConfigureVfGUID", vfAddr, pfAddr, vfID, pfLink) + ret0, _ := ret[0].(error) + return ret0 +} + +// ConfigureVfGUID indicates an expected call of ConfigureVfGUID. +func (mr *MockHostManagerInterfaceMockRecorder) ConfigureVfGUID(vfAddr, pfAddr, vfID, pfLink interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConfigureVfGUID", reflect.TypeOf((*MockHostManagerInterface)(nil).ConfigureVfGUID), vfAddr, pfAddr, vfID, pfLink) +} + // CreateVDPADevice mocks base method. func (m *MockHostManagerInterface) CreateVDPADevice(pciAddr, vdpaType string) error { m.ctrl.T.Helper() diff --git a/pkg/platforms/platforms.go b/pkg/platforms/platforms.go index 19c74cef3..3320dbaac 100644 --- a/pkg/platforms/platforms.go +++ b/pkg/platforms/platforms.go @@ -1,6 +1,8 @@ package platforms import ( + "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openstack" @@ -24,7 +26,11 @@ func NewDefaultPlatformHelper() (Interface, error) { return nil, err } utilsHelper := utils.New() - hostManager := host.NewHostManager(utilsHelper) + hostManager, err := host.NewHostManager(utilsHelper) + if err != nil { + log.Log.Error(err, "failed to create host manager") + return nil, err + } openstackContext := openstack.New(hostManager) return &platformHelper{ diff --git a/pkg/plugins/k8s/k8s_plugin_test.go b/pkg/plugins/k8s/k8s_plugin_test.go index 28a2e85c1..b4344b95f 100644 --- a/pkg/plugins/k8s/k8s_plugin_test.go +++ b/pkg/plugins/k8s/k8s_plugin_test.go @@ -79,7 +79,7 @@ var _ = Describe("K8s plugin", func() { testCtrl = gomock.NewController(GinkgoT()) hostHelper = mock_helper.NewMockHostHelpersInterface(testCtrl) - realHostMgr := host.NewHostManager(hostHelper) + realHostMgr, _ := host.NewHostManager(hostHelper) // proxy some functions to real host manager to simplify testing and to additionally validate manifests for _, f := range []string{ From f199eb95f46bbc278f56b8f31a54568f95ad96bf Mon Sep 17 00:00:00 2001 From: amaslennikov Date: Thu, 27 Jun 2024 20:48:23 +0300 Subject: [PATCH 5/5] Add doc article Signed-off-by: amaslennikov --- doc/ib-vf-guid-static-configuration.md | 90 ++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 doc/ib-vf-guid-static-configuration.md diff --git a/doc/ib-vf-guid-static-configuration.md b/doc/ib-vf-guid-static-configuration.md new file mode 100644 index 000000000..f0ae51fe2 --- /dev/null +++ b/doc/ib-vf-guid-static-configuration.md @@ -0,0 +1,90 @@ +# Infiniband VF GUID Static Configuration + +SR-IOV Network Operator is able to use a static configuration file from the host filesystem to assign GUIDs to IB VFs. + +## Prerequisites + +- Infiniband NICs +- Static GUID configuration file on the host filesystem + +### Configuration file + +Config file should be stored at `/etc/sriov-operator/infiniband/guids`. This location is writable across most cloud platforms. + +VF to GUID assignment, based on this file, is ordered. VF0 takes the GUID0, VF1 takes the GUID1 etc. + +Each PF has its own set of GUIDs. + +Example of the config file: + +```json +[ + { + "pci_address": "", + "guids": [ + "02:00:00:00:00:00:00:00", + "02:00:00:00:00:00:00:01" + ] + }, + { + "pf_guid": "", + "guidsRange": { + "start": "02:00:00:00:00:aa:00:02", + "end": "02:00:00:00:00:aa:00:0a" + } + } +] +``` + +Config file parameters: + +* `pci_address` is a PCI address of a PF +* `pf_guid` is a GUID of a PF. Can be obtained with `ibstat` command. +* `guids` is an array of VF GUID strings +* `guidsRange` is an object representing the start and end of a VF GUID range. It has two fields: + * `start` is a VF GUID range start + * `end` is a VF GUID range end + +Requirements for the config file: + +* `pci_address` and `pf_guid` cannot be set at the same time for a single device - should return an error +* if the list contains multiple entries for the same device, the first one shall be taken +* `rangeStart` and `rangeEnd` are both included in the range +* `guids` list and range cannot be both set at the same time for a single device - should return an error +* GUIDs are assigned once and not change throughout the lifecycle of the host + +### Deploy SriovNetworkNodePolicy + +```yaml +apiVersion: sriovnetwork.openshift.io/v1 +kind: SriovNetworkNodePolicy +metadata: + name: ib-policy + namespace: sriov-network-operator +spec: + nodeSelector: + feature.node.kubernetes.io/network-sriov.capable: "true" + resourceName: ib_vfs + priority: 10 + numVfs: 2 + nicSelector: + vendor: "15b3" + rootDevices: + - 0000:d8.00.0 + deviceType: netdevice +``` + +## Verify assigned GUIDs + +Run ip link tool to verify assigned GUIDs: + +```bash +ip link + +... +6: ibp216s0f0: mtu 4092 qdisc noop state UP mode DEFAULT group default qlen 256 + link/infiniband 00:00:05:a9:fe:80:00:00:00:00:00:00:b8:59:9f:03:00:f9:8f:86 brd 00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:ff:ff:ff:ff + vf 0 link/infiniband ... NODE_GUID 02:00:00:00:00:00:00:00, PORT_GUID 02:00:00:00:00:00:00:00, link-state enable, trust off, query_rss off + vf 1 link/infiniband ... NODE_GUID 02:00:00:00:00:00:00:01, PORT_GUID 02:00:00:00:00:00:00:01, link-state enable, trust off, query_rss off +``` +