Skip to content

Commit

Permalink
Integrate IB GUID Pool into the sriov VF configuration flow
Browse files Browse the repository at this point in the history 
Signed-off-by: amaslennikov <[email protected]>
  • Loading branch information
@almaslennikov
almaslennikov committed Mar 18, 2024
1 parent bddb5da commit 18d5e84
Show file tree
Hide file tree
Showing 5 changed files with 45 additions and 21 deletions.
2 changes: 2 additions & 0 deletions pkg/consts/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ const (
KernelArgIommuPt = "iommu=pt"

ParallelNicConfigFeatureGate = "parallelNicConfig"

InfinibandGUIDConfigFilePath = "/host/var/opt/infiniband_guids"
)

const (
Expand Down
9 changes: 5 additions & 4 deletions pkg/helper/mock/mock_helper.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

42 changes: 30 additions & 12 deletions pkg/host/internal/sriov/sriov.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package sriov
import (
"errors"
"fmt"
"net"
"os"
"path/filepath"
"strconv"
Expand All @@ -17,6 +18,7 @@ import (

sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts"
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/infiniband"
dputilsPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/dputils"
netlinkPkg "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/internal/lib/netlink"
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/host/store"
Expand Down Expand Up @@ -145,14 +147,14 @@ func (s *sriov) GetVfInfo(pciAddr string, devices []*ghw.PCIDevice) sriovnetwork
return vf
}

func (s *sriov) SetVfGUID(vfAddr string, pfLink netlink.Link) error {
func (s *sriov) SetVfGUID(vfAddr string, guid net.HardwareAddr, pfLink netlink.Link) error {
log.Log.Info("SetVfGUID()", "vf", vfAddr)
vfID, err := s.dputilsLib.GetVFID(vfAddr)
if err != nil {
log.Log.Error(err, "SetVfGUID(): unable to get VF id", "address", vfAddr)
return err
}
guid := utils.GenerateRandomGUID()

if err := s.netlinkLib.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil {
return err
}
Expand Down Expand Up @@ -403,7 +405,7 @@ func (s *sriov) checkExternallyManagedPF(iface *sriovnetworkv1.Interface) error
return nil
}

func (s *sriov) configSriovVFDevices(iface *sriovnetworkv1.Interface) error {
func (s *sriov) configSriovVFDevices(iface *sriovnetworkv1.Interface, ibGUIDPool infiniband.IbGUIDPool) error {
log.Log.V(2).Info("configSriovVFDevices(): configure PF sriov device",
"device", iface.PciAddress)
if iface.NumVfs > 0 {
Expand Down Expand Up @@ -456,7 +458,18 @@ func (s *sriov) configSriovVFDevices(iface *sriovnetworkv1.Interface) error {
linkType = s.GetLinkType(iface.Name)
}
if strings.EqualFold(linkType, consts.LinkTypeIB) {
if err = s.SetVfGUID(addr, pfLink); err != nil {
guid := utils.GenerateRandomGUID()

if ibGUIDPool != nil {
guidFromPool, err := ibGUIDPool.GetNextFreeGUID(iface.PciAddress)
if err != nil {
log.Log.Info("configSriovVFDevices(): failed to get GUID from IB GUID pool, fallback to the random GUID", "address", addr, "error", err)
} else {
guid = guidFromPool
}
}

if err = s.SetVfGUID(addr, guid, pfLink); err != nil {
return err
}
} else {
Expand Down Expand Up @@ -527,7 +540,7 @@ func (s *sriov) configSriovVFDevices(iface *sriovnetworkv1.Interface) error {
return nil
}

func (s *sriov) configSriovDevice(iface *sriovnetworkv1.Interface, skipVFConfiguration bool) error {
func (s *sriov) configSriovDevice(iface *sriovnetworkv1.Interface, skipVFConfiguration bool, ibGUIDPool infiniband.IbGUIDPool) error {
log.Log.V(2).Info("configSriovDevice(): configure sriov device",
"device", iface.PciAddress, "config", iface, "skipVFConfiguration", skipVFConfiguration)
if !iface.ExternallyManaged {
Expand All @@ -552,7 +565,7 @@ func (s *sriov) configSriovDevice(iface *sriovnetworkv1.Interface, skipVFConfigu
return err
}
}
if err := s.configSriovVFDevices(iface); err != nil {
if err := s.configSriovVFDevices(iface, ibGUIDPool); err != nil {
return err
}
// Set PF link up
Expand Down Expand Up @@ -582,10 +595,15 @@ func (s *sriov) ConfigSriovInterfaces(storeManager store.ManagerInterface,
return fmt.Errorf("cannot get a list of interfaces to configure")
}

infinibandGUIDPool, err := infiniband.NewIbGUIDPool(consts.InfinibandGUIDConfigFilePath, s.netlinkLib)
if err != nil {
log.Log.Info("Cannot use infiniband guid pool", "error", err)
}

if vars.ParallelNicConfig {
err = s.configSriovInterfacesInParallel(storeManager, toBeConfigured, skipVFConfiguration)
err = s.configSriovInterfacesInParallel(storeManager, toBeConfigured, skipVFConfiguration, infinibandGUIDPool)
} else {
err = s.configSriovInterfaces(storeManager, toBeConfigured, skipVFConfiguration)
err = s.configSriovInterfaces(storeManager, toBeConfigured, skipVFConfiguration, infinibandGUIDPool)
}
if err != nil {
log.Log.Error(err, "cannot configure sriov interfaces")
Expand Down Expand Up @@ -640,7 +658,7 @@ func (s *sriov) getConfigureAndReset(storeManager store.ManagerInterface, interf
return toBeConfigured, toBeResetted, nil
}

func (s *sriov) configSriovInterfacesInParallel(storeManager store.ManagerInterface, interfaces []interfaceToConfigure, skipVFConfiguration bool) error {
func (s *sriov) configSriovInterfacesInParallel(storeManager store.ManagerInterface, interfaces []interfaceToConfigure, skipVFConfiguration bool, ibGUIDPool infiniband.IbGUIDPool) error {
log.Log.V(2).Info("configSriovInterfacesInParallel(): start sriov configuration")

var result error
Expand All @@ -650,7 +668,7 @@ func (s *sriov) configSriovInterfacesInParallel(storeManager store.ManagerInterf
interfacesToConfigure += 1
go func(iface *interfaceToConfigure) {
var err error
if err = s.configSriovDevice(&iface.iface, skipVFConfiguration); err != nil {
if err = s.configSriovDevice(&iface.iface, skipVFConfiguration, ibGUIDPool); err != nil {
log.Log.Error(err, "configSriovInterfacesInParallel(): fail to configure sriov interface. resetting interface.", "address", iface.iface.PciAddress)
if iface.iface.ExternallyManaged {
log.Log.V(2).Info("configSriovInterfacesInParallel(): skipping device reset as the nic is marked as externally created")
Expand Down Expand Up @@ -711,10 +729,10 @@ func (s *sriov) resetSriovInterfacesInParallel(storeManager store.ManagerInterfa
return nil
}

func (s *sriov) configSriovInterfaces(storeManager store.ManagerInterface, interfaces []interfaceToConfigure, skipVFConfiguration bool) error {
func (s *sriov) configSriovInterfaces(storeManager store.ManagerInterface, interfaces []interfaceToConfigure, skipVFConfiguration bool, ibGUIDPool infiniband.IbGUIDPool) error {
log.Log.V(2).Info("configSriovInterfaces(): start sriov configuration")
for _, iface := range interfaces {
if err := s.configSriovDevice(&iface.iface, skipVFConfiguration); err != nil {
if err := s.configSriovDevice(&iface.iface, skipVFConfiguration, ibGUIDPool); err != nil {
log.Log.Error(err, "configSriovInterfaces(): fail to configure sriov interface. resetting interface.", "address", iface.iface.PciAddress)
if iface.iface.ExternallyManaged {
log.Log.V(2).Info("configSriovInterfaces(): skipping device reset as the nic is marked as externally created")
Expand Down
9 changes: 5 additions & 4 deletions pkg/host/mock/mock_host.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion pkg/host/types/interfaces.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package types

import (
"net"

"github.com/coreos/go-systemd/v22/unit"
"github.com/jaypipes/ghw"
"github.com/vishvananda/netlink"
Expand Down Expand Up @@ -138,7 +140,7 @@ type SriovInterface interface {
// GetVfInfo returns the virtual function information is the operator struct from the host information
GetVfInfo(pciAddr string, devices []*ghw.PCIDevice) sriovnetworkv1.VirtualFunction
// SetVfGUID sets the GUID for a virtual function
SetVfGUID(vfAddr string, pfLink netlink.Link) error
SetVfGUID(vfAddr string, guid net.HardwareAddr, pfLink netlink.Link) error
// VFIsReady returns the interface virtual function if the device is ready
VFIsReady(pciAddr string) (netlink.Link, error)
// SetVfAdminMac sets the virtual function administrative mac address via the physical function
Expand Down

0 comments on commit 18d5e84

Please sign in to comment.