From 022c33a7b7c157a98e29ba6178583264babc6d29 Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Mon, 22 Nov 2021 03:15:00 -0500 Subject: [PATCH] Reload VF driver only when rdma is enabled To set the GUID to VF, we need to set the admin MAC and reload the VF driver. It will take a long time if users set big 'numVFs'. This PR changes the logic to only reload the VF driver when isRdma is set. So that, users not using ROCE will no longer need to wait for the VF driver reload. --- api/v1/helper.go | 1 + api/v1/sriovnetworknodestate_types.go | 1 + ...k.openshift.io_sriovnetworknodestates.yaml | 2 + pkg/utils/utils.go | 105 ++++++++---------- 4 files changed, 52 insertions(+), 57 deletions(-) diff --git a/api/v1/helper.go b/api/v1/helper.go index c17a85718..3dee6681d 100644 --- a/api/v1/helper.go +++ b/api/v1/helper.go @@ -348,6 +348,7 @@ func (p *SriovNetworkNodePolicy) generateVfGroup(iface *InterfaceExt) (*VfGroup, VfRange: rng, PolicyName: p.GetName(), Mtu: p.Spec.Mtu, + IsRdma: p.Spec.IsRdma, }, nil } diff --git a/api/v1/sriovnetworknodestate_types.go b/api/v1/sriovnetworknodestate_types.go index abcc121ad..b3147ce9e 100644 --- a/api/v1/sriovnetworknodestate_types.go +++ b/api/v1/sriovnetworknodestate_types.go @@ -45,6 +45,7 @@ type VfGroup struct { VfRange string `json:"vfRange,omitempty"` PolicyName string `json:"policyName,omitempty"` Mtu int `json:"mtu,omitempty"` + IsRdma bool `json:"isRdma,omitempty"` } type Interfaces []Interface diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml index d2595edcb..8ccd4ef29 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml @@ -59,6 +59,8 @@ spec: properties: deviceType: type: string + isRdma: + type: boolean mtu: type: integer policyName: diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 181216dfa..b02793410 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -233,13 +233,6 @@ func configSriovDevice(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetwor glog.Errorf("configSriovDevice(): fail to set NumVfs for device %s", iface.PciAddress) return err } - if strings.EqualFold(iface.LinkType, "IB") { - if err = setVfsGuid(iface); err != nil { - return err - } - } else if err = setVfsAdminMac(ifaceStatus); err != nil { - return err - } } // set PF mtu if iface.Mtu > 0 && iface.Mtu != ifaceStatus.Mtu { @@ -255,22 +248,40 @@ func configSriovDevice(iface *sriovnetworkv1.Interface, ifaceStatus *sriovnetwor if err != nil { glog.Warningf("configSriovDevice(): unable to parse VFs for device %+v %q", iface.PciAddress, err) } + pfLink, err := netlink.LinkByName(iface.Name) + if err != nil { + glog.Errorf("setVfGuid(): unable to get PF link for device %+v %q", iface, err) + return err + } + for _, addr := range vfAddrs { var group sriovnetworkv1.VfGroup i := 0 - driver := "" + var driver string + var isRdma bool vfID, err := dputils.GetVFID(addr) for i, group = range iface.VfGroups { if err != nil { glog.Warningf("configSriovDevice(): unable to get VF id %+v %q", iface.PciAddress, err) } if sriovnetworkv1.IndexInRange(vfID, group.VfRange) { + isRdma = group.IsRdma if sriovnetworkv1.StringInArray(group.DeviceType, DpdkDrivers) { driver = group.DeviceType } break } } + if strings.EqualFold(iface.LinkType, "IB") { + if err = setVfGuid(addr, pfLink); err != nil { + return err + } + } else if err = setVfAdminMac(addr, pfLink); err != nil { + return err + } + if err = unbindDriverIfNeeded(addr, isRdma); err != nil { + return err + } if driver == "" { if err := BindDefaultDriver(addr); err != nil { glog.Warningf("configSriovDevice(): fail to bind default driver for device %s", addr) @@ -529,39 +540,33 @@ func vfIsReady(pciAddr string) (netlink.Link, error) { return vfLink, nil } -func setVfsAdminMac(iface *sriovnetworkv1.InterfaceExt) error { - glog.Infof("setVfsAdminMac(): device %s", iface.PciAddress) - pfLink, err := netlink.LinkByName(iface.Name) +func setVfAdminMac(vfAddr string, pfLink netlink.Link) error { + glog.Infof("setVfAdminMac(): VF %s", vfAddr) + + vfID, err := dputils.GetVFID(vfAddr) if err != nil { - glog.Errorf("setVfsAdminMac(): unable to get PF link for device %+v %q", iface, err) + glog.Errorf("setVfAdminMac(): unable to get VF id %+v %q", vfAddr, err) return err } - vfs, err := dputils.GetVFList(iface.PciAddress) + vfLink, err := vfIsReady(vfAddr) if err != nil { + glog.Errorf("setVfAdminMac(): VF link is not ready for device %+v %q", vfAddr, err) return err } - for _, addr := range vfs { - vfID, err := dputils.GetVFID(addr) - if err != nil { - glog.Errorf("setVfsAdminMac(): unable to get VF id %+v %q", iface.PciAddress, err) - return err - } - vfLink, err := vfIsReady(addr) - if err != nil { - glog.Errorf("setVfsAdminMac(): VF link is not ready for device %+v %q", addr, err) - return err - } - if err := netlink.LinkSetVfHardwareAddr(pfLink, vfID, vfLink.Attrs().HardwareAddr); err != nil { - return err - } - if err = Unbind(addr); err != nil { - return err - } - if err = BindDefaultDriver(addr); err != nil { + if err := netlink.LinkSetVfHardwareAddr(pfLink, vfID, vfLink.Attrs().HardwareAddr); err != nil { + return err + } + + return nil +} + +func unbindDriverIfNeeded(vfAddr string, isRdma bool) error { + if isRdma { + glog.Infof("unbindDriverIfNeeded(): unbind driver for %s", vfAddr) + if err := Unbind(vfAddr); err != nil { return err } } - return nil } @@ -584,36 +589,22 @@ func getLinkType(ifaceStatus sriovnetworkv1.InterfaceExt) string { return "" } -func setVfsGuid(iface *sriovnetworkv1.Interface) error { - glog.Infof("setVfsGuid(): device %s", iface.PciAddress) - pfLink, err := netlink.LinkByName(iface.Name) +func setVfGuid(vfAddr string, pfLink netlink.Link) error { + glog.Infof("setVfGuid(): VF %s", vfAddr) + vfID, err := dputils.GetVFID(vfAddr) if err != nil { - glog.Errorf("setVfsGuid(): unable to get PF link for device %+v %q", iface, err) + glog.Errorf("setVfGuid(): unable to get VF id %+v %q", vfAddr, err) return err } - vfs, err := dputils.GetVFList(iface.PciAddress) - if err != nil { + guid := generateRandomGuid() + if err := netlink.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil { return err } - for _, addr := range vfs { - vfID, err := dputils.GetVFID(addr) - if err != nil { - glog.Errorf("setVfsGuid(): unable to get VF id %+v %q", iface.PciAddress, err) - return err - } - guid := generateRandomGuid() - if err := netlink.LinkSetVfNodeGUID(pfLink, vfID, guid); err != nil { - return err - } - if err := netlink.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil { - return err - } - if err = Unbind(addr); err != nil { - return err - } - if err = BindDefaultDriver(addr); err != nil { - return err - } + if err := netlink.LinkSetVfPortGUID(pfLink, vfID, guid); err != nil { + return err + } + if err = Unbind(vfAddr); err != nil { + return err } return nil