Skip to content

Commit

Permalink
Add driver change for nvme
Browse files Browse the repository at this point in the history
  • Loading branch information
@mowangdk
mowangdk committed Oct 30, 2024
1 parent 8d21509 commit 811a254
Show file tree
Hide file tree
Showing 4 changed files with 126 additions and 65 deletions.
2 changes: 1 addition & 1 deletion pkg/cloud/metadata/k8s.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ var (
"alibabacloud.com/ecs-instance-id",
"sigma.ali/ecs-instance-id",
}
VmocLabels = []string {
VmocLabels = []string{
"rm.alibaba-inc.com/vbm",
}
)
Expand Down
67 changes: 46 additions & 21 deletions pkg/disk/bdf.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@ import (
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"syscall"
"time"
"regexp"

"github.com/aliyun/alibaba-cloud-sdk-go/services/ecs"
"github.com/kubernetes-sigs/alibaba-cloud-csi-driver/pkg/utils"
Expand Down Expand Up @@ -534,6 +534,7 @@ type Driver interface {
UnbindDriver() error
BindDriver(targetDriver string) error
GetDeviceNumber() string
GetPCIDeviceDriverType() string
CheckVFIOUsage() error
}

Expand All @@ -544,21 +545,28 @@ func NewDeviceDriver(volumeId, blockDevice, deviceNumber string, _type MachineTy
machineType: _type,
extras: extras,
}
deviceNumberFromDevice := ""
if blockDevice != "" {
if deviceNumber == "" {
busRegex, err := d.machineType.BusRegex()
if err != nil {
klog.Errorf("NewDeviceDriver: get device number from block device err: %v", err)
return nil, err
}
deviceNumber, err := DefaultDeviceManager.GetDeviceNumberFromBlockDevice(blockDevice, busRegex)
if err != nil {
klog.Errorf("NewDeviceDriver: get device number from block device err: %v", err)
klog.Infof("NewDeviceDriver: start to get deviceNumber from device: %s", blockDevice)
busRegex, err := d.machineType.BusRegex()
if err != nil {
klog.Errorf("NewDeviceDriver: get bus type: %v", err)
return nil, err
}
deviceNumberFromDevice, err = DefaultDeviceManager.GetDeviceNumberFromBlockDevice(blockDevice, busRegex)
if err != nil {
klog.Errorf("NewDeviceDriver: get device number from block device err: %v", err)
if deviceNumber == "" {
return nil, err
}
d.deviceNumber = deviceNumber
}
}
}
if deviceNumberFromDevice != "" {
if deviceNumber != "" && deviceNumberFromDevice != deviceNumber {
klog.Warningf("NewDeviceDriver: newGeneratedDeviceNumber: %s is different from the one from exists file: %s, override with new deviceNumber", deviceNumberFromDevice, deviceNumber)
}
d.deviceNumber = deviceNumberFromDevice
}
if d.deviceNumber != "" {
return d, nil
}
Expand Down Expand Up @@ -588,7 +596,7 @@ func NewDeviceDriver(volumeId, blockDevice, deviceNumber string, _type MachineTy
} else {
output, err := utils.CommandOnNode("xdragon-bdf", "--nvme", "-id=%s", volumeId).CombinedOutput()
if err != nil {
return nil, fmt.Errorf("Failed to excute bdf command: %s, err: %v", volumeId, err)
return nil, fmt.Errorf("Failed to excute bdf command: %s, err: %v", volumeId, err)
}
d.deviceNumber = string(output)
}
Expand Down Expand Up @@ -623,7 +631,26 @@ func (d *driver) UnbindDriver() error {
}

func (d *driver) BindDriver(targetDriver string) error {
return utilsio.WriteTrunc(unix.AT_FDCWD, filepath.Join(sysPrefix, "sys/bus", d.machineType.BusName(), "drivers", targetDriver, "bind"), []byte(d.deviceNumber))
err := utilsio.WriteTrunc(unix.AT_FDCWD, filepath.Join(sysPrefix, "sys/bus", d.machineType.BusName(), "devices", d.deviceNumber, "driver_override"), []byte(targetDriver))
if err != nil {
return err
}
if d.machineType == BDF {
return utilsio.WriteTrunc(unix.AT_FDCWD, filepath.Join(sysPrefix, "sys/bus", d.machineType.BusName(), "drivers_probe"), []byte(d.deviceNumber))
}
return nil
}

func(d *driver) GetPCIDeviceDriverType() string {
output, _ := exec.Command("lspci", "-s", d.deviceNumber, "-n").CombinedOutput()
klog.InfoS("GetDeviceDriverType: get driver type output", "deviceNumber", d.deviceNumber, "output", output)
// #define PCI_DEVICE_ID_VIRTIO_BLOCK 0x1001
// #define PCI_DEVICE_ID_ALIBABA_NVME 0×5004
if strings.HasSuffix(strings.TrimSpace(string(output)), "1001") {
return PCITypeVIRTIO
} else {
return PCITypeNVME
}
}

func (d *driver) CheckVFIOUsage() error {
Expand All @@ -633,12 +660,10 @@ func (d *driver) CheckVFIOUsage() error {
}
klog.V(5).InfoS("CheckVFIOUsage: eval symlink success", "path", actualPath)
groupNumber := filepath.Base(actualPath)
output, err := exec.Command("lsof", filepath.Join(sysPrefix, "dev/vfio", groupNumber)).CombinedOutput()
if err != nil {
return err
}
if string(output) != "" {
return errors.Errorf("CheckVFIOUsage: device: %s is still be in used, return error", d.deviceNumber)
// the command returns -1 if nothing is returned
output, _ := exec.Command("lsof", filepath.Join("/dev/vfio", groupNumber)).CombinedOutput()
if strings.TrimSpace(string(output)) != "" {
return errors.Errorf("CheckVFIOUsage: device: %s is still be in used, output: %s", d.deviceNumber, output)
}
return nil
}
}
4 changes: 2 additions & 2 deletions pkg/disk/device_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import (
"fmt"
"os"
"path/filepath"
"strings"
"regexp"
"strings"

utilsio "github.com/kubernetes-sigs/alibaba-cloud-csi-driver/pkg/utils/io"
"golang.org/x/sys/unix"
Expand Down Expand Up @@ -271,7 +271,7 @@ func (m *DeviceManager) GetDeviceNumberFromBlockDevice(blockDevice string, busRe
return "", fmt.Errorf("NewDeviceDriver: not found device number, blockDevice: %s", blockDevice)
}
parentDir := filepath.Base(filepath.Dir(dirEntry))

matched := busRegex.MatchString(parentDir)
klog.Infof("NewDeviceDriver: busPrefix: %s, parentDir: %s, matched: %v", busRegex.String(), parentDir, matched)
if matched {
Expand Down
118 changes: 77 additions & 41 deletions pkg/disk/nodeserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ import (
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
"syscall"
"regexp"

"github.com/container-storage-interface/spec/lib/go/csi"
"github.com/kubernetes-sigs/alibaba-cloud-csi-driver/pkg/cloud/metadata"
Expand All @@ -50,6 +50,7 @@ import (
"k8s.io/mount-utils"
k8smount "k8s.io/mount-utils"
utilexec "k8s.io/utils/exec"
"k8s.io/apimachinery/pkg/util/sets"
)

type nodeServer struct {
Expand Down Expand Up @@ -125,6 +126,8 @@ const (
PCITypeVFIO = "vfio-pci"
// PCITypeVIRTIO defines pci bus virtio driver type
PCITypeVIRTIO = "virtio-pci"
// PCITypeVIRTIO defines pci bus nvme driver type
PCITypeNVME = "nvme"
)

var (
Expand All @@ -137,11 +140,14 @@ var (
// but setting trusted xaattr requires CAP_SYS_ADMIN capability, we may use user namespace instead in unit tests.
DiskXattrName = "trusted.csi-managed-disk"


// BDFTypeDevice defines the regexp of bdf number
BDFTypeDevice = regexp.MustCompile(`^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}`)
BDFTypeDevice = regexp.MustCompile(`^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}`)
// DFBusTypeDevice defines the regexp of dfnumber
DFBusTypeDevice = regexp.MustCompile(`^dfvirtio.*`)


vfioDrivers = sets.New[string](DFBusTypeVFIO, PCITypeVFIO)
defaultDrivers = sets.New[string](PCITypeNVME, PCITypeVIRTIO, DFBusTypeVIRTIO)
)

// QueryResponse response struct for query server
Expand Down Expand Up @@ -1222,54 +1228,74 @@ func (ns *nodeServer) umountRunDVolumes(volumePath string) (bool, error) {
}

mountInfo, isRunD3 := directvolume.IsRunD3VolumePath(filepath.Dir(volumePath))
if ns.kataBMIOType == DFBus {
if !isRunD3 {
return false, status.Error(codes.Internal, "vmoc(DFBus) mode only support csi-runD protocol 3.0")
if isRunD3 {
removeRunD3File := func() error {
klog.Infof("NodeUnPublishVolume:: start delete mount info for KataVolume: %s", volumePath)
err := directvolume.Remove(filepath.Dir(volumePath))
if err != nil {
klog.Errorf("NodeUnPublishVolume:: Failed to remove DirectVolume mount info, potentially disrupting kubelet's next operation: %v", err)
}
return err
}

d, _ := NewDeviceDriver("", "", mountInfo.Source, DFBus, nil)
var d Driver
if ns.kataBMIOType == DFBus {
d, _ = NewDeviceDriver("", "", mountInfo.Source, DFBus, nil)
} else {
d, _ = NewDeviceDriver("", "", mountInfo.Source, BDF, nil)
}
cDriver, err := d.CurentDriver()
if err != nil {
if IsNoSuchFileErr(err) {
klog.Infof("vmoc(DFBus) dfbus driver has been removed: %s", DFBus)
klog.Infof("driver has been removed, device: %s has empty driver", mountInfo.Source)
if err = removeRunD3File(); err != nil {
return true, status.Error(codes.Internal, err.Error())
}
return true, nil
}
return true, status.Error(codes.Internal, err.Error())
}
klog.Infof("vmoc(DFBus) current dfbus driver : %v", DFBus)
if cDriver == DFBusTypeVIRTIO {
klog.Infof("current device driver : %v", cDriver)
if defaultDrivers.Has(cDriver) {
if err = removeRunD3File(); err != nil {
return true, status.Error(codes.Internal, err.Error())
}
return true, nil
}
if cDriver != DFBusTypeVFIO {
return true, status.Error(codes.Internal, "vmoc(DFBus) mode only support vfio, virtio driver")
if !vfioDrivers.Has(cDriver) {
return true, status.Error(codes.Internal, "vmoc(DFBus) mode only support vfio, virtio, nvme driver")
}
// Check driver usage before unbind
// Check driver usage before unbind
if err = d.CheckVFIOUsage(); err != nil {
return true, status.Errorf(codes.Internal, "vmoc(DFBus) still being used, return immediately, err: %s", err.Error())
}

klog.Infof("start to unbind driver: %v of device: %v", cDriver, mountInfo.Source)
err = d.UnbindDriver()
if err != nil {
return true, status.Errorf(codes.Internal, "vmoc(DFBus) unbind err: %s", err.Error())
}
err = d.BindDriver(DFBusTypeVIRTIO)
if err != nil {
return true, status.Errorf(codes.Internal, "vmoc(DFBus) bind err: %s", err.Error())
klog.Infof("start to rebind the driver of device: %v", mountInfo.Source)
if ns.kataBMIOType == DFBus {
err = d.BindDriver(DFBusTypeVIRTIO)
if err != nil {
return true, status.Errorf(codes.Internal, "vmoc(DFBus) bind err: %s", err.Error())
}
} else {
err = d.BindDriver(d.GetPCIDeviceDriverType())
if err != nil {
return true, status.Errorf(codes.Internal, "vmoc(DFBus) bind err: %s", err.Error())
}
}
}

klog.Infof("NodeUnPublishVolume:: start delete mount info for KataVolume: %s", volumePath)
if isRunD3 {
err := directvolume.Remove(filepath.Dir(volumePath))
if err != nil {
klog.Errorf("NodeUnPublishVolume:: Remove mount info for DirectVolume failed: %v", err)
if err = removeRunD3File(); err != nil {
return true, status.Error(codes.Internal, err.Error())
}
return true, nil
}

klog.Infof("NodeUnPublishVolume:: start delete mount info for DirectVolume: %s", volumePath)
if directvolume.IsRunD2VolumePath(volumePath) {
klog.Infof("NodeUnPublishVolume: Path: %s is already mounted in csi runD 3.0 mode", volumePath)
klog.Infof("NodeUnPublishVolume: Path: %s is already mounted in csi runD 2.0 mode", volumePath)
if err := os.Remove(filepath.Join(volumePath, directvolume.RunD2MountInfoFileName)); err != nil {
if os.IsNotExist(err) {
return false, nil
Expand Down Expand Up @@ -1335,7 +1361,6 @@ func (ns *nodeServer) mountRunDVolumes(volumeId, pvName, sourcePath, targetPath,
if err != nil {
deviceName = getVolumeConfig(volumeId)
}
deviceNumber := ""
volumePath := filepath.Dir(targetPath)

if features.FunctionalMutableFeatureGate.Enabled(features.RundCSIProtocol3) {
Expand All @@ -1345,16 +1370,25 @@ func (ns *nodeServer) mountRunDVolumes(volumeId, pvName, sourcePath, targetPath,
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: unmountStageTarget "+sourcePath+" with error: "+err.Error())
}

klog.InfoS("mountRunDVolumes: ", "deviceName", deviceName)
deviceNumber := ""
if volumeMount, exists := directvolume.IsRunD3VolumePath(volumePath); exists {
deviceNumber = volumeMount.Source
}

klog.InfoS("mountRunDVolumes: ", "deviceName", deviceName, "deviceNumber", deviceNumber)
driver, err := NewDeviceDriver(volumeId, deviceName, deviceNumber, ns.kataBMIOType, map[string]string{})
if err != nil {
klog.Errorf("NodePublishVolume(rund3.0): can't get bdf number of volume: %s: err: %v", volumeId, err)
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot get bdf number of volume: "+volumeId)
}
currentDriver, err := driver.CurentDriver()
cDriver, err := driver.CurentDriver()
if err != nil {
return true, status.Errorf(codes.Internal, "NodePublishVolume(rund3.0): can't get current volume driver: %+v", err)
}
if deviceNumber == driver.GetDeviceNumber() && vfioDrivers.Has(cDriver) {
klog.InfoS("NodePublishVolume(rund3.0): volume are already mounted, return normally", "volumeId", volumeId, "deviceNumber", deviceNumber)
return true, nil
}
deviceType := directvolume.DeviceTypePCI
if ns.kataBMIOType == DFBus {
deviceType = directvolume.DeviceTypeDFBusPort
Expand Down Expand Up @@ -1394,30 +1428,32 @@ func (ns *nodeServer) mountRunDVolumes(volumeId, pvName, sourcePath, targetPath,
FSType: fsType,
}

if currentDriver == DFBusTypeVIRTIO || currentDriver == PCITypeVIRTIO {
// write meta before changing the device driver, incase any error occurs
klog.Info("NodePublishVolume(rund3.0): Starting add vmoc(DFBus) mount info to DirectVolume")
err = directvolume.AddMountInfo(volumePath, mountInfo)
if err != nil {
klog.Errorf("NodePublishVolume(rund3.0): vmoc(DFBus) Adding vmoc mount infomation to DirectVolume failed: %v", err)
return true, err
}

if defaultDrivers.Has(cDriver) {
if err := driver.UnbindDriver(); err != nil {
klog.Errorf("NodePublishVolume(rund3.0): can't unbind current device driver: %s", deviceNumber)
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot unbind current device driver: "+deviceNumber)
klog.Errorf("NodePublishVolume(rund3.0): can't unbind current device driver: %s, err: %s", driver.GetDeviceNumber(), err.Error())
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot unbind current device driver: "+driver.GetDeviceNumber())
}
if ns.kataBMIOType == DFBus {
if err = driver.BindDriver(DFBusTypeVFIO); err != nil {
klog.Errorf("NodePublishVolume(rund3.0): can't bind vfio driver to device: %s", deviceNumber)
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot bind current device driver: "+deviceNumber)
klog.Errorf("NodePublishVolume(rund3.0): can't bind bdf vfio driver to device: %s err: %s", driver.GetDeviceNumber(), err.Error())
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot bind current device driver: "+driver.GetDeviceNumber())
}
} else {
if err = driver.BindDriver(PCITypeVFIO); err != nil {
klog.Errorf("NodePublishVolume(rund3.0): can't bind vfio driver to device: %s", deviceNumber)
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot bind current device driver: "+deviceNumber)
klog.Errorf("NodePublishVolume(rund3.0): can't bind pci vfio driver to device: %s err: %s", driver.GetDeviceNumber(), err.Error())
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot bind current device driver: "+driver.GetDeviceNumber())
}
}
}

klog.Info("NodePublishVolume(rund3.0): Starting add vmoc(DFBus) mount info to DirectVolume")
err = directvolume.AddMountInfo(volumePath, mountInfo)
if err != nil {
klog.Errorf("NodePublishVolume(rund3.0): vmoc(DFBus) Adding vmoc mount infomation to DirectVolume failed: %v", err)
return true, err
}
klog.Info("NodePublishVolume(rund3.0): Adding vmoc(DFBus) mount information to DirectVolume succeeds, return immediately")
return true, nil
}
Expand All @@ -1437,7 +1473,7 @@ func (ns *nodeServer) mountRunDVolumes(volumeId, pvName, sourcePath, targetPath,
fileName = filepath.Join(filepath.Dir(filepath.Dir(targetPath)), directvolume.RunD2MountInfoFileName)
}
if err = utils.AppendJSONData(fileName, volumeData); err != nil {
klog.Warning("NodeStageVolume: append volume spec to %s with error: %s", fileName, err.Error())
klog.Warningf("NodeStageVolume: append volume spec to %s with error: %s", fileName, err.Error())
}
return false, nil

Expand Down

0 comments on commit 811a254

Please sign in to comment.