Skip to content

Commit

Permalink
Bump the version of rund-csi protocol to 3.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@mowangdk
mowangdk committed Oct 8, 2024
1 parent 857f077 commit 2beaa0a
Show file tree
Hide file tree
Showing 2 changed files with 124 additions and 128 deletions.
51 changes: 45 additions & 6 deletions pkg/disk/bdf.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -498,6 +498,10 @@ const (
DFBus // 1
)

const (
dfBusDevicePathPattern = "/sys/bus/dragonfly/devices/dfvirtio*/type"
)

func (_type MachineType) BusName() string {
busNames := [...]string{
BDFTypeBus,
Expand Down Expand Up @@ -531,20 +535,55 @@ type Driver interface {
GetDeviceNumber() string
}

func NewDeviceDriver(blockDevice, deviceNumber string, _type MachineType, extras map[string]string) (Driver, error) {
func NewDeviceDriver(volumeId, blockDevice, deviceNumber string, _type MachineType, extras map[string]string) (Driver, error) {
d := &driver{
blockDevice: blockDevice,
deviceNumber: deviceNumber,
machineType: _type,
extras: extras,
}
if d.deviceNumber == "" {
deviceNumber, err := DefaultDeviceManager.GetDeviceNumberFromBlockDevice(blockDevice, d.machineType.BusPrefix())
if blockDevice != "" {
if deviceNumber == "" {
deviceNumber, err := DefaultDeviceManager.GetDeviceNumberFromBlockDevice(blockDevice, d.machineType.BusPrefix())
if err != nil {
klog.Errorf("NewDeviceDriver: get device number from block device err: %v", err)
return nil, err
}
d.deviceNumber = deviceNumber
}
}
if deviceNumber != "" {
return d, nil
}
if _type == DFBus {
matchesFile, err := filepath.Glob(dfBusDevicePathPattern)
if err != nil {
return nil, fmt.Errorf("Failed to list DFbus type files path. err: %v", err)
}
for _, path := range matchesFile {
body, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("Dfbus read type file %q failed: %v", path, err)
}
infos := strings.Split(string(body), " ")
if len(infos) != 2 {
return nil, fmt.Errorf("Dfbus type file format error")
}
if infos[0] != "block" {
continue
}
if infos[1] == strings.TrimPrefix(volumeId, "d-") {
DFNumber := filepath.Base(filepath.Dir(path))
d.deviceNumber = DFNumber
return d, nil
}
}
} else {
output, err := utils.CommandOnNode("xdragon-bdf", "--nvme", "-id=%s", volumeId).CombinedOutput()
if err != nil {
klog.Errorf("NewDeviceDriver: get device number from block device err: %v", err)
return nil, err
return nil, fmt.Errorf("Failed to excute bdf command: %s, err: %v", volumeId, err)
}
d.deviceNumber = deviceNumber
d.deviceNumber = string(output)
}
return d, nil
}
Expand Down
201 changes: 79 additions & 122 deletions pkg/disk/nodeserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,13 +188,13 @@ func NewNodeServer(m metadata.MetadataProvider) csi.NodeServer {
kataBMIOType := BDF
value, err := m.Get(metadata.VmocType)
if err != nil {
log.Debugf("get vmoc failed: %+v", err)
klog.Errorf("get vmoc failed: %+v", err)
}
if err == nil && value == "true" {
kataBMIOType = DFBus
}

log.Infof("KATA BFIO Type: %v", kataBMIOType)
klog.Infof("KATA BFIO Type: %v", kataBMIOType)
podCgroup, err := utils.NewPodCGroup()
if err != nil {
klog.Fatalf("Failed to initialize pod cgroup: %v", err)
Expand Down Expand Up @@ -1199,16 +1199,16 @@ func (ns *nodeServer) umountRunDVolumes(volumePath string) (bool, error) {
return false, status.Error(codes.Internal, "vmoc(DFBus) mode only support csi-runD protocol 3.0")
}

d, _ := NewDeviceDriver("", mountInfo.Source, DFBus, nil)
d, _ := NewDeviceDriver("", "", mountInfo.Source, DFBus, nil)
cDriver, err := d.CurentDriver()
if err != nil {
if IsNoSuchFileErr(err) {
log.Infof("vmoc(DFBus) dfbus driver has been removed: %s", DFBus)
klog.Infof("vmoc(DFBus) dfbus driver has been removed: %s", DFBus)
return true, nil
}
return true, status.Error(codes.Internal, err.Error())
}
log.Infof("vmoc(DFBus) current dfbus driver : %v", DFBus)
klog.Infof("vmoc(DFBus) current dfbus driver : %v", DFBus)
if cDriver == DFBusTypeVIRTIO {
return true, nil
}
Expand Down Expand Up @@ -1302,46 +1302,32 @@ func (ns *nodeServer) mountRunDVolumes(volumeId, sourcePath, targetPath, fsType,
if err != nil {
deviceName = getVolumeConfig(volumeId)
}
if deviceName == "" {
klog.Errorf("NodePublishVolume(rund): cannot get local deviceName for volume: %s", volumeId)
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot get local deviceName for volume: "+volumeId)
}
deviceNumber := ""
volumePath := filepath.Dir(targetPath)

// umount the stage path, which is mounted in Stage
if err := ns.unmountStageTarget(sourcePath); err != nil {
klog.Errorf("NodePublishVolume(rund3.0): unmountStageTarget %s with error: %s", sourcePath, err.Error())
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: unmountStageTarget "+sourcePath+" with error: "+err.Error())
}

driver, err := NewDeviceDriver(volumeId, deviceName, deviceNumber, ns.kataBMIOType, map[string]string{})
if err != nil {
klog.Errorf("NodePublishVolume(rund3.0): can't get bdf number of volume: %s: err: %v", volumeId, err)
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot get bdf number of volume: "+volumeId)
}
currentDriver, err := driver.CurentDriver()
if err != nil {
return true, status.Errorf(codes.Internal, "NodePublishVolume(rund3.0): can't get current volume driver: %+v", err)
}
deviceType := directvolume.DeviceTypePCI
if ns.kataBMIOType == DFBus {
deviceType = directvolume.DeviceTypeDFBusPort
}
extras := make(map[string]string)
// Block runs csi3.0 protocol
if isRawBlock {
// umount the stage path, which is mounted in Stage
if err := ns.unmountStageTarget(sourcePath); err != nil {
klog.Errorf("NodePublishVolume(rund3.0): unmountStageTarget %s with error: %s", sourcePath, err.Error())
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: unmountStageTarget "+sourcePath+" with error: "+err.Error())
}
klog.Infof("NodePublishVolume(rund3.0): get bdf number by device: %s", deviceName)
deviceNumber := ""
deviceType := directvolume.DeviceTypePCI
if ns.kataBMIOType == DFBus {
deviceType = directvolume.DeviceTypeDFBusPort
driver, err := NewDeviceDriver(deviceName, "", DFBus, map[string]string{})
if err != nil {
klog.Errorf("NodePublishVolume(rund3.0): can't get dfbusport number of volume: %s: err: %v", volumeId, err)
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot get bdf number of volume: "+volumeId)
}
deviceNumber = driver.GetDeviceNumber()
// we can find deviceName means that device is bind to virtio driver
if err := driver.UnbindDriver(); err != nil {
klog.Errorf("NodePublishVolume(rund3.0): can't unbind current device driver: %s", deviceNumber)
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot unbind current device driver: "+deviceNumber)
}
if err = driver.BindDriver(DFBusTypeVFIO); err != nil {
klog.Errorf("NodePublishVolume(rund3.0): can't bind vfio driver to device: %s", deviceNumber)
}
} else {
driver, err := NewDeviceDriver(deviceName, "", BDF, map[string]string{})
if err != nil {
klog.Errorf("NodePublishVolume(rund3.0): can't get bdf number of volume: %s: err: %v", volumeId, err)
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot get bdf number of volume: "+volumeId)
}
deviceNumber = driver.GetDeviceNumber()
}
klog.V(2).Infof("NodePublishVolume(rund3.0): get bdf number by device: %s", deviceName)
deviceUid := 0
deviceGid := 0
deviceInfo, err := os.Stat(deviceName)
Expand All @@ -1352,99 +1338,70 @@ func (ns *nodeServer) mountRunDVolumes(volumeId, sourcePath, targetPath, fsType,
deviceUid = int(stat.Uid)
deviceGid = int(stat.Gid)
}
extras := make(map[string]string)
extras["Type"] = "b"
extras["FileMode"] = directvolume.BlockFileModeReadWrite
extras["Uid"] = strconv.Itoa(deviceUid)
extras["Gid"] = strconv.Itoa(deviceGid)
mountOptions := []string{}
if mountFlags != nil {
mountOptions = mountFlags
}
}

mountInfo := directvolume.MountInfo{
Source: deviceNumber,
DeviceType: deviceType,
MountOpts: mountOptions,
Extra: extras,
FSType: fsType,
}
mountOptions := []string{}
if mountFlags != nil {
mountOptions = mountFlags
}

klog.Info("NodePublishVolume(rund3.0): Starting add mount info to DirectVolume")
err = directvolume.AddMountInfo(volumePath, mountInfo)
if err != nil {
klog.Errorf("NodePublishVolume(rund3.0): Adding mount infomation to DirectVolume failed: %v", err)
return true, err
}
klog.Info("NodePublishVolume(rund3.0): Adding mount information to DirectVolume succeeds, return immediately")
return true, nil
mountInfo := directvolume.MountInfo{
Source: driver.GetDeviceNumber(),
DeviceType: deviceType,
MountOpts: mountOptions,
Extra: extras,
FSType: fsType,
}

if ns.kataBMIOType == DFBus {
// umount the stage path, which is mounted in Stage
if err := ns.unmountStageTarget(sourcePath); err != nil {
klog.Errorf("NodePublishVolume(rund3.0): unmountStageTarget in vmoc(DFBus) mode %s with error: %s", sourcePath, err.Error())
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: unmountStageTarget in vmoc(DFBus) "+sourcePath+" with error: "+err.Error())
}
klog.Infof("NodePublishVolume(rund3.0): get dfbusport number by device: %s", deviceName)
driver, err := NewDeviceDriver(deviceName, "", DFBus, map[string]string{})
if err != nil {
klog.Errorf("NodePublishVolume(rund3.0): can't get dfbusport number of volume: %s: err: %v", volumeId, err)
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot get bdf number of volume: "+volumeId)
}
// we can find deviceName means that device is bind to virtio driver
if currentDriver == DFBusTypeVIRTIO || currentDriver == PCITypeVIRTIO {
if err := driver.UnbindDriver(); err != nil {
klog.Errorf("NodePublishVolume(rund3.0): can't unbind current device driver: %s", driver.GetDeviceNumber())
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot unbind current device driver: "+driver.GetDeviceNumber())
}
if err = driver.BindDriver(DFBusTypeVFIO); err != nil {
klog.Errorf("NodePublishVolume(rund3.0): can't bind vfio driver to device: %s", driver.GetDeviceNumber())
klog.Errorf("NodePublishVolume(rund3.0): can't unbind current device driver: %s", deviceNumber)
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot unbind current device driver: "+deviceNumber)
}
if err != nil {
klog.Errorf("NodePublishVolume(rund3.0): can't get dfbusport number of volume: %s: err: %v", volumeId, err)
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot get dfbusport number of volume: "+volumeId)
}
mountOptions := []string{}
if mountFlags != nil {
mountOptions = mountFlags
}

mountInfo := directvolume.MountInfo{
Source: driver.GetDeviceNumber(),
DeviceType: directvolume.DeviceTypeDFBusPort,
MountOpts: mountOptions,
Extra: map[string]string{},
FSType: fsType,
}

klog.Info("NodePublishVolume(rund3.0): Starting add vmoc(DFBus) mount info to DirectVolume")
err = directvolume.AddMountInfo(volumePath, mountInfo)
if err != nil {
klog.Errorf("NodePublishVolume(rund3.0): vmoc(DFBus) Adding vmoc mount infomation to DirectVolume failed: %v", err)
return true, err
if ns.kataBMIOType == DFBus {
if err = driver.BindDriver(DFBusTypeVFIO); err != nil {
klog.Errorf("NodePublishVolume(rund3.0): can't bind vfio driver to device: %s", deviceNumber)
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot bind current device driver: "+deviceNumber)
}
} else {
if err = driver.BindDriver(PCITypeVFIO); err != nil {
klog.Errorf("NodePublishVolume(rund3.0): can't bind vfio driver to device: %s", deviceNumber)
return true, status.Error(codes.InvalidArgument, "NodePublishVolume: cannot bind current device driver: "+deviceNumber)
}
}
klog.Info("NodePublishVolume(rund3.0): Adding vmoc(DFBus) mount information to DirectVolume succeeds, return immediately")
return true, nil
}

// (runD2.0) Need write mountOptions(metadata) parameters to file, and run normal runc process
klog.Infof("NodePublishVolume(rund): run csi runD protocol 2.0 logic")
volumeData := map[string]string{}
volumeData["csi.alibabacloud.com/fsType"] = fsType
if len(mountFlags) != 0 {
volumeData["csi.alibabacloud.com/mountOptions"] = strings.Join(mountFlags, ",")
}
if mkfsOptions != "" {
volumeData["csi.alibabacloud.com/mkfsOptions"] = mkfsOptions
}
volumeData["csi.alibabacloud.com/disk-mounted"] = "true"
fileName := filepath.Join(filepath.Dir(targetPath), directvolume.RunD2MountInfoFileName)
if strings.HasSuffix(targetPath, "/") {
fileName = filepath.Join(filepath.Dir(filepath.Dir(targetPath)), directvolume.RunD2MountInfoFileName)
}
if err = utils.AppendJSONData(fileName, volumeData); err != nil {
klog.Warningf("NodeStageVolume: append volume spec to %s with error: %s", fileName, err.Error())
}
return false, nil
klog.Info("NodePublishVolume(rund3.0): Starting add vmoc(DFBus) mount info to DirectVolume")
err = directvolume.AddMountInfo(volumePath, mountInfo)
if err != nil {
klog.Errorf("NodePublishVolume(rund3.0): vmoc(DFBus) Adding vmoc mount infomation to DirectVolume failed: %v", err)
return true, err
}
klog.Info("NodePublishVolume(rund3.0): Adding vmoc(DFBus) mount information to DirectVolume succeeds, return immediately")
return true, nil

// // (runD2.0) Need write mountOptions(metadata) parameters to file, and run normal runc process
// log.Infof("NodePublishVolume(rund): run csi runD protocol 2.0 logic")
// volumeData := map[string]string{}
// volumeData["csi.alibabacloud.com/fsType"] = fsType
// if len(mountFlags) != 0 {
// volumeData["csi.alibabacloud.com/mountOptions"] = strings.Join(mountFlags, ",")
// }
// if mkfsOptions != "" {
// volumeData["csi.alibabacloud.com/mkfsOptions"] = mkfsOptions
// }
// volumeData["csi.alibabacloud.com/disk-mounted"] = "true"
// fileName := filepath.Join(filepath.Dir(targetPath), directvolume.RunD2MountInfoFileName)
// if strings.HasSuffix(targetPath, "/") {
// fileName = filepath.Join(filepath.Dir(filepath.Dir(targetPath)), directvolume.RunD2MountInfoFileName)
// }
// if err = utils.AppendJSONData(fileName, volumeData); err != nil {
// log.Warnf("NodeStageVolume: append volume spec to %s with error: %s", fileName, err.Error())
// }
// return false, nil

}

0 comments on commit 2beaa0a

Please sign in to comment.