diff --git a/frontend/csi/node_server.go b/frontend/csi/node_server.go index 486ed423f..1ef92e1b5 100644 --- a/frontend/csi/node_server.go +++ b/frontend/csi/node_server.go @@ -2517,9 +2517,10 @@ func (p *Plugin) nodeUnstageNVMeVolume( disconnect := p.nvmeHandler.RemovePublishedNVMeSession(&publishedNVMeSessions, publishInfo.NVMeSubsystemNQN, publishInfo.NVMeNamespaceUUID) + nvmeSubsys := p.nvmeHandler.NewNVMeSubsystem(ctx, publishInfo.NVMeSubsystemNQN) // Get the device using 'nvme-cli' commands. Flush the device IOs. // Proceed further with unstage flow, if device is not found. - nvmeDev, err := p.nvmeHandler.NewNVMeDevice(ctx, publishInfo.NVMeNamespaceUUID) + nvmeDev, err := nvmeSubsys.GetNVMeDevice(ctx, publishInfo.NVMeNamespaceUUID) if err != nil && !errors.IsNotFoundError(err) { return nil, fmt.Errorf("failed to get NVMe device; %v", err) } @@ -2586,7 +2587,6 @@ func (p *Plugin) nodeUnstageNVMeVolume( } // Get the number of namespaces associated with the subsystem - nvmeSubsys := p.nvmeHandler.NewNVMeSubsystem(ctx, publishInfo.NVMeSubsystemNQN) numNs, err := nvmeSubsys.GetNamespaceCount(ctx) if err != nil { Logc(ctx).WithField( diff --git a/mocks/mock_utils/mock_filesystem/mock_filesystem_client.go b/mocks/mock_utils/mock_filesystem/mock_filesystem_client.go index 21604e858..3f5f487d5 100644 --- a/mocks/mock_utils/mock_filesystem/mock_filesystem_client.go +++ b/mocks/mock_utils/mock_filesystem/mock_filesystem_client.go @@ -11,6 +11,7 @@ package mock_filesystem import ( context "context" + fs "io/fs" reflect "reflect" models "github.com/netapp/trident/utils/models" @@ -145,3 +146,33 @@ func (mr *MockFilesystemMockRecorder) RepairVolume(arg0, arg1, arg2 any) *gomock mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RepairVolume", reflect.TypeOf((*MockFilesystem)(nil).RepairVolume), arg0, arg1, arg2) } + +// ScanDir mocks base method. +func (m *MockFilesystem) ScanDir(arg0 string) ([]fs.FileInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ScanDir", arg0) + ret0, _ := ret[0].([]fs.FileInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ScanDir indicates an expected call of ScanDir. +func (mr *MockFilesystemMockRecorder) ScanDir(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ScanDir", reflect.TypeOf((*MockFilesystem)(nil).ScanDir), arg0) +} + +// ScanFile mocks base method. +func (m *MockFilesystem) ScanFile(arg0 string) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ScanFile", arg0) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ScanFile indicates an expected call of ScanFile. +func (mr *MockFilesystemMockRecorder) ScanFile(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ScanFile", reflect.TypeOf((*MockFilesystem)(nil).ScanFile), arg0) +} diff --git a/utils/filesystem/filesystem.go b/utils/filesystem/filesystem.go index 1757156d6..be8413fd7 100644 --- a/utils/filesystem/filesystem.go +++ b/utils/filesystem/filesystem.go @@ -65,6 +65,8 @@ type Filesystem interface { ctx context.Context, path string, ) (available, capacity, usage, inodes, inodesFree, inodesUsed int64, err error) GetUnmountPath(ctx context.Context, trackingInfo *models.VolumeTrackingInfo) (string, error) + ScanFile(filename string) ([]byte, error) + ScanDir(path string) ([]os.FileInfo, error) } type Mount interface { @@ -340,3 +342,32 @@ func (f *FSClient) DeleteFile(ctx context.Context, filepath, fileDescription str return filepath, nil } + +func (f *FSClient) ScanFile(filename string) ([]byte, error) { + fs := afero.NewOsFs() + + file, err := fs.Open(filename) + if err != nil { + fmt.Println("Failed to open file:", err) + return nil, err + } + defer file.Close() + + data, err := afero.ReadAll(file) + if err != nil { + fmt.Println("Failed to read file:", err) + return nil, err + } + return data, nil +} + +func (f *FSClient) ScanDir(path string) ([]os.FileInfo, error) { + fs := afero.NewOsFs() + + dirEntries, err := afero.ReadDir(fs, path) + if err != nil { + fmt.Println("Failed to read directory:", err) + return nil, err + } + return dirEntries, nil +} diff --git a/utils/filesystem/filesystem_windows.go b/utils/filesystem/filesystem_windows.go index 7a0358cb1..764475469 100644 --- a/utils/filesystem/filesystem_windows.go +++ b/utils/filesystem/filesystem_windows.go @@ -21,7 +21,8 @@ func (f *FSClient) getFilesystemSize(ctx context.Context, _ string) (int64, erro } func (f *FSClient) GetFilesystemStats(ctx context.Context, path string) (int64, int64, int64, int64, int64, int64, - error) { + error, +) { Logc(ctx).Debug(">>>> filesystem_windows.GetFilesystemStats") defer Logc(ctx).Debug("<<<< filesystem_windows.GetFilesystemStats") diff --git a/utils/nvme.go b/utils/nvme.go index cb8bca206..a6ad0eebc 100644 --- a/utils/nvme.go +++ b/utils/nvme.go @@ -14,7 +14,6 @@ import ( . "github.com/netapp/trident/logging" "github.com/netapp/trident/utils/devices/luks" - "github.com/netapp/trident/utils/errors" "github.com/netapp/trident/utils/exec" "github.com/netapp/trident/utils/filesystem" "github.com/netapp/trident/utils/models" @@ -22,39 +21,18 @@ import ( const NVMeAttachTimeout = 20 * time.Second -var fsClient = filesystem.New(mountClient) - -// getNVMeSubsystem returns the NVMe subsystem details. -func getNVMeSubsystem(ctx context.Context, subsysNqn string) (*NVMeSubsystem, error) { - Logc(ctx).Debug(">>>> nvme.getNVMeSubsystem") - defer Logc(ctx).Debug("<<<< nvme.getNVMeSubsystem") - - subsys, err := GetNVMeSubsystemList(ctx) - if err != nil { - Logc(ctx).WithField("Error", err).Errorf("Failed to get subsystem list: %v", err) - return nil, err - } - - // Getting current subsystem details. - for _, sub := range subsys.Subsystems { - if sub.NQN == subsysNqn { - return &sub, nil - } - } - - return nil, fmt.Errorf("couldn't find subsystem %s", subsysNqn) -} +var fsClient = *filesystem.New(mountClient) // updatePaths updates the paths with the current state of the subsystem on the k8s node. func (s *NVMeSubsystem) updatePaths(ctx context.Context) error { // Getting current state of subsystem on the k8s node. - sub, err := getNVMeSubsystem(ctx, s.NQN) + paths, err := GetNVMeSubsystemPaths(ctx, fsClient, s.Name) if err != nil { Logc(ctx).WithField("Error", err).Errorf("Failed to update subsystem paths: %v", err) return fmt.Errorf("failed to update subsystem paths: %v", err) } - s.Paths = sub.Paths + s.Paths = paths return nil } @@ -137,43 +115,38 @@ func (s *NVMeSubsystem) Disconnect(ctx context.Context) error { // GetNamespaceCount returns the number of namespaces mapped to the subsystem. func (s *NVMeSubsystem) GetNamespaceCount(ctx context.Context) (int, error) { - var combinedError error - + credibility := false for _, path := range s.Paths { - count, err := GetNamespaceCountForSubsDevice(ctx, "/dev/"+path.Name) - if err != nil { - Logc(ctx).WithField("Error", err).Warnf("Failed to get namespace count: %v", err) - combinedError = multierr.Append(combinedError, err) - continue + if path.State == "live" { + credibility = true + break } + } - return count, nil + if !credibility { + return 0, fmt.Errorf("nvme paths are down, couldn't get the number of namespaces") + } + + count, err := GetNVMeDeviceCountAt(ctx, s.FS, s.Name) + if err != nil { + Logc(ctx).Errorf("Failed to get namespace count: %v", err) + return 0, err } - // Couldn't find any sessions, so no namespaces are attached to this subsystem. - // But if there was error getting the number of namespaces from all the paths, return error. - return 0, combinedError + return count, nil } -// getNVMeDevice returns the NVMe device corresponding to nsPath namespace. -func getNVMeDevice(ctx context.Context, nsUUID string) (*NVMeDevice, error) { - Logc(ctx).Debug(">>>> nvme.getNVMeDevice") - defer Logc(ctx).Debug("<<<< nvme.getNVMeDevice") +func (s *NVMeSubsystem) GetNVMeDevice(ctx context.Context, nsUUID string) (NVMeDeviceInterface, error) { + Logc(ctx).Debug(">>>> nvme.GetNVMeDevice") + defer Logc(ctx).Debug("<<<< nvme.GetNVMeDevice") - dList, err := GetNVMeDeviceList(ctx) + devInterface, err := GetNVMeDeviceAt(ctx, s.Name, nsUUID) if err != nil { - return nil, fmt.Errorf("failed to get device: %v", err) - } - - for _, dev := range dList.Devices { - if dev.UUID == nsUUID { - Logc(ctx).Debugf("Device found: %v.", dev) - return &dev, nil - } + Logc(ctx).Errorf("Failed to get NVMe device, %v", err) + return nil, err } - Logc(ctx).WithField("nsUUID", nsUUID).Debug("No device found for this Namespace.") - return nil, errors.NotFoundError("no device found for the given namespace %v", nsUUID) + return devInterface, nil } // GetPath returns the device path where we mount the filesystem in NodePublish. @@ -203,7 +176,7 @@ func (d *NVMeDevice) FlushDevice(ctx context.Context, ignoreErrors, force bool) // IsNil returns true if Device and NamespacePath are not set. func (d *NVMeDevice) IsNil() bool { - if d == nil || (d.Device == "" && d.NamespacePath == "") { + if d == nil || d.Device == "" { return true } return false @@ -214,20 +187,15 @@ func NewNVMeHandler() NVMeInterface { return &NVMeHandler{} } -// NewNVMeDevice returns new NVMe device -func (nh *NVMeHandler) NewNVMeDevice(ctx context.Context, nsUUID string) (NVMeDeviceInterface, error) { - return getNVMeDevice(ctx, nsUUID) -} - // NewNVMeSubsystem returns NVMe subsystem object. Even if a subsystem is not connected to the k8s node, // this function returns a minimal NVMe subsystem object. func (nh *NVMeHandler) NewNVMeSubsystem(ctx context.Context, subsNqn string) NVMeSubsystemInterface { - sub, err := getNVMeSubsystem(ctx, subsNqn) + sub, err := GetNVMeSubsystem(ctx, fsClient, subsNqn) if err != nil { Logc(ctx).WithField("Error", err).Warnf("Failed to get subsystem: %v; returning minimal subsystem", err) return &NVMeSubsystem{NQN: subsNqn} } - return sub + return &sub } // GetHostNqn returns the NQN of the k8s node. @@ -295,7 +263,7 @@ func AttachNVMeVolume( } } - nvmeDev, err := nvmeHandler.NewNVMeDevice(ctx, publishInfo.NVMeNamespaceUUID) + nvmeDev, err := nvmeSubsys.GetNVMeDevice(ctx, publishInfo.NVMeNamespaceUUID) if err != nil { return err } @@ -589,7 +557,7 @@ func (nh *NVMeHandler) PopulateCurrentNVMeSessions(ctx context.Context, currSess } // Get the list of the subsystems currently present on the k8s node. - subs, err := GetNVMeSubsystemList(ctx) + subs, err := listSubsystemsFromSysFs(fsClient, ctx) if err != nil { Logc(ctx).WithField("Error", err).Errorf("Failed to get subsystem list: %v", err) return err diff --git a/utils/nvme_darwin.go b/utils/nvme_darwin.go index bde166830..02c9b762b 100644 --- a/utils/nvme_darwin.go +++ b/utils/nvme_darwin.go @@ -7,6 +7,7 @@ import ( . "github.com/netapp/trident/logging" "github.com/netapp/trident/utils/errors" + "github.com/netapp/trident/utils/filesystem" ) // NVMeActiveOnHost checks if NVMe is active on host @@ -24,10 +25,10 @@ func GetHostNqn(ctx context.Context) (string, error) { } // GetNVMeSubsystemList returns the list of subsystems connected to the k8s node. -func GetNVMeSubsystemList(ctx context.Context) (Subsystems, error) { - Logc(ctx).Debug(">>>> nvme_darwin.GetNVMeSubsystemList") - defer Logc(ctx).Debug("<<<< nvme_darwin.GetNVMeSubsystemList") - return Subsystems{}, errors.UnsupportedError("GetNVMeSubsystemList is not supported for darwin") +func listSubsystemsFromSysFs(fs filesystem.FSClient, ctx context.Context) (Subsystems, error) { + Logc(ctx).Debug(">>>> nvme_darwin.listSubsystemsFromSysFs") + defer Logc(ctx).Debug("<<<< nvme_darwin.listSubsystemsFromSysFs") + return Subsystems{}, errors.UnsupportedError("listSubsystemsFromSysFs is not supported for darwin") } // ConnectSubsystemToHost creates a path (or session) from the ONTAP subsystem to the k8s node using svmDataLIF. @@ -44,6 +45,36 @@ func DisconnectSubsystemFromHost(ctx context.Context, subsysNqn string) error { return errors.UnsupportedError("DisconnectSubsystemFromHost is not supported for darwin") } +func GetNVMeSubsystem(ctx context.Context, fs filesystem.FSClient, nqn string) (NVMeSubsystem, error) { + Logc(ctx).Debug(">>>> nvme_darwin.GetNVMeSubsystem") + defer Logc(ctx).Debug("<<<< nvme_darwin.GetNVMeSubsystem") + return NVMeSubsystem{}, errors.UnsupportedError("GetNVMeSubsystem is not supported for darwin") +} + +func GetNVMeSubsystemPaths(ctx context.Context, fs filesystem.FSClient, subsystemDirPath string) ([]Path, error) { + Logc(ctx).Debug(">>>> nvme_darwin.GetNVMeSubsystemPaths") + defer Logc(ctx).Debug("<<<< nvme_darwin.GetNVMeSubsystemPaths") + return []Path{}, errors.UnsupportedError("GetNVMeSubsystemPaths is not supported for darwin") +} + +func InitializeNVMeSubsystemPath(ctx context.Context, path *Path) error { + Logc(ctx).Debug(">>>> nvme_darwin.InitializeNVMeSubsystemPath") + defer Logc(ctx).Debug("<<<< nvme_darwin.InitializeNVMeSubsystemPath") + return errors.UnsupportedError("InitializeNVMeSubsystemPath is not supported for darwin") +} + +func GetNVMeDeviceCountAt(ctx context.Context, fs filesystem.FSClient, path string) (int, error) { + Logc(ctx).Debug(">>>> nvme_darwin.GetNVMeDeviceCountAt") + defer Logc(ctx).Debug("<<<< nvme_darwin.GetNVMeDeviceCountAt") + return 0, errors.UnsupportedError("GetNVMeDeviceCountAt is not supported for darwin") +} + +func GetNVMeDeviceAt(ctx context.Context, path, nsUUID string) (NVMeDeviceInterface, error) { + Logc(ctx).Debug(">>>> nvme_darwin.GetNVMeDeviceAt") + defer Logc(ctx).Debug("<<<< nvme_darwin.GetNVMeDeviceAt") + return nil, errors.UnsupportedError("GetNVMeDeviceAt is not supported for darwin") +} + // GetNamespaceCountForSubsDevice returns the number of namespaces present in a given subsystem device. func GetNamespaceCountForSubsDevice(ctx context.Context, subsDevice string) (int, error) { Logc(ctx).Debug(">>>> nvme_darwin.GetNamespaceCount") diff --git a/utils/nvme_linux.go b/utils/nvme_linux.go index 99c46cb7b..3e3d34226 100644 --- a/utils/nvme_linux.go +++ b/utils/nvme_linux.go @@ -6,14 +6,43 @@ import ( "context" "encoding/json" "fmt" + "os" + "regexp" "strings" "time" . "github.com/netapp/trident/logging" sa "github.com/netapp/trident/storage_attribute" + "github.com/netapp/trident/utils/errors" + "github.com/netapp/trident/utils/filesystem" ) -var transport = "tcp" +var ( + transport = "tcp" + nvmeNQNRegex = regexp.MustCompile(`^nvme([0-9]+)n([0-9]+)$`) + nvmeRegex = regexp.MustCompile(`^nvme([0-9]+)$`) +) + +const ( + NVME_PATH = "/sys/class/nvme-subsystem" + SUBSYSNQN = "/subsysnqn" +) + +func ReadFile(fs filesystem.FSClient, filename string) ([]byte, error) { + data, err := fs.ScanFile(filename) + if err != nil { + return nil, err + } + return data, nil +} + +func ReadDir(fs filesystem.FSClient, path string) ([]os.FileInfo, error) { + dir, err := fs.ScanDir(path) + if err != nil { + return nil, err + } + return dir, nil +} // GetHostNqn returns the Nqn string of the k8s node. func GetHostNqn(ctx context.Context) (string, error) { @@ -57,6 +86,38 @@ func NVMeActiveOnHost(ctx context.Context) (bool, error) { return false, fmt.Errorf("NVMe driver is not loaded on the host") } +func listSubsystemsFromSysFs(fs filesystem.FSClient, ctx context.Context) (Subsystems, error) { + Logc(ctx).Trace(">>>> nvme_linux.listSubsystemsFromSysFs") + defer Logc(ctx).Trace("<<<< nvme_linux.listSubsystemsFromSysFs") + + var subsystems Subsystems + subsystemDirs, err := ReadDir(fs, NVME_PATH) + if err != nil { + return subsystems, fmt.Errorf("failed to open nvme subsystems directory: %v", err) + } + + for _, subsystemDir := range subsystemDirs { + subsystemDirPath := NVME_PATH + "/" + subsystemDir.Name() + subsystemNqnPath := subsystemDirPath + SUBSYSNQN + fileBytes, err := ReadFile(fs, subsystemNqnPath) + if err != nil { + return subsystems, fmt.Errorf("failed to read subsystem nqn: %v", err) + } + + fileContent := strings.TrimSpace(string(fileBytes)) + + sub := NVMeSubsystem{NQN: fileContent, Name: subsystemDirPath} + paths, err := GetNVMeSubsystemPaths(ctx, fs, subsystemDirPath) + if err != nil { + return subsystems, err + } + sub.Paths = paths + subsystems.Subsystems = append(subsystems.Subsystems, sub) + } + + return subsystems, nil +} + // GetNVMeSubsystemList returns the list of subsystems connected to the k8s node. func GetNVMeSubsystemList(ctx context.Context) (Subsystems, error) { Logc(ctx).Debug(">>>> nvme_linux.GetNVMeSubsystemList") @@ -139,6 +200,156 @@ func GetNamespaceCountForSubsDevice(ctx context.Context, subsDevice string) (int return strings.Count(string(out), "["), nil } +func GetNVMeSubsystem(ctx context.Context, fs filesystem.FSClient, nqn string) (NVMeSubsystem, error) { + Logc(ctx).Trace(">>>> nvme_linux.GetNVMeSubsystem") + defer Logc(ctx).Trace("<<<< nvme_linux.GetNVMeSubsystem") + + sub := NVMeSubsystem{NQN: nqn} + subsystemDirs, err := ReadDir(fs, NVME_PATH) + if err != nil { + return sub, fmt.Errorf("failed to open nvme subsystems directory: %v", err) + } + + subsystemDirPath := "" + for _, subsystemDir := range subsystemDirs { + subsystemDirPath = NVME_PATH + "/" + subsystemDir.Name() + subsystemNqnPath := subsystemDirPath + SUBSYSNQN + + // Example of subsysnqn file : nqn.1992-08.com.netapp:sn.6628417f7bec11ef9bf2005056b3e634:subsystem.scspa3014048001-b06e4d9a-6817-446b-8dc6-e819c100f935 + fileBytes, err := ReadFile(fs, subsystemNqnPath) + if err != nil { + return sub, fmt.Errorf("failed to read subsystem nqn: %v", err) + } + + fileContent := strings.TrimSpace(string(fileBytes)) + // Ignore this subsystem because it doesn't have the right NQN. + if nqn != fileContent { + continue + } + + // Gather the subsystem paths. + sub.Name = subsystemDirPath + paths, err := GetNVMeSubsystemPaths(ctx, fs, subsystemDirPath) + if err != nil { + return sub, err + } + sub.Paths = paths + } + + if len(sub.Paths) == 0 { + return sub, errors.NotFoundError("no subsystem paths found") + } + + return sub, nil +} + +func GetNVMeSubsystemPaths(ctx context.Context, fs filesystem.FSClient, subsystemDirPath string) ([]Path, error) { + Logc(ctx).Trace(">>>> nvme_linux.GetNVMeSubsystemPaths") + defer Logc(ctx).Trace("<<<< nvme_linux.GetNVMeSubsystemPaths") + + var paths []Path + + subsystemDirContents, err := ReadDir(fs, subsystemDirPath) + if err != nil { + return paths, fmt.Errorf("failed to read subsystem directory contents, %v", err) + } + + for _, subsystemDirContent := range subsystemDirContents { + if nvmeRegex.MatchString(subsystemDirContent.Name()) { + path := Path{Name: subsystemDirPath + "/" + subsystemDirContent.Name()} + if err := updateNVMeSubsystemPathAttributes(ctx, fs, &path); err != nil { + return paths, fmt.Errorf("failed to get path, %v", err) + } + + paths = append(paths, path) + } + } + + return paths, nil +} + +func updateNVMeSubsystemPathAttributes(ctx context.Context, fs filesystem.FSClient, path *Path) error { + Logc(ctx).Trace(">>>> nvme_linux.updateNVMeSubsystemPathAttributes") + defer Logc(ctx).Trace("<<<< nvme_linux.updateNVMeSubsystemPathAttributes") + + if path == nil { + return fmt.Errorf("path is nil") + } + var err error + // Example of state: live + if path.State, err = getSessionFileContent("state", path.Name, fs); err != nil { + Logc(ctx).WithError(err).Error("state is nil") + return err + } + // Example of address: traddr=fd20:8b1e:b258:2014:9c83:2d91:44a:b618,trsvcid=4420 + if path.Address, err = getSessionFileContent("address", path.Name, fs); err != nil { + Logc(ctx).WithError(err).Error("address is nil") + return err + } + // Example of transport: tcp + if path.Transport, err = getSessionFileContent("transport", path.Name, fs); err != nil { + Logc(ctx).WithError(err).Error("transport is nil") + return err + } + return nil +} + +func getSessionFileContent(sessionFileName, pathName string, fs filesystem.FSClient) (string, error) { + fileBytes, err := ReadFile(fs, pathName+"/"+sessionFileName) + if err != nil { + return "", err + } + return strings.TrimSpace(string(fileBytes)), nil +} + +func GetNVMeDeviceCountAt(ctx context.Context, fs filesystem.FSClient, path string) (int, error) { + Logc(ctx).Trace(">>>> nvme_linux.GetNVMeDeviceCountAt") + defer Logc(ctx).Trace("<<<< nvme_linux.GetNVMeDeviceCountAt") + + count := 0 + + pathDirContents, err := ReadDir(fs, path) + if err != nil { + return count, fmt.Errorf("failed to open %s directory, %v", path, err) + } + + for _, pathDirContent := range pathDirContents { + if nvmeNQNRegex.MatchString(pathDirContent.Name()) { + count++ + } + } + + return count, nil +} + +func GetNVMeDeviceAt(ctx context.Context, path, nsUUID string) (NVMeDeviceInterface, error) { + Logc(ctx).Trace(">>>> nvme_linux.GetNVMeDeviceAt") + defer Logc(ctx).Trace("<<<< nvme_linux.GetNVMeDeviceAt") + + pathContents, err := os.ReadDir(path) + if err != nil { + return nil, fmt.Errorf("failed to open %s directory, %v", path, err) + } + + for _, pathContent := range pathContents { + if nvmeNQNRegex.MatchString(pathContent.Name()) { + uuidPath := path + "/" + pathContent.Name() + "/uuid" + fileBytes, err := os.ReadFile(uuidPath) + if err != nil { + return nil, fmt.Errorf("failed to read uuid, %v", err) + } + + fileContent := strings.TrimSpace(string(fileBytes)) + + if nsUUID == fileContent { + return &NVMeDevice{UUID: nsUUID, Device: "/dev/" + pathContent.Name()}, nil + } + } + } + + return nil, fmt.Errorf("nvme device not found") +} + // GetNVMeDeviceList returns the list of NVMe devices present on the k8s node. func GetNVMeDeviceList(ctx context.Context) (NVMeDevices, error) { Logc(ctx).Debug(">>>> nvme_linux.GetNVMeDeviceList") diff --git a/utils/nvme_linux_test.go b/utils/nvme_linux_test.go index 785d66441..576848096 100644 --- a/utils/nvme_linux_test.go +++ b/utils/nvme_linux_test.go @@ -1,3 +1,5 @@ +//go:build linux + package utils import ( @@ -5,14 +7,17 @@ import ( "context" "encoding/json" "fmt" + "io/fs" "testing" "time" + "github.com/spf13/afero" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" mockexec "github.com/netapp/trident/mocks/mock_utils/mock_exec" "github.com/netapp/trident/utils/exec" + "github.com/netapp/trident/utils/filesystem" ) func TestGetHostNQN(t *testing.T) { @@ -384,3 +389,103 @@ func TestFlushNVMeDevice(t *testing.T) { assert.Error(t, err) } + +// MockDirEntry is a mock implementation of os.DirEntry. +type MockDirEntry struct { + name string +} + +// Name returns the name of the mock directory entry. +func (m *MockDirEntry) Name() string { + return m.name +} + +// IsDir always returns false for the mock directory entry. +func (m *MockDirEntry) IsDir() bool { + return false +} + +// Type returns the type of the mock directory entry. +func (m *MockDirEntry) Type() fs.FileMode { + return fs.ModeDir +} + +// Info returns the file info of the mock directory entry. +func (m *MockDirEntry) Info() (fs.FileInfo, error) { + // Return a dummy file info for the mock entry. + return nil, nil +} + +func TestListSubsystemsFromSysFs(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + osFs := afero.NewMemMapFs() + mockFsClient := filesystem.NewDetailed(nil, osFs, nil) + _, err := listSubsystemsFromSysFs(*mockFsClient, ctx()) + assert.Nil(t, err) + + filePath := "/sys/class/nvme-subsystem/nvme1/subsysnqn" + fileContent := []byte("This is a test file") + err = afero.WriteFile(osFs, filePath, fileContent, 0o644) + if err != nil { + t.Errorf("Failed to create test file: %v", err) + } + + _, err = listSubsystemsFromSysFs(*mockFsClient, ctx()) + assert.Nil(t, err) +} + +func TestGetNVMeSubsystem(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + osFs := afero.NewMemMapFs() + mockFsClient := filesystem.NewDetailed(nil, osFs, nil) + + _, err := GetNVMeSubsystem(ctx(), *mockFsClient, "nqn") + assert.NotNil(t, err) + + filePath := "/sys/class/nvme-subsystem/" + fileContent := []byte("This is a test file") + err = afero.WriteFile(osFs, filePath, fileContent, 0o644) + if err != nil { + t.Errorf("Failed to create test file: %v", err) + } + + _, err = GetNVMeSubsystem(ctx(), *mockFsClient, "/sys/class/nvme-subsystem") + assert.NotNil(t, err) +} + +func TestGetNVMeSubsystemPaths(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + osFs := afero.NewMemMapFs() + mockFsClient := filesystem.NewDetailed(nil, osFs, nil) + + filePath := "/sys/class/nvme-subsystem/test" + fileContent := []byte("This is a test file") + err := afero.WriteFile(osFs, filePath, fileContent, 0o644) + if err != nil { + t.Errorf("Failed to create test file: %v", err) + } + _, err = GetNVMeSubsystemPaths(ctx(), *mockFsClient, "/sys/class/nvme-subsystem") + assert.Nil(t, err) +} + +func TestGetNVMeDeviceCountAt(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + osFs := afero.NewMemMapFs() + mockFsClient := filesystem.NewDetailed(nil, osFs, nil) + filePath := "/sys/class/nvme-subsystem/test" + fileContent := []byte("This is a test file") + err := afero.WriteFile(osFs, filePath, fileContent, 0o644) + if err != nil { + t.Errorf("Failed to create test file: %v", err) + } + _, err = GetNVMeDeviceCountAt(ctx(), *mockFsClient, "transport") + assert.NotNil(t, err) +} diff --git a/utils/nvme_types.go b/utils/nvme_types.go index 2c4d07324..846f33594 100644 --- a/utils/nvme_types.go +++ b/utils/nvme_types.go @@ -6,6 +6,7 @@ import ( "context" "time" + "github.com/netapp/trident/utils/filesystem" "github.com/netapp/trident/utils/models" ) @@ -92,6 +93,7 @@ type NVMeSubsystem struct { Name string `json:"Name"` NQN string `json:"NQN"` Paths []Path `json:"Paths"` + FS filesystem.FSClient } type Subsystems struct { @@ -130,6 +132,7 @@ type NVMeSubsystemInterface interface { Disconnect(ctx context.Context) error GetNamespaceCount(ctx context.Context) (int, error) IsNetworkPathPresent(ip string) bool + GetNVMeDevice(ctx context.Context, nsUUID string) (NVMeDeviceInterface, error) } type NVMeDeviceInterface interface { @@ -147,7 +150,6 @@ type NVMeInterface interface { NVMeActiveOnHost(ctx context.Context) (bool, error) GetHostNqn(ctx context.Context) (string, error) NewNVMeSubsystem(ctx context.Context, subsNqn string) NVMeSubsystemInterface - NewNVMeDevice(ctx context.Context, nsUUID string) (NVMeDeviceInterface, error) AddPublishedNVMeSession(pubSessions *NVMeSessions, publishInfo *models.VolumePublishInfo) RemovePublishedNVMeSession(pubSessions *NVMeSessions, subNQN, nsUUID string) bool PopulateCurrentNVMeSessions(ctx context.Context, currSessions *NVMeSessions) error diff --git a/utils/nvme_windows.go b/utils/nvme_windows.go index 031c8d444..810fac009 100644 --- a/utils/nvme_windows.go +++ b/utils/nvme_windows.go @@ -7,6 +7,7 @@ import ( . "github.com/netapp/trident/logging" "github.com/netapp/trident/utils/errors" + "github.com/netapp/trident/utils/filesystem" ) // NVMeActiveOnHost checks if NVMe is active on host @@ -24,10 +25,10 @@ func GetHostNqn(ctx context.Context) (string, error) { } // GetNVMeSubsystemList returns the list of subsystems connected to the k8s node. -func GetNVMeSubsystemList(ctx context.Context) (Subsystems, error) { - Logc(ctx).Debug(">>>> nvme_windows.GetNVMeSubsystemList") - defer Logc(ctx).Debug("<<<< nvme_windows.GetNVMeSubsystemList") - return Subsystems{}, errors.UnsupportedError("GetNVMeSubsystemList is not supported for windows") +func listSubsystemsFromSysFs(fs filesystem.FSClient, ctx context.Context) (Subsystems, error) { + Logc(ctx).Debug(">>>> nvme_windows.listSubsystemsFromSysFs") + defer Logc(ctx).Debug("<<<< nvme_windows.listSubsystemsFromSysFs") + return Subsystems{}, errors.UnsupportedError("ListSubsystemsFromSysFs is not supported for windows") } // ConnectSubsystemToHost creates a path (or session) from the ONTAP subsystem to the k8s node using svmDataLIF. @@ -44,6 +45,36 @@ func DisconnectSubsystemFromHost(ctx context.Context, subsysNqn string) error { return errors.UnsupportedError("DisconnectSubsystemFromHost is not supported for windows") } +func GetNVMeSubsystem(ctx context.Context, fs filesystem.FSClient, nqn string) (NVMeSubsystem, error) { + Logc(ctx).Debug(">>>> nvme_windows.GetNVMeSubsystem") + defer Logc(ctx).Debug("<<<< nvme_windows.GetNVMeSubsystem") + return NVMeSubsystem{}, errors.UnsupportedError("GetNVMeSubsystem is not supported for windows") +} + +func GetNVMeSubsystemPaths(ctx context.Context, fs filesystem.FSClient, subsystemDirPath string) ([]Path, error) { + Logc(ctx).Debug(">>>> nvme_windows.GetNVMeSubsystemPaths") + defer Logc(ctx).Debug("<<<< nvme_windows.GetNVMeSubsystemPaths") + return []Path{}, errors.UnsupportedError("GetNVMeSubsystemPaths is not supported for windows") +} + +func InitializeNVMeSubsystemPath(ctx context.Context, path *Path) error { + Logc(ctx).Debug(">>>> nvme_windows.InitializeNVMeSubsystemPath") + defer Logc(ctx).Debug("<<<< nvme_windows.InitializeNVMeSubsystemPath") + return errors.UnsupportedError("InitializeNVMeSubsystemPath is not supported for windows") +} + +func GetNVMeDeviceCountAt(ctx context.Context, fs filesystem.FSClient, path string) (int, error) { + Logc(ctx).Debug(">>>> nvme_windows.GetNVMeDeviceCountAt") + defer Logc(ctx).Debug("<<<< nvme_windows.GetNVMeDeviceCountAt") + return 0, errors.UnsupportedError("GetNVMeDeviceCountAt is not supported for windows") +} + +func GetNVMeDeviceAt(ctx context.Context, path, nsUUID string) (NVMeDeviceInterface, error) { + Logc(ctx).Debug(">>>> nvme_windows.GetNVMeDeviceAt") + defer Logc(ctx).Debug("<<<< nvme_windows.GetNVMeDeviceAt") + return nil, errors.UnsupportedError("GetNVMeDeviceAt is not supported for windows") +} + // GetNamespaceCountForSubsDevice returns the number of namespaces present in a given subsystem device. func GetNamespaceCountForSubsDevice(ctx context.Context, subsDevice string) (int, error) { Logc(ctx).Debug(">>>> nvme_windows.GetNamespaceCount")