Merge pull request kubernetes-csi#19 from 27149chen/delete_volume

delete individual volume

iscsi/iscsi.go

@@ -19,8 +19,9 @@ import (
 const defaultPort = "3260"
 
 var (
-	debug       *log.Logger
-	execCommand = exec.Command
+	debug           *log.Logger
+	execCommand     = exec.Command
+	execWithTimeout = ExecWithTimeout
 )
 
 type statFunc func(string) (os.FileInfo, error)
@@ -50,10 +51,14 @@ type Connector struct {
 	SessionSecrets   Secrets      `json:"session_secrets"`
 	Interface        string       `json:"interface"`
 	Multipath        bool         `json:"multipath"`
-	RetryCount       int32        `json:"retry_count"`
-	CheckInterval    int32        `json:"check_interval"`
-	DoDiscovery      bool         `json:"do_discovery"`
-	DoCHAPDiscovery  bool         `json:"do_chap_discovery"`
+
+	// DevicePath is dm-x for a multipath device, and sdx for a normal device.
+	DevicePath string `json:"device_path"`
+
+	RetryCount      int32 `json:"retry_count"`
+	CheckInterval   int32 `json:"check_interval"`
+	DoDiscovery     bool  `json:"do_discovery"`
+	DoCHAPDiscovery bool  `json:"do_chap_discovery"`
 }
 
 func init() {
@@ -352,6 +357,81 @@ func Disconnect(tgtIqn string, portals []string) error {
 	return err
 }
 
+// DisconnectVolume removes a volume from a Linux host.
+func DisconnectVolume(c Connector) error {
+	// Steps to safely remove an iSCSI storage volume from a Linux host are as following:
+	// 1. Unmount the disk from a filesystem on the system.
+	// 2. Flush the multipath map for the disk we’re removing (if multipath is enabled).
+	// 3. Remove the physical disk entities that Linux maintains.
+	// 4. Take the storage volume (disk) offline on the storage subsystem.
+	// 5. Rescan the iSCSI sessions.
+	//
+	// DisconnectVolume focuses on step 2 and 3.
+	// Note: make sure the volume is already unmounted before calling this method.
+
+	debug.Printf("Disconnecting volume in path %s.\n", c.DevicePath)
+	if c.Multipath {
+		debug.Printf("Removing multipath device.\n")
+		devices, err := GetSysDevicesFromMultipathDevice(c.DevicePath)
+		if err != nil {
+			return err
+		}
+		err := FlushMultipathDevice(c.DevicePath)
+		if err != nil {
+			return err
+		}
+		debug.Printf("Found multipath slaves %v, removing all of them.\n", devices)
+		if err := RemovePhysicalDevice(devices...); err != nil {
+			return err
+		}
+	} else {
+		debug.Printf("Removing normal device.\n")
+		if err := RemovePhysicalDevice(c.DevicePath); err != nil {
+			return err
+		}
+	}
+
+	debug.Printf("Finished disconnecting volume.\n")
+	return nil
+}
+
+// RemovePhysicalDevice removes device(s) sdx from a Linux host.
+func RemovePhysicalDevice(devices ...string) error {
+	debug.Printf("Removing scsi device %v.\n", devices)
+	var errs []error
+	for _, deviceName := range devices {
+		if deviceName == "" {
+			continue
+		}
+
+		debug.Printf("Delete scsi device %v.\n", deviceName)
+		// Remove a scsi device by executing 'echo "1" > /sys/block/sdx/device/delete
+		filename := filepath.Join(sysBlockPath, deviceName, "device", "delete")
+		if f, err := os.OpenFile(filename, os.O_TRUNC|os.O_WRONLY, 0200); err != nil {
+			if os.IsNotExist(err) {
+				continue
+			} else {
+				debug.Printf("Error while opening file %v: %v\n", filename, err)
+				errs = append(errs, err)
+				continue
+			}
+		} else {
+			defer f.Close()
+			if _, err := f.WriteString("1"); err != nil {
+				debug.Printf("Error while writing to file %v: %v", filename, err)
+				errs = append(errs, err)
+				continue
+			}
+		}
+	}
+
+	if len(errs) > 0 {
+		return errs[0]
+	}
+	debug.Println("Finshed removing SCSI devices.")
+	return nil
+}
+
 // PersistConnector persists the provided Connector to the specified file (ie /var/lib/pfile/myConnector.json)
 func PersistConnector(c *Connector, filePath string) error {
 	//file := path.Join("mnt", c.VolumeName+".json")

iscsi/iscsi_test.go

@@ -1,12 +1,16 @@
 package iscsi
 
 import (
+	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"reflect"
 	"strconv"
 	"testing"
+	"time"
 )
 
 var nodeDB = `
@@ -79,10 +83,16 @@ node.conn[0].iscsi.OFMarker = No
 var emptyTransportName = "iface.transport_name = \n"
 var emptyDbRecord = "\n\n\n"
 var testCmdOutput = ""
+var testCmdTimeout = false
 var testCmdError error
+var testExecWithTimeoutError error
 var mockedExitStatus = 0
 var mockedStdout string
 
+var normalDevice = "sda"
+var multipathDevice = "dm-1"
+var slaves = []string{"sdb", "sdc"}
+
 type testCmdRunner struct{}
 
 func fakeExecCommand(command string, args ...string) *exec.Cmd {
@@ -96,6 +106,13 @@ func fakeExecCommand(command string, args ...string) *exec.Cmd {
 	return cmd
 }
 
+func fakeExecWithTimeout(command string, args []string, timeout time.Duration) ([]byte, error) {
+	if testCmdTimeout {
+		return nil, context.DeadlineExceeded
+	}
+	return []byte(testCmdOutput), testExecWithTimeoutError
+}
+
 func TestExecCommandHelper(t *testing.T) {
 	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
 		return
@@ -111,6 +128,33 @@ func (tr testCmdRunner) execCmd(cmd string, args ...string) (string, error) {
 
 }
 
+func preparePaths(sysBlockPath string) error {
+	for _, d := range append(slaves, normalDevice) {
+		devicePath := filepath.Join(sysBlockPath, d, "device")
+		err := os.MkdirAll(devicePath, os.ModePerm)
+		if err != nil {
+			return err
+		}
+		err = ioutil.WriteFile(filepath.Join(devicePath, "delete"), []byte(""), 0600)
+		if err != nil {
+			return err
+		}
+	}
+	for _, s := range slaves {
+		err := os.MkdirAll(filepath.Join(sysBlockPath, multipathDevice, "slaves", s), os.ModePerm)
+		if err != nil {
+			return err
+		}
+	}
+
+	err := os.MkdirAll(filepath.Join(sysBlockPath, "dev", multipathDevice), os.ModePerm)
+	if err != nil {
+		return err
+	}
+	return nil
+
+}
+
 func Test_parseSessions(t *testing.T) {
 	var sessions []iscsiSession
 	output := "tcp: [2] 192.168.1.107:3260,1 iqn.2010-10.org.openstack:volume-eb393993-73d0-4e39-9ef4-b5841e244ced (non-flash)\n" +
@@ -226,3 +270,130 @@ func Test_sessionExists(t *testing.T) {
 		})
 	}
 }
+
+func Test_DisconnectNormalVolume(t *testing.T) {
+
+	tmpDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Errorf("can not create temp directory: %v", err)
+		return
+	}
+	sysBlockPath = tmpDir
+	defer os.RemoveAll(tmpDir)
+
+	err = preparePaths(tmpDir)
+	if err != nil {
+		t.Errorf("can not create temp directories and files: %v", err)
+		return
+	}
+
+	execWithTimeout = fakeExecWithTimeout
+	devicePath := normalDevice
+
+	tests := []struct {
+		name         string
+		removeDevice bool
+		wantErr      bool
+	}{
+		{"DisconnectNormalVolume", false, false},
+		{"DisconnectNonexistentNormalVolume", true, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.removeDevice {
+				os.RemoveAll(filepath.Join(sysBlockPath, devicePath))
+			}
+			c := Connector{Multipath: false, DevicePath: devicePath}
+			err := DisconnectVolume(c)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("DisconnectVolume() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+
+			if !tt.removeDevice {
+				deleteFile := filepath.Join(sysBlockPath, devicePath, "device", "delete")
+				out, err := ioutil.ReadFile(deleteFile)
+				if err != nil {
+					t.Errorf("can not read file %v: %v", deleteFile, err)
+					return
+				}
+				if string(out) != "1" {
+					t.Errorf("file content mismatch, got = %s, want = 1", string(out))
+					return
+				}
+			}
+		})
+	}
+}
+
+func Test_DisconnectMultipathVolume(t *testing.T) {
+
+	execWithTimeout = fakeExecWithTimeout
+	devicePath := multipathDevice
+
+	tests := []struct {
+		name         string
+		timeout      bool
+		removeDevice bool
+		wantErr      bool
+		cmdError     error
+	}{
+		{"DisconnectMultipathVolume", false, false, false, nil},
+		{"DisconnectMultipathVolumeFlushFailed", false, false, true, fmt.Errorf("error")},
+		{"DisconnectMultipathVolumeFlushTimeout", true, false, true, nil},
+		{"DisconnectNonexistentMultipathVolume", false, true, false, fmt.Errorf("error")},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			tmpDir, err := ioutil.TempDir("", "")
+			if err != nil {
+				t.Errorf("can not create temp directory: %v", err)
+				return
+			}
+			sysBlockPath = tmpDir
+			devPath = filepath.Join(tmpDir, "dev")
+			defer os.RemoveAll(tmpDir)
+
+			err = preparePaths(tmpDir)
+			if err != nil {
+				t.Errorf("can not create temp directories and files: %v", err)
+				return
+			}
+			testExecWithTimeoutError = tt.cmdError
+			testCmdTimeout = tt.timeout
+			if tt.removeDevice {
+				os.RemoveAll(filepath.Join(sysBlockPath, devicePath))
+				os.RemoveAll(devPath)
+			}
+			c := Connector{Multipath: true, DevicePath: devicePath}
+			err = DisconnectVolume(c)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("DisconnectVolume() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if tt.timeout {
+				if err != context.DeadlineExceeded {
+					t.Errorf("DisconnectVolume() error = %v, wantErr %v", err, context.DeadlineExceeded)
+					return
+				}
+			}
+
+			if !tt.removeDevice && !tt.wantErr {
+				for _, s := range slaves {
+					deleteFile := filepath.Join(sysBlockPath, s, "device", "delete")
+					out, err := ioutil.ReadFile(deleteFile)
+					if err != nil {
+						t.Errorf("can not read file %v: %v", deleteFile, err)
+						return
+					}
+					if string(out) != "1" {
+						t.Errorf("file content mismatch, got = %s, want = 1", string(out))
+						return
+					}
+				}
+
+			}
+		})
+	}
+}