Gobrick release v1.2

base.go

@@ -20,33 +20,46 @@ package gobrick
 
 import (
 	"context"
-	"github.com/dell/gobrick/internal/logger"
-	intmultipath "github.com/dell/gobrick/internal/multipath"
-	intscsi "github.com/dell/gobrick/internal/scsi"
-	"github.com/dell/gobrick/internal/tracer"
 	"errors"
 	"fmt"
 	"path"
 	"strings"
 	"time"
+
+	"github.com/dell/gobrick/internal/logger"
+	intmultipath "github.com/dell/gobrick/internal/multipath"
+	intscsi "github.com/dell/gobrick/internal/scsi"
+	"github.com/dell/gobrick/internal/tracer"
 )
 
 const (
-	multipathFlushTimeoutDefault = time.Second * 120
-	deviceMapperPrefix           = "dm-"
+	multipathFlushTimeoutDefault      = time.Second * 120
+	multipathFlushRetriesDefault      = 10
+	multipathFlushRetryTimeoutDefault = time.Second * 5
+	deviceMapperPrefix                = "dm-"
 )
 
 type baseConnectorParams struct {
-	MultipathFlushTimeout time.Duration
+	MultipathFlushRetries      int
+	MultipathFlushTimeout      time.Duration
+	MultipathFlushRetryTimeout time.Duration
 }
 
 func newBaseConnector(mp intmultipath.Multipath, s intscsi.SCSI, params baseConnectorParams) *baseConnector {
 	conn := &baseConnector{
 		multipath: mp,
 		scsi:      s,
 	}
+
+	if params.MultipathFlushRetries == 0 {
+		conn.multipathFlushRetries = multipathFlushRetriesDefault
+	} else {
+		conn.multipathFlushRetries = params.MultipathFlushRetries
+	}
 	setTimeouts(&conn.multipathFlushTimeout,
 		params.MultipathFlushTimeout, multipathFlushTimeoutDefault)
+	setTimeouts(&conn.multipathFlushRetryTimeout,
+		params.MultipathFlushRetryTimeout, multipathFlushRetryTimeoutDefault)
 
 	return conn
 }
@@ -55,7 +68,9 @@ type baseConnector struct {
 	multipath intmultipath.Multipath
 	scsi      intscsi.SCSI
 
-	multipathFlushTimeout time.Duration
+	multipathFlushRetries      int
+	multipathFlushTimeout      time.Duration
+	multipathFlushRetryTimeout time.Duration
 }
 
 func (bc *baseConnector) disconnectDevicesByDeviceName(ctx context.Context, name string) error {
@@ -120,11 +135,27 @@ func (bc *baseConnector) cleanMultipathDevice(ctx context.Context, dm string) er
 	defer tracer.TraceFuncCall(ctx, "baseConnector.cleanMultipathDevice")()
 	ctx, cancelFunc := context.WithTimeout(ctx, bc.multipathFlushTimeout)
 	defer cancelFunc()
-	err := bc.multipath.FlushDevice(ctx, path.Join("/dev/", dm))
-	if err != nil {
-		return err
+
+	for i := 0; i < bc.multipathFlushRetries; i++ {
+		logger.Info(ctx, "trying to flush multipath device with retries: retry %d", i)
+		err := bc.retryFlushMultipathDevice(ctx, dm)
+		if err == nil {
+			return nil
+		}
 	}
-	return nil
+
+	return fmt.Errorf("can't flush multipath device, timed out after multiple attempts")
+}
+
+func (bc *baseConnector) retryFlushMultipathDevice(ctx context.Context, dm string) error {
+	smallCtx, cancel := context.WithTimeout(ctx, bc.multipathFlushRetryTimeout)
+	defer cancel()
+	err := bc.multipath.FlushDevice(smallCtx, path.Join("/dev/", dm))
+	if !bc.scsi.IsDeviceExist(ctx, dm) {
+		logger.Info(ctx, "device %s no longer exists", dm)
+		return nil
+	}
+	return err
 }
 
 func (bc *baseConnector) getDMWWN(ctx context.Context, dm string) (string, error) {

fc.go

@@ -20,21 +20,22 @@ package gobrick
 
 import (
 	"context"
+	"errors"
+	"fmt"
+	"math"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
 	"github.com/dell/gobrick/internal/logger"
 	intmultipath "github.com/dell/gobrick/internal/multipath"
 	intscsi "github.com/dell/gobrick/internal/scsi"
 	"github.com/dell/gobrick/internal/tracer"
 	wrp "github.com/dell/gobrick/internal/wrappers"
 	"github.com/dell/gobrick/pkg/multipath"
 	"github.com/dell/gobrick/pkg/scsi"
-	"errors"
-	"fmt"
 	"golang.org/x/sync/semaphore"
-	"math"
-	"path"
-	"strconv"
-	"strings"
-	"time"
 )
 
 const (
@@ -51,6 +52,10 @@ type FCConnectorParams struct {
 	WaitDeviceRegisterTimeout time.Duration
 	// timeout for multipath flush command
 	MultipathFlushTimeout time.Duration
+	// timeout for each multipath flush retry
+	MultipathFlushRetryTimeout time.Duration
+	// how many retries for multipath flush
+	MultipathFlushRetries int
 
 	// how many parallel operations allowed
 	MaxParallelOperations int
@@ -69,7 +74,9 @@ func NewFCConnector(params FCConnectorParams) *FCConnector {
 		ioutil:    &wrp.IOUTILWrapper{},
 		baseConnector: newBaseConnector(mp, s,
 			baseConnectorParams{
-				MultipathFlushTimeout: params.MultipathFlushTimeout}),
+				MultipathFlushTimeout:      params.MultipathFlushTimeout,
+				MultipathFlushRetryTimeout: params.MultipathFlushRetryTimeout,
+				MultipathFlushRetries:      params.MultipathFlushRetries}),
 	}
 
 	setTimeouts(&conn.waitDeviceRegisterTimeout, params.WaitDeviceRegisterTimeout,

iscsi.go

@@ -22,6 +22,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
+	"os/exec"
+	"path"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
 	"github.com/dell/gobrick/internal/logger"
 	intmultipath "github.com/dell/gobrick/internal/multipath"
 	intscsi "github.com/dell/gobrick/internal/scsi"
@@ -32,13 +40,6 @@ import (
 	"github.com/dell/goiscsi"
 	"golang.org/x/sync/semaphore"
 	"golang.org/x/sync/singleflight"
-	"math"
-	"os/exec"
-	"path"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
 )
 
 const (
@@ -63,7 +64,9 @@ type ISCSIConnectorParams struct {
 	WaitDeviceRegisterTimeout              time.Duration
 	FailedSessionMinimumLoginRetryInterval time.Duration
 	MultipathFlushTimeout                  time.Duration
+	MultipathFlushRetryTimeout             time.Duration
 
+	MultipathFlushRetries int
 	MaxParallelOperations int
 }
 
@@ -77,10 +80,12 @@ func NewISCSIConnector(params ISCSIConnectorParams) *ISCSIConnector {
 		filePath:  &wrp.FilepathWrapper{},
 		baseConnector: newBaseConnector(mp, s,
 			baseConnectorParams{
-				MultipathFlushTimeout: params.MultipathFlushTimeout}),
+				MultipathFlushTimeout:      params.MultipathFlushTimeout,
+				MultipathFlushRetryTimeout: params.MultipathFlushRetryTimeout,
+				MultipathFlushRetries:      params.MultipathFlushRetries}),
 		chapPassword: params.ChapPassword,
-		chapUser: params.ChapUser,
-		chapEnabled: params.ChapEnabled,
+		chapUser:     params.ChapUser,
+		chapEnabled:  params.ChapEnabled,
 	}
 
 	iSCSIOpts := make(map[string]string)