Add retry timeout

lib/backend/sqlbk/backend.go

@@ -101,6 +101,9 @@ func (b *Backend) CloseWatchers() {
 //
 // Returning an error from txFn will rollback the transaction and stop retries.
 func (b *Backend) retryTx(ctx context.Context, begin func(context.Context) Tx, txFn func(tx Tx) error) error {
+	ctx, cancel := context.WithTimeout(ctx, b.RetryTimeout)
+	defer cancel()
+
 	var delay *utils.Linear
 	tx := begin(ctx)
 	for {
@@ -122,14 +125,14 @@ func (b *Backend) retryTx(ctx context.Context, begin func(context.Context) Tx, t
 
 		// Retry transaction after delay.
 		if delay == nil {
-			period := b.RetryDelayPeriod
-			if period == 0 { // sanity check (0 produces an error in NewLinear)
-				period = DefaultRetryDelayPeriod
+			retryDelayPeriod := b.RetryDelayPeriod
+			if retryDelayPeriod == 0 { // sanity check (0 produces an error in NewLinear)
+				retryDelayPeriod = DefaultRetryDelayPeriod
 			}
 			delay, err = utils.NewLinear(utils.LinearConfig{
-				First:  period,
-				Step:   period,
-				Max:    period,
+				First:  retryDelayPeriod,
+				Step:   retryDelayPeriod,
+				Max:    retryDelayPeriod,
 				Jitter: utils.NewJitter(),
 			})
 			if err != nil {
@@ -142,7 +145,7 @@ func (b *Backend) retryTx(ctx context.Context, begin func(context.Context) Tx, t
 			delay.Inc()
 
 		case <-ctx.Done():
-			return ctx.Err()
+			return trace.Wrap(ctx.Err())
 		}
 	}
 }

lib/backend/sqlbk/config.go

@@ -32,9 +32,12 @@ const (
 	// DefaultDatabase is default name of the backend database.
 	DefaultDatabase = "teleport"
 
-	// DefaultRetryDelayPeriod  is the default delay before a transaction will retry on
+	// DefaultRetryDelayPeriod is the default delay before a transaction will retry on
 	// serialization failure.
 	DefaultRetryDelayPeriod = 250 * time.Millisecond
+
+	// DefaultRetryTimeout is the default amount time allocated to retrying transactions.
+	DefaultRetryTimeout = 10 * time.Second
 )
 
 // Config defines a configuration for the Backend.
@@ -74,6 +77,10 @@ type Config struct {
 	// serialization conflict.
 	RetryDelayPeriod time.Duration `json:"retry_period,omitempty"`
 
+	// RetryTimeout is the amount of time allocated to retrying transactions.
+	// Setting a value less than RetryDelayPeriod disables retries.
+	RetryTimeout time.Duration `json:"retry_timeout,omitempty"`
+
 	// Clock overrides the clock used by the backend.
 	Clock clockwork.Clock `json:"-"`
 
@@ -117,5 +124,8 @@ func (c *Config) CheckAndSetDefaults() error {
 	if c.RetryDelayPeriod == 0 {
 		c.RetryDelayPeriod = DefaultRetryDelayPeriod
 	}
+	if c.RetryTimeout == 0 {
+		c.RetryTimeout = DefaultRetryTimeout
+	}
 	return nil
 }