br: stop log when full restore failed

br/pkg/checkpoint/checkpoint.go

@@ -596,7 +596,7 @@ func (r *CheckpointRunner[K, V]) getTS(ctx context.Context) (int64, int64, error
 		}
 
 		return nil
-	}, utils.NewPDReqBackoffer())
+	}, utils.NewPDReqBackofferExt())
 
 	return p, l, errors.Trace(errRetry)
 }

br/pkg/conn/conn.go

@@ -103,7 +103,7 @@ func GetAllTiKVStoresWithRetry(ctx context.Context,
 
 			return errors.Trace(err)
 		},
-		utils.NewPDReqBackoffer(),
+		utils.NewPDReqBackofferExt(),
 	)
 
 	return stores, errors.Trace(errRetry)
@@ -383,7 +383,7 @@ func (mgr *Mgr) GetConfigFromTiKV(ctx context.Context, cli *http.Client, fn func
 			}
 			_ = resp.Body.Close()
 			return nil
-		}, utils.NewPDReqBackoffer())
+		}, utils.NewPDReqBackofferExt())
 		if err != nil {
 			// if one store failed, break and return error
 			return err

br/pkg/restore/client.go

@@ -733,7 +733,7 @@ func (rc *Client) GetTSWithRetry(ctx context.Context) (uint64, error) {
 			log.Warn("failed to get TS, retry it", zap.Uint("retry time", retry), logutil.ShortError(getTSErr))
 		}
 		return getTSErr
-	}, utils.NewPDReqBackoffer())
+	}, utils.NewPDReqBackofferExt())
 
 	if err != nil {
 		log.Error("failed to get TS", zap.Error(err))
@@ -747,7 +747,7 @@ func (rc *Client) ResetTS(ctx context.Context, pdCtrl *pdutil.PdController) erro
 	log.Info("reset pd timestamp", zap.Uint64("ts", restoreTS))
 	return utils.WithRetry(ctx, func() error {
 		return pdCtrl.ResetTS(ctx, restoreTS)
-	}, utils.NewPDReqBackoffer())
+	}, utils.NewPDReqBackofferExt())
 }
 
 // GetPlacementRules return the current placement rules.
@@ -760,7 +760,7 @@ func (rc *Client) GetPlacementRules(ctx context.Context, pdAddrs []string) ([]pd
 		i++
 		placementRules, err = pdutil.GetPlacementRules(ctx, pdAddrs[idx], rc.tlsConf)
 		return errors.Trace(err)
-	}, utils.NewPDReqBackoffer())
+	}, utils.NewPDReqBackofferExt())
 	return placementRules, errors.Trace(errRetry)
 }
 
@@ -1520,12 +1520,14 @@ LOOPFORTABLE:
 			restoreFn := func() error {
 				filesGroups := getGroupFiles(filesReplica, rc.fileImporter.supportMultiIngest)
 				for _, filesGroup := range filesGroups {
-					if importErr := func(fs []*backuppb.File) error {
+					if importErr := func(fs []*backuppb.File) (err error) {
 						fileStart := time.Now()
 						defer func() {
-							log.Info("import files done", logutil.Files(filesGroup),
-								zap.Duration("take", time.Since(fileStart)))
-							updateCh.Inc()
+							if err == nil {
+								log.Info("import files done", logutil.Files(filesGroup),
+									zap.Duration("take", time.Since(fileStart)))
+								updateCh.Inc()
+							}
 						}()
 						return rc.fileImporter.ImportSSTFiles(ectx, fs, rewriteRules, rc.cipher, rc.dom.Store().GetCodec().GetAPIVersion())
 					}(filesGroup); importErr != nil {

br/pkg/restore/import.go

@@ -1186,13 +1186,11 @@ func (importer *FileImporter) downloadRawKVSSTV2(
 }
 
 func (importer *FileImporter) ingest(
-	c context.Context,
+	ctx context.Context,
 	files []*backuppb.File,
 	info *split.RegionInfo,
 	downloadMetas []*import_sstpb.SSTMeta,
 ) error {
-	ctx, cancel := context.WithTimeout(c, gRPCTimeOut)
-	defer cancel()
 	for {
 		ingestResp, errIngest := importer.ingestSSTs(ctx, downloadMetas, info)
 		if errIngest != nil {

br/pkg/utils/backoff.go

@@ -31,10 +31,6 @@ const (
 	backupSSTWaitInterval    = 2 * time.Second
 	backupSSTMaxWaitInterval = 3 * time.Second
 
-	resetTSRetryTime       = 16
-	resetTSWaitInterval    = 50 * time.Millisecond
-	resetTSMaxWaitInterval = 500 * time.Millisecond
-
 	resetTSRetryTimeExt       = 600
 	resetTSWaitIntervalExt    = 500 * time.Millisecond
 	resetTSMaxWaitIntervalExt = 300 * time.Second
@@ -167,7 +163,11 @@ func NewBackupSSTBackoffer() Backoffer {
 }
 
 func (bo *importerBackoffer) NextBackoff(err error) time.Duration {
-	log.Warn("retry to import ssts", zap.Int("attempt", bo.attempt), zap.Error(err))
+	defer func() {
+		if bo.attempt == 0 {
+			log.Warn("failed to import ssts by unretryable error or retry attempt exhausted", zap.Int("attempt", bo.attempt), zap.Error(err))
+		}
+	}()
 	// we don't care storeID here.
 	res := bo.errContext.HandleErrorMsg(err.Error(), 0)
 	if res.Strategy == RetryStrategy {
@@ -220,14 +220,6 @@ type pdReqBackoffer struct {
 	maxDelayTime time.Duration
 }
 
-func NewPDReqBackoffer() Backoffer {
-	return &pdReqBackoffer{
-		attempt:      resetTSRetryTime,
-		delayTime:    resetTSWaitInterval,
-		maxDelayTime: resetTSMaxWaitInterval,
-	}
-}
-
 func NewPDReqBackofferExt() Backoffer {
 	return &pdReqBackoffer{
 		attempt:      resetTSRetryTimeExt,
@@ -249,8 +241,12 @@ func (bo *pdReqBackoffer) NextBackoff(err error) time.Duration {
 		bo.delayTime = 2 * bo.delayTime
 		bo.attempt--
 	default:
+		// If the connection timeout, pd client would cancel the context, and return grpc context cancel error.
+		// So make the codes.Canceled retryable too.
+		// It's OK to retry the grpc context cancel error, because the parent context cancel returns context.Canceled.
+		// For example, cancel the `ectx` and then pdClient.GetTS(ectx) returns context.Canceled instead of grpc context canceled.
 		switch status.Code(e) {
-		case codes.DeadlineExceeded, codes.NotFound, codes.AlreadyExists, codes.PermissionDenied, codes.ResourceExhausted, codes.Aborted, codes.OutOfRange, codes.Unavailable, codes.DataLoss, codes.Unknown:
+		case codes.DeadlineExceeded, codes.Canceled, codes.NotFound, codes.AlreadyExists, codes.PermissionDenied, codes.ResourceExhausted, codes.Aborted, codes.OutOfRange, codes.Unavailable, codes.DataLoss, codes.Unknown:
 			bo.delayTime = 2 * bo.delayTime
 			bo.attempt--
 		default:

br/pkg/utils/backoff_test.go

@@ -116,7 +116,7 @@ func TestBackoffWithRetryableError(t *testing.T) {
 
 func TestPdBackoffWithRetryableError(t *testing.T) {
 	var counter int
-	backoffer := utils.NewPDReqBackoffer()
+	backoffer := utils.NewPDReqBackofferExt()
 	gRPCError := status.Error(codes.Unavailable, "transport is closing")
 	err := utils.WithRetry(context.Background(), func() error {
 		defer func() { counter++ }()