From 705793db45d73ce302f4177b2378f99f73ddf38a Mon Sep 17 00:00:00 2001 From: Benjamin2037 Date: Mon, 11 Jul 2022 16:00:50 +0800 Subject: [PATCH] Preparation for add index acceleration, handle 'import cycle'(#35983) --- br/pkg/backup/client.go | 3 +- br/pkg/conn/conn.go | 65 +- br/pkg/conn/conn_test.go | 27 +- br/pkg/conn/util/util.go | 62 ++ br/pkg/lightning/backend/kv/session_test.go | 5 +- br/pkg/lightning/backend/kv/sql2kv.go | 30 +- br/pkg/lightning/backend/kv/sql2kv_test.go | 162 ++-- br/pkg/lightning/backend/local/duplicate.go | 18 +- br/pkg/lightning/backend/local/local.go | 2 +- br/pkg/lightning/backend/local/local_test.go | 10 +- br/pkg/lightning/backend/local/localhelper.go | 2 +- .../backend/local/localhelper_test.go | 87 ++- br/pkg/lightning/config/config.go | 68 +- br/pkg/lightning/lightning.go | 2 + br/pkg/mock/backend.go | 14 + br/pkg/restore/client.go | 15 +- br/pkg/restore/import.go | 36 +- br/pkg/restore/import_retry.go | 17 +- br/pkg/restore/import_retry_test.go | 49 +- br/pkg/restore/range.go | 19 - br/pkg/restore/split.go | 93 +-- br/pkg/restore/split/region.go | 25 + br/pkg/restore/split/split.go | 27 + br/pkg/restore/split/split_client.go | 725 ++++++++++++++++++ br/pkg/restore/split_client.go | 614 +-------------- br/pkg/restore/split_test.go | 77 +- br/pkg/restore/util.go | 3 +- br/pkg/restore/util_test.go | 17 +- br/pkg/task/common.go | 3 +- 29 files changed, 1243 insertions(+), 1034 deletions(-) create mode 100644 br/pkg/conn/util/util.go create mode 100644 br/pkg/restore/split/region.go create mode 100644 br/pkg/restore/split/split.go create mode 100644 br/pkg/restore/split/split_client.go diff --git a/br/pkg/backup/client.go b/br/pkg/backup/client.go index 598f9d50c3006..a8fd02e769d49 100644 --- a/br/pkg/backup/client.go +++ b/br/pkg/backup/client.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" "github.com/pingcap/tidb/br/pkg/conn" + butil "github.com/pingcap/tidb/br/pkg/conn/util" berrors "github.com/pingcap/tidb/br/pkg/errors" "github.com/pingcap/tidb/br/pkg/logutil" "github.com/pingcap/tidb/br/pkg/metautil" @@ -607,7 +608,7 @@ func (bc *Client) BackupRange( zap.Uint32("concurrency", req.Concurrency)) var allStores []*metapb.Store - allStores, err = conn.GetAllTiKVStoresWithRetry(ctx, bc.mgr.GetPDClient(), conn.SkipTiFlash) + allStores, err = conn.GetAllTiKVStoresWithRetry(ctx, bc.mgr.GetPDClient(), butil.SkipTiFlash) if err != nil { return errors.Trace(err) } diff --git a/br/pkg/conn/conn.go b/br/pkg/conn/conn.go index c5996c3530b87..5adbe0a33ab1c 100755 --- a/br/pkg/conn/conn.go +++ b/br/pkg/conn/conn.go @@ -19,6 +19,7 @@ import ( logbackup "github.com/pingcap/kvproto/pkg/logbackuppb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" + "github.com/pingcap/tidb/br/pkg/conn/util" berrors "github.com/pingcap/tidb/br/pkg/errors" "github.com/pingcap/tidb/br/pkg/glue" "github.com/pingcap/tidb/br/pkg/logutil" @@ -27,7 +28,6 @@ import ( "github.com/pingcap/tidb/br/pkg/version" "github.com/pingcap/tidb/domain" "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/util/engine" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/tikv" "github.com/tikv/client-go/v2/txnkv/txnlock" @@ -68,60 +68,9 @@ type Mgr struct { *utils.StoreManager } -// StoreBehavior is the action to do in GetAllTiKVStores when a non-TiKV -// store (e.g. TiFlash store) is found. -type StoreBehavior uint8 - -const ( - // ErrorOnTiFlash causes GetAllTiKVStores to return error when the store is - // found to be a TiFlash node. - ErrorOnTiFlash StoreBehavior = 0 - // SkipTiFlash causes GetAllTiKVStores to skip the store when it is found to - // be a TiFlash node. - SkipTiFlash StoreBehavior = 1 - // TiFlashOnly caused GetAllTiKVStores to skip the store which is not a - // TiFlash node. - TiFlashOnly StoreBehavior = 2 -) - -// GetAllTiKVStores returns all TiKV stores registered to the PD client. The -// stores must not be a tombstone and must never contain a label `engine=tiflash`. -func GetAllTiKVStores( - ctx context.Context, - pdClient pd.Client, - storeBehavior StoreBehavior, -) ([]*metapb.Store, error) { - // get all live stores. - stores, err := pdClient.GetAllStores(ctx, pd.WithExcludeTombstone()) - if err != nil { - return nil, errors.Trace(err) - } - - // filter out all stores which are TiFlash. - j := 0 - for _, store := range stores { - isTiFlash := false - if engine.IsTiFlash(store) { - if storeBehavior == SkipTiFlash { - continue - } else if storeBehavior == ErrorOnTiFlash { - return nil, errors.Annotatef(berrors.ErrPDInvalidResponse, - "cannot restore to a cluster with active TiFlash stores (store %d at %s)", store.Id, store.Address) - } - isTiFlash = true - } - if !isTiFlash && storeBehavior == TiFlashOnly { - continue - } - stores[j] = store - j++ - } - return stores[:j], nil -} - func GetAllTiKVStoresWithRetry(ctx context.Context, pdClient pd.Client, - storeBehavior StoreBehavior, + storeBehavior util.StoreBehavior, ) ([]*metapb.Store, error) { stores := make([]*metapb.Store, 0) var err error @@ -129,7 +78,7 @@ func GetAllTiKVStoresWithRetry(ctx context.Context, errRetry := utils.WithRetry( ctx, func() error { - stores, err = GetAllTiKVStores(ctx, pdClient, storeBehavior) + stores, err = util.GetAllTiKVStores(ctx, pdClient, storeBehavior) failpoint.Inject("hint-GetAllTiKVStores-error", func(val failpoint.Value) { if val.(bool) { logutil.CL(ctx).Debug("failpoint hint-GetAllTiKVStores-error injected.") @@ -154,9 +103,9 @@ func GetAllTiKVStoresWithRetry(ctx context.Context, func checkStoresAlive(ctx context.Context, pdclient pd.Client, - storeBehavior StoreBehavior) error { + storeBehavior util.StoreBehavior) error { // Check live tikv. - stores, err := GetAllTiKVStores(ctx, pdclient, storeBehavior) + stores, err := util.GetAllTiKVStores(ctx, pdclient, storeBehavior) if err != nil { log.Error("fail to get store", zap.Error(err)) return errors.Trace(err) @@ -184,7 +133,7 @@ func NewMgr( tlsConf *tls.Config, securityOption pd.SecurityOption, keepalive keepalive.ClientParameters, - storeBehavior StoreBehavior, + storeBehavior util.StoreBehavior, checkRequirements bool, needDomain bool, versionCheckerType VersionCheckerType, @@ -368,7 +317,7 @@ func (mgr *Mgr) GetMergeRegionSizeAndCount(ctx context.Context, client *http.Cli // GetConfigFromTiKV get configs from all alive tikv stores. func (mgr *Mgr) GetConfigFromTiKV(ctx context.Context, cli *http.Client, fn func(*http.Response) error) error { - allStores, err := GetAllTiKVStoresWithRetry(ctx, mgr.GetPDClient(), SkipTiFlash) + allStores, err := GetAllTiKVStoresWithRetry(ctx, mgr.GetPDClient(), util.SkipTiFlash) if err != nil { return errors.Trace(err) } diff --git a/br/pkg/conn/conn_test.go b/br/pkg/conn/conn_test.go index ada9b25811d0b..01ce8bc08203e 100644 --- a/br/pkg/conn/conn_test.go +++ b/br/pkg/conn/conn_test.go @@ -14,6 +14,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/tidb/br/pkg/conn/util" "github.com/pingcap/tidb/br/pkg/pdutil" "github.com/pingcap/tidb/br/pkg/utils" "github.com/stretchr/testify/require" @@ -56,7 +57,7 @@ func TestGetAllTiKVStoresWithRetryCancel(t *testing.T) { Stores: stores, } - _, err := GetAllTiKVStoresWithRetry(ctx, fpdc, SkipTiFlash) + _, err := GetAllTiKVStoresWithRetry(ctx, fpdc, util.SkipTiFlash) require.Error(t, err) require.Equal(t, codes.Canceled, status.Code(errors.Cause(err))) } @@ -96,7 +97,7 @@ func TestGetAllTiKVStoresWithUnknown(t *testing.T) { Stores: stores, } - _, err := GetAllTiKVStoresWithRetry(ctx, fpdc, SkipTiFlash) + _, err := GetAllTiKVStoresWithRetry(ctx, fpdc, util.SkipTiFlash) require.Error(t, err) require.Equal(t, codes.Unknown, status.Code(errors.Cause(err))) } @@ -151,19 +152,19 @@ func TestCheckStoresAlive(t *testing.T) { Stores: stores, } - kvStores, err := GetAllTiKVStoresWithRetry(ctx, fpdc, SkipTiFlash) + kvStores, err := GetAllTiKVStoresWithRetry(ctx, fpdc, util.SkipTiFlash) require.NoError(t, err) require.Len(t, kvStores, 2) require.Equal(t, stores[2:], kvStores) - err = checkStoresAlive(ctx, fpdc, SkipTiFlash) + err = checkStoresAlive(ctx, fpdc, util.SkipTiFlash) require.NoError(t, err) } func TestGetAllTiKVStores(t *testing.T) { testCases := []struct { stores []*metapb.Store - storeBehavior StoreBehavior + storeBehavior util.StoreBehavior expectedStores map[uint64]int expectedError string }{ @@ -171,14 +172,14 @@ func TestGetAllTiKVStores(t *testing.T) { stores: []*metapb.Store{ {Id: 1}, }, - storeBehavior: SkipTiFlash, + storeBehavior: util.SkipTiFlash, expectedStores: map[uint64]int{1: 1}, }, { stores: []*metapb.Store{ {Id: 1}, }, - storeBehavior: ErrorOnTiFlash, + storeBehavior: util.ErrorOnTiFlash, expectedStores: map[uint64]int{1: 1}, }, { @@ -186,7 +187,7 @@ func TestGetAllTiKVStores(t *testing.T) { {Id: 1}, {Id: 2, Labels: []*metapb.StoreLabel{{Key: "engine", Value: "tiflash"}}}, }, - storeBehavior: SkipTiFlash, + storeBehavior: util.SkipTiFlash, expectedStores: map[uint64]int{1: 1}, }, { @@ -194,7 +195,7 @@ func TestGetAllTiKVStores(t *testing.T) { {Id: 1}, {Id: 2, Labels: []*metapb.StoreLabel{{Key: "engine", Value: "tiflash"}}}, }, - storeBehavior: ErrorOnTiFlash, + storeBehavior: util.ErrorOnTiFlash, expectedError: "^cannot restore to a cluster with active TiFlash stores", }, { @@ -206,7 +207,7 @@ func TestGetAllTiKVStores(t *testing.T) { {Id: 5, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tikv"}, {Key: "engine", Value: "tiflash"}}}, {Id: 6, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tiflash"}, {Key: "engine", Value: "tikv"}}}, }, - storeBehavior: SkipTiFlash, + storeBehavior: util.SkipTiFlash, expectedStores: map[uint64]int{1: 1, 3: 1, 4: 1, 6: 1}, }, { @@ -218,7 +219,7 @@ func TestGetAllTiKVStores(t *testing.T) { {Id: 5, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tikv"}, {Key: "engine", Value: "tiflash"}}}, {Id: 6, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tiflash"}, {Key: "engine", Value: "tikv"}}}, }, - storeBehavior: ErrorOnTiFlash, + storeBehavior: util.ErrorOnTiFlash, expectedError: "^cannot restore to a cluster with active TiFlash stores", }, { @@ -230,14 +231,14 @@ func TestGetAllTiKVStores(t *testing.T) { {Id: 5, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tikv"}, {Key: "engine", Value: "tiflash"}}}, {Id: 6, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tiflash"}, {Key: "engine", Value: "tikv"}}}, }, - storeBehavior: TiFlashOnly, + storeBehavior: util.TiFlashOnly, expectedStores: map[uint64]int{2: 1, 5: 1}, }, } for _, testCase := range testCases { pdClient := utils.FakePDClient{Stores: testCase.stores} - stores, err := GetAllTiKVStores(context.Background(), pdClient, testCase.storeBehavior) + stores, err := util.GetAllTiKVStores(context.Background(), pdClient, testCase.storeBehavior) if len(testCase.expectedError) != 0 { require.Error(t, err) require.Regexp(t, testCase.expectedError, err.Error()) diff --git a/br/pkg/conn/util/util.go b/br/pkg/conn/util/util.go new file mode 100644 index 0000000000000..5593dd016d412 --- /dev/null +++ b/br/pkg/conn/util/util.go @@ -0,0 +1,62 @@ +package util + +import ( + "context" + + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/metapb" + berrors "github.com/pingcap/tidb/br/pkg/errors" + "github.com/pingcap/tidb/util/engine" + pd "github.com/tikv/pd/client" +) + +// StoreBehavior is the action to do in GetAllTiKVStores when a non-TiKV +// store (e.g. TiFlash store) is found. +type StoreBehavior uint8 + +const ( + // ErrorOnTiFlash causes GetAllTiKVStores to return error when the store is + // found to be a TiFlash node. + ErrorOnTiFlash StoreBehavior = 0 + // SkipTiFlash causes GetAllTiKVStores to skip the store when it is found to + // be a TiFlash node. + SkipTiFlash StoreBehavior = 1 + // TiFlashOnly caused GetAllTiKVStores to skip the store which is not a + // TiFlash node. + TiFlashOnly StoreBehavior = 2 +) + +// GetAllTiKVStores returns all TiKV stores registered to the PD client. The +// stores must not be a tombstone and must never contain a label `engine=tiflash`. +func GetAllTiKVStores( + ctx context.Context, + pdClient pd.Client, + storeBehavior StoreBehavior, +) ([]*metapb.Store, error) { + // get all live stores. + stores, err := pdClient.GetAllStores(ctx, pd.WithExcludeTombstone()) + if err != nil { + return nil, errors.Trace(err) + } + + // filter out all stores which are TiFlash. + j := 0 + for _, store := range stores { + isTiFlash := false + if engine.IsTiFlash(store) { + if storeBehavior == SkipTiFlash { + continue + } else if storeBehavior == ErrorOnTiFlash { + return nil, errors.Annotatef(berrors.ErrPDInvalidResponse, + "cannot restore to a cluster with active TiFlash stores (store %d at %s)", store.Id, store.Address) + } + isTiFlash = true + } + if !isTiFlash && storeBehavior == TiFlashOnly { + continue + } + stores[j] = store + j++ + } + return stores[:j], nil +} diff --git a/br/pkg/lightning/backend/kv/session_test.go b/br/pkg/lightning/backend/kv/session_test.go index a37f48c190ed8..d6ce8784fdc7b 100644 --- a/br/pkg/lightning/backend/kv/session_test.go +++ b/br/pkg/lightning/backend/kv/session_test.go @@ -12,18 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kv +package kv_test import ( "testing" + "github.com/pingcap/tidb/br/pkg/lightning/backend/kv" "github.com/pingcap/tidb/br/pkg/lightning/log" "github.com/pingcap/tidb/parser/mysql" "github.com/stretchr/testify/require" ) func TestSession(t *testing.T) { - session := newSession(&SessionOptions{SQLMode: mysql.ModeNone, Timestamp: 1234567890}, log.L()) + session := kv.NewSession(&kv.SessionOptions{SQLMode: mysql.ModeNone, Timestamp: 1234567890}, log.L()) _, err := session.Txn(true) require.NoError(t, err) } diff --git a/br/pkg/lightning/backend/kv/sql2kv.go b/br/pkg/lightning/backend/kv/sql2kv.go index 66be51a19ec5e..d8d4f2940c527 100644 --- a/br/pkg/lightning/backend/kv/sql2kv.go +++ b/br/pkg/lightning/backend/kv/sql2kv.go @@ -33,8 +33,7 @@ import ( "github.com/pingcap/tidb/meta/autoid" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/parser/mysql" //nolint: goimports - // Import tidb/planner/core to initialize expression.RewriteAstExpr - _ "github.com/pingcap/tidb/planner/core" + "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/table/tables" @@ -65,6 +64,10 @@ type tableKVEncoder struct { metrics *metric.Metrics } +func GetSession4test(encoder Encoder) sessionctx.Context { + return encoder.(*tableKVEncoder).se +} + func NewTableKVEncoder( tbl table.Table, options *SessionOptions, @@ -322,6 +325,14 @@ func KvPairsFromRows(rows Rows) []common.KvPair { return rows.(*KvPairs).pairs } +// KvPairsFromRow converts a Rows instance constructed from MakeRowsFromKvPairs +// back into a slice of KvPair. This method panics if the Rows is not +// constructed in such way. +// nolint:golint // kv.KvPairsFromRow sounds good. +func KvPairsFromRow(rows Row) []common.KvPair { + return rows.(*KvPairs).pairs +} + func evaluateGeneratedColumns(se *session, record []types.Datum, cols []*table.Column, genCols []genCol) (err error, errCol *model.ColumnInfo) { mutRow := chunk.MutRowFromDatums(record) for _, gc := range genCols { @@ -449,6 +460,21 @@ func isPKCol(colInfo *model.ColumnInfo) bool { return mysql.HasPriKeyFlag(colInfo.GetFlag()) } +// GetEncoderAutoIDFn return Auto increment id. +func GetEncoderAutoIDFn(encoder Encoder, id int64) int64 { + return encoder.(*tableKVEncoder).autoIDFn(id) +} + +// GetEncoderSe return session. +func GetEncoderSe(encoder Encoder) *session { + return encoder.(*tableKVEncoder).se +} + +// GetActualDatum export getActualDatum function. +func GetActualDatum(encoder Encoder, rowID int64, colIndex int, inputDatum *types.Datum) (types.Datum, error) { + return encoder.(*tableKVEncoder).getActualDatum(70, 0, inputDatum) +} + func (kvcodec *tableKVEncoder) getActualDatum(rowID int64, colIndex int, inputDatum *types.Datum) (types.Datum, error) { var ( value types.Datum diff --git a/br/pkg/lightning/backend/kv/sql2kv_test.go b/br/pkg/lightning/backend/kv/sql2kv_test.go index b604942e38756..6a929f2c4e81d 100644 --- a/br/pkg/lightning/backend/kv/sql2kv_test.go +++ b/br/pkg/lightning/backend/kv/sql2kv_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kv +package kv_test import ( "errors" @@ -20,6 +20,7 @@ import ( "reflect" "testing" + lkv "github.com/pingcap/tidb/br/pkg/lightning/backend/kv" "github.com/pingcap/tidb/br/pkg/lightning/common" "github.com/pingcap/tidb/br/pkg/lightning/log" "github.com/pingcap/tidb/br/pkg/lightning/verification" @@ -47,7 +48,7 @@ func TestMarshal(t *testing.T) { minNotNull := types.Datum{} minNotNull.SetMinNotNull() encoder := zapcore.NewMapObjectEncoder() - err := encoder.AddArray("test", RowArrayMarshaler{types.NewStringDatum("1"), nullDatum, minNotNull, types.MaxValueDatum()}) + err := encoder.AddArray("test", lkv.RowArrayMarshaler{types.NewStringDatum("1"), nullDatum, minNotNull, types.MaxValueDatum()}) require.NoError(t, err) require.Equal(t, encoder.Fields["test"], []interface{}{ map[string]interface{}{"kind": "string", "val": "1"}, @@ -58,7 +59,7 @@ func TestMarshal(t *testing.T) { invalid := types.Datum{} invalid.SetInterface(1) - err = encoder.AddArray("bad-test", RowArrayMarshaler{minNotNull, invalid}) + err = encoder.AddArray("bad-test", lkv.RowArrayMarshaler{minNotNull, invalid}) require.Regexp(t, "cannot convert.*", err) require.Equal(t, encoder.Fields["bad-test"], []interface{}{ map[string]interface{}{"kind": "min", "val": "-inf"}, @@ -77,7 +78,7 @@ func TestEncode(t *testing.T) { c1 := &model.ColumnInfo{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeTiny)} cols := []*model.ColumnInfo{c1} tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} - tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) require.NoError(t, err) logger := log.Logger{Logger: zap.NewNop()} @@ -86,7 +87,7 @@ func TestEncode(t *testing.T) { } // Strict mode - strictMode, err := NewTableKVEncoder(tbl, &SessionOptions{ + strictMode, err := lkv.NewTableKVEncoder(tbl, &lkv.SessionOptions{ SQLMode: mysql.ModeStrictAllTables, Timestamp: 1234567890, }, nil, logger) @@ -108,17 +109,17 @@ func TestEncode(t *testing.T) { } pairs, err = strictMode.Encode(logger, rowsWithPk2, 2, []int{0, 1}, "1.csv", 1234) require.NoError(t, err) - require.Equal(t, pairs, &KvPairs{pairs: []common.KvPair{ + require.Equal(t, pairs, lkv.MakeRowFromKvPairs([]common.KvPair{ { Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, Val: []uint8{0x8, 0x2, 0x8, 0x2}, RowID: 2, }, - }}) + })) // Mock add record error mockTbl := &mockTable{Table: tbl} - mockMode, err := NewTableKVEncoder(mockTbl, &SessionOptions{ + mockMode, err := lkv.NewTableKVEncoder(mockTbl, &lkv.SessionOptions{ SQLMode: mysql.ModeStrictAllTables, Timestamp: 1234567891, }, nil, logger) @@ -127,7 +128,7 @@ func TestEncode(t *testing.T) { require.EqualError(t, err, "mock error") // Non-strict mode - noneMode, err := NewTableKVEncoder(tbl, &SessionOptions{ + noneMode, err := lkv.NewTableKVEncoder(tbl, &lkv.SessionOptions{ SQLMode: mysql.ModeNone, Timestamp: 1234567892, SysVars: map[string]string{"tidb_row_format_version": "1"}, @@ -135,22 +136,22 @@ func TestEncode(t *testing.T) { require.NoError(t, err) pairs, err = noneMode.Encode(logger, rows, 1, []int{0, 1}, "1.csv", 1234) require.NoError(t, err) - require.Equal(t, pairs, &KvPairs{pairs: []common.KvPair{ + require.Equal(t, pairs, lkv.MakeRowFromKvPairs([]common.KvPair{ { Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, Val: []uint8{0x8, 0x2, 0x8, 0xfe, 0x1}, RowID: 1, }, - }}) + })) } func TestDecode(t *testing.T) { c1 := &model.ColumnInfo{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeTiny)} cols := []*model.ColumnInfo{c1} tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} - tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) require.NoError(t, err) - decoder, err := NewTableKVDecoder(tbl, "`test`.`c1`", &SessionOptions{ + decoder, err := lkv.NewTableKVDecoder(tbl, "`test`.`c1`", &lkv.SessionOptions{ SQLMode: mysql.ModeStrictAllTables, Timestamp: 1234567890, }, log.L()) @@ -171,6 +172,15 @@ func TestDecode(t *testing.T) { }) } +type LocalKvPairs struct { + pairs []common.KvPair +} + +func fromRow(r lkv.Row) (l LocalKvPairs) { + l.pairs = lkv.KvPairsFromRow(r) + return l +} + func TestDecodeIndex(t *testing.T) { logger := log.Logger{Logger: zap.NewNop()} tblInfo := &model.TableInfo{ @@ -194,7 +204,7 @@ func TestDecodeIndex(t *testing.T) { State: model.StatePublic, PKIsHandle: false, } - tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) if err != nil { fmt.Printf("error: %v", err.Error()) } @@ -205,16 +215,16 @@ func TestDecodeIndex(t *testing.T) { } // Strict mode - strictMode, err := NewTableKVEncoder(tbl, &SessionOptions{ + strictMode, err := lkv.NewTableKVEncoder(tbl, &lkv.SessionOptions{ SQLMode: mysql.ModeStrictAllTables, Timestamp: 1234567890, }, nil, log.L()) require.NoError(t, err) pairs, err := strictMode.Encode(logger, rows, 1, []int{0, 1, -1}, "1.csv", 123) - data := pairs.(*KvPairs) + data := fromRow(pairs) require.Len(t, data.pairs, 2) - decoder, err := NewTableKVDecoder(tbl, "`test`.``", &SessionOptions{ + decoder, err := lkv.NewTableKVDecoder(tbl, "`test`.``", &lkv.SessionOptions{ SQLMode: mysql.ModeStrictAllTables, Timestamp: 1234567890, }, log.L()) @@ -235,7 +245,7 @@ func TestEncodeRowFormatV2(t *testing.T) { c1 := &model.ColumnInfo{ID: 1, Name: model.NewCIStr("c1"), State: model.StatePublic, Offset: 0, FieldType: *types.NewFieldType(mysql.TypeTiny)} cols := []*model.ColumnInfo{c1} tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} - tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) require.NoError(t, err) logger := log.Logger{Logger: zap.NewNop()} @@ -243,7 +253,7 @@ func TestEncodeRowFormatV2(t *testing.T) { types.NewIntDatum(10000000), } - noneMode, err := NewTableKVEncoder(tbl, &SessionOptions{ + noneMode, err := lkv.NewTableKVEncoder(tbl, &lkv.SessionOptions{ SQLMode: mysql.ModeNone, Timestamp: 1234567892, SysVars: map[string]string{"tidb_row_format_version": "2"}, @@ -251,7 +261,7 @@ func TestEncodeRowFormatV2(t *testing.T) { require.NoError(t, err) pairs, err := noneMode.Encode(logger, rows, 1, []int{0, 1}, "1.csv", 1234) require.NoError(t, err) - require.Equal(t, pairs, &KvPairs{pairs: []common.KvPair{ + require.Equal(t, pairs, lkv.MakeRowFromKvPairs([]common.KvPair{ { // the key should be the same as TestEncode() Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, @@ -266,7 +276,7 @@ func TestEncodeRowFormatV2(t *testing.T) { }, RowID: 1, }, - }}) + })) } func TestEncodeTimestamp(t *testing.T) { @@ -283,12 +293,12 @@ func TestEncodeTimestamp(t *testing.T) { } cols := []*model.ColumnInfo{c1} tblInfo := &model.TableInfo{ID: 1, Columns: cols, PKIsHandle: false, State: model.StatePublic} - tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) require.NoError(t, err) logger := log.Logger{Logger: zap.NewNop()} - encoder, err := NewTableKVEncoder(tbl, &SessionOptions{ + encoder, err := lkv.NewTableKVEncoder(tbl, &lkv.SessionOptions{ SQLMode: mysql.ModeStrictAllTables, Timestamp: 1234567893, SysVars: map[string]string{ @@ -299,23 +309,23 @@ func TestEncodeTimestamp(t *testing.T) { require.NoError(t, err) pairs, err := encoder.Encode(logger, nil, 70, []int{-1, 1}, "1.csv", 1234) require.NoError(t, err) - require.Equal(t, pairs, &KvPairs{pairs: []common.KvPair{ + require.Equal(t, pairs, lkv.MakeRowFromKvPairs([]common.KvPair{ { Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x46}, Val: []uint8{0x8, 0x2, 0x9, 0x80, 0x80, 0x80, 0xf0, 0xfd, 0x8e, 0xf7, 0xc0, 0x19}, RowID: 70, }, - }}) + })) } func TestEncodeDoubleAutoIncrement(t *testing.T) { tblInfo := mockTableInfo(t, "create table t (id double not null auto_increment, unique key `u_id` (`id`));") - tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) require.NoError(t, err) logger := log.Logger{Logger: zap.NewNop()} - encoder, err := NewTableKVEncoder(tbl, &SessionOptions{ + encoder, err := lkv.NewTableKVEncoder(tbl, &lkv.SessionOptions{ SQLMode: mysql.ModeStrictAllTables, SysVars: map[string]string{ "tidb_row_format_version": "2", @@ -324,7 +334,7 @@ func TestEncodeDoubleAutoIncrement(t *testing.T) { require.NoError(t, err) strDatumForID := types.NewStringDatum("1") - actualDatum, err := encoder.(*tableKVEncoder).getActualDatum(70, 0, &strDatumForID) + actualDatum, err := lkv.GetActualDatum(encoder, 70, 0, &strDatumForID) require.NoError(t, err) require.Equal(t, types.NewFloat64Datum(1.0), actualDatum) @@ -332,7 +342,7 @@ func TestEncodeDoubleAutoIncrement(t *testing.T) { types.NewFloat64Datum(1.0), }, 70, []int{0, -1}, "1.csv", 1234) require.NoError(t, err) - require.Equal(t, &KvPairs{pairs: []common.KvPair{ + require.Equal(t, lkv.MakeRowFromKvPairs([]common.KvPair{ { Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x46}, Val: []uint8{0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x1, 0x8, 0x0, 0xbf, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, @@ -343,7 +353,7 @@ func TestEncodeDoubleAutoIncrement(t *testing.T) { Val: []uint8{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x46}, RowID: 70, }, - }}, pairsExpect) + }), pairsExpect) pairs, err := encoder.Encode(logger, []types.Datum{ types.NewStringDatum("1"), @@ -351,7 +361,7 @@ func TestEncodeDoubleAutoIncrement(t *testing.T) { require.NoError(t, err) require.Equal(t, pairsExpect, pairs) - require.Equal(t, tbl.Allocators(encoder.(*tableKVEncoder).se).Get(autoid.AutoIncrementType).Base(), int64(70)) + require.Equal(t, tbl.Allocators(lkv.GetEncoderSe(encoder)).Get(autoid.AutoIncrementType).Base(), int64(70)) } func TestEncodeMissingAutoValue(t *testing.T) { @@ -378,10 +388,10 @@ func TestEncodeMissingAutoValue(t *testing.T) { // seems parser can't parse auto_random properly. tblInfo.AutoRandomBits = 3 } - tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) require.NoError(t, err) - encoder, err := NewTableKVEncoder(tbl, &SessionOptions{ + encoder, err := lkv.NewTableKVEncoder(tbl, &lkv.SessionOptions{ SQLMode: mysql.ModeStrictAllTables, SysVars: map[string]string{ "tidb_row_format_version": "2", @@ -389,17 +399,17 @@ func TestEncodeMissingAutoValue(t *testing.T) { }, nil, log.L()) require.NoError(t, err) - realRowID := encoder.(*tableKVEncoder).autoIDFn(rowID) + realRowID := lkv.GetEncoderAutoIDFn(encoder, rowID) var nullDatum types.Datum nullDatum.SetNull() expectIDDatum := types.NewIntDatum(realRowID) - actualIDDatum, err := encoder.(*tableKVEncoder).getActualDatum(rowID, 0, nil) + actualIDDatum, err := lkv.GetActualDatum(encoder, rowID, 0, nil) require.NoError(t, err) require.Equal(t, expectIDDatum, actualIDDatum) - actualIDDatum, err = encoder.(*tableKVEncoder).getActualDatum(rowID, 0, &nullDatum) + actualIDDatum, err = lkv.GetActualDatum(encoder, rowID, 0, &nullDatum) require.NoError(t, err) require.Equal(t, expectIDDatum, actualIDDatum) @@ -414,13 +424,13 @@ func TestEncodeMissingAutoValue(t *testing.T) { }, rowID, []int{0}, "1.csv", 1234) require.NoError(t, err) require.Equalf(t, pairsExpect, pairs, "test table info: %+v", testTblInfo) - require.Equalf(t, rowID, tbl.Allocators(encoder.(*tableKVEncoder).se).Get(testTblInfo.AllocType).Base(), "test table info: %+v", testTblInfo) + require.Equalf(t, rowID, tbl.Allocators(lkv.GetEncoderSe(encoder)).Get(testTblInfo.AllocType).Base(), "test table info: %+v", testTblInfo) // test insert a row without specifying the auto_xxxx column pairs, err = encoder.Encode(logger, []types.Datum{}, rowID, []int{0}, "1.csv", 1234) require.NoError(t, err) require.Equalf(t, pairsExpect, pairs, "test table info: %+v", testTblInfo) - require.Equalf(t, rowID, tbl.Allocators(encoder.(*tableKVEncoder).se).Get(testTblInfo.AllocType).Base(), "test table info: %+v", testTblInfo) + require.Equalf(t, rowID, tbl.Allocators(lkv.GetEncoderSe(encoder)).Get(testTblInfo.AllocType).Base(), "test table info: %+v", testTblInfo) } } @@ -440,9 +450,9 @@ func TestDefaultAutoRandoms(t *testing.T) { tblInfo := mockTableInfo(t, "create table t (id bigint unsigned NOT NULL auto_random primary key clustered, a varchar(100));") // seems parser can't parse auto_random properly. tblInfo.AutoRandomBits = 5 - tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) require.NoError(t, err) - encoder, err := NewTableKVEncoder(tbl, &SessionOptions{ + encoder, err := lkv.NewTableKVEncoder(tbl, &lkv.SessionOptions{ SQLMode: mysql.ModeStrictAllTables, Timestamp: 1234567893, SysVars: map[string]string{"tidb_row_format_version": "2"}, @@ -452,32 +462,32 @@ func TestDefaultAutoRandoms(t *testing.T) { logger := log.Logger{Logger: zap.NewNop()} pairs, err := encoder.Encode(logger, []types.Datum{types.NewStringDatum("")}, 70, []int{-1, 0}, "1.csv", 1234) require.NoError(t, err) - require.Equal(t, pairs, &KvPairs{pairs: []common.KvPair{ + require.Equal(t, pairs, lkv.MakeRowFromKvPairs([]common.KvPair{ { Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x46}, Val: []uint8{0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0}, RowID: 70, }, - }}) - require.Equal(t, tbl.Allocators(encoder.(*tableKVEncoder).se).Get(autoid.AutoRandomType).Base(), int64(70)) + })) + require.Equal(t, tbl.Allocators(lkv.GetSession4test(encoder)).Get(autoid.AutoRandomType).Base(), int64(70)) pairs, err = encoder.Encode(logger, []types.Datum{types.NewStringDatum("")}, 71, []int{-1, 0}, "1.csv", 1234) require.NoError(t, err) - require.Equal(t, pairs, &KvPairs{pairs: []common.KvPair{ + require.Equal(t, pairs, lkv.MakeRowFromKvPairs([]common.KvPair{ { Key: []uint8{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x5f, 0x72, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x47}, Val: []uint8{0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0}, RowID: 71, }, - }}) - require.Equal(t, tbl.Allocators(encoder.(*tableKVEncoder).se).Get(autoid.AutoRandomType).Base(), int64(71)) + })) + require.Equal(t, tbl.Allocators(lkv.GetSession4test(encoder)).Get(autoid.AutoRandomType).Base(), int64(71)) } func TestShardRowId(t *testing.T) { tblInfo := mockTableInfo(t, "create table t (s varchar(16)) shard_row_id_bits = 3;") - tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tblInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo) require.NoError(t, err) - encoder, err := NewTableKVEncoder(tbl, &SessionOptions{ + encoder, err := lkv.NewTableKVEncoder(tbl, &lkv.SessionOptions{ SQLMode: mysql.ModeStrictAllTables, Timestamp: 1234567893, SysVars: map[string]string{"tidb_row_format_version": "2"}, @@ -489,7 +499,7 @@ func TestShardRowId(t *testing.T) { for i := int64(1); i <= 32; i++ { pairs, err := encoder.Encode(logger, []types.Datum{types.NewStringDatum(fmt.Sprintf("%d", i))}, i, []int{0, -1}, "1.csv", i*32) require.NoError(t, err) - kvs := pairs.(*KvPairs) + kvs := fromRow(pairs) require.Len(t, kvs.pairs, 1) _, h, err := tablecodec.DecodeRecordKey(kvs.pairs[0].Key) require.NoError(t, err) @@ -498,7 +508,7 @@ func TestShardRowId(t *testing.T) { keyMap[rowID>>60] = struct{}{} } require.Len(t, keyMap, 8) - require.Equal(t, tbl.Allocators(encoder.(*tableKVEncoder).se).Get(autoid.RowIDAllocType).Base(), int64(32)) + require.Equal(t, tbl.Allocators(lkv.GetSession4test(encoder)).Get(autoid.RowIDAllocType).Base(), int64(32)) } func TestSplitIntoChunks(t *testing.T) { @@ -521,35 +531,35 @@ func TestSplitIntoChunks(t *testing.T) { }, } - splitBy10 := MakeRowsFromKvPairs(pairs).SplitIntoChunks(10) - require.Equal(t, splitBy10, []Rows{ - MakeRowsFromKvPairs(pairs[0:2]), - MakeRowsFromKvPairs(pairs[2:3]), - MakeRowsFromKvPairs(pairs[3:4]), + splitBy10 := lkv.MakeRowsFromKvPairs(pairs).SplitIntoChunks(10) + require.Equal(t, splitBy10, []lkv.Rows{ + lkv.MakeRowsFromKvPairs(pairs[0:2]), + lkv.MakeRowsFromKvPairs(pairs[2:3]), + lkv.MakeRowsFromKvPairs(pairs[3:4]), }) - splitBy12 := MakeRowsFromKvPairs(pairs).SplitIntoChunks(12) - require.Equal(t, splitBy12, []Rows{ - MakeRowsFromKvPairs(pairs[0:2]), - MakeRowsFromKvPairs(pairs[2:4]), + splitBy12 := lkv.MakeRowsFromKvPairs(pairs).SplitIntoChunks(12) + require.Equal(t, splitBy12, []lkv.Rows{ + lkv.MakeRowsFromKvPairs(pairs[0:2]), + lkv.MakeRowsFromKvPairs(pairs[2:4]), }) - splitBy1000 := MakeRowsFromKvPairs(pairs).SplitIntoChunks(1000) - require.Equal(t, splitBy1000, []Rows{ - MakeRowsFromKvPairs(pairs[0:4]), + splitBy1000 := lkv.MakeRowsFromKvPairs(pairs).SplitIntoChunks(1000) + require.Equal(t, splitBy1000, []lkv.Rows{ + lkv.MakeRowsFromKvPairs(pairs[0:4]), }) - splitBy1 := MakeRowsFromKvPairs(pairs).SplitIntoChunks(1) - require.Equal(t, splitBy1, []Rows{ - MakeRowsFromKvPairs(pairs[0:1]), - MakeRowsFromKvPairs(pairs[1:2]), - MakeRowsFromKvPairs(pairs[2:3]), - MakeRowsFromKvPairs(pairs[3:4]), + splitBy1 := lkv.MakeRowsFromKvPairs(pairs).SplitIntoChunks(1) + require.Equal(t, splitBy1, []lkv.Rows{ + lkv.MakeRowsFromKvPairs(pairs[0:1]), + lkv.MakeRowsFromKvPairs(pairs[1:2]), + lkv.MakeRowsFromKvPairs(pairs[2:3]), + lkv.MakeRowsFromKvPairs(pairs[3:4]), }) } func TestClassifyAndAppend(t *testing.T) { - kvs := MakeRowFromKvPairs([]common.KvPair{ + kvs := lkv.MakeRowFromKvPairs([]common.KvPair{ { Key: []byte("txxxxxxxx_ryyyyyyyy"), Val: []byte("value1"), @@ -564,14 +574,14 @@ func TestClassifyAndAppend(t *testing.T) { }, }) - data := MakeRowsFromKvPairs(nil) - indices := MakeRowsFromKvPairs(nil) + data := lkv.MakeRowsFromKvPairs(nil) + indices := lkv.MakeRowsFromKvPairs(nil) dataChecksum := verification.MakeKVChecksum(0, 0, 0) indexChecksum := verification.MakeKVChecksum(0, 0, 0) kvs.ClassifyAndAppend(&data, &dataChecksum, &indices, &indexChecksum) - require.Equal(t, data, MakeRowsFromKvPairs([]common.KvPair{ + require.Equal(t, data, lkv.MakeRowsFromKvPairs([]common.KvPair{ { Key: []byte("txxxxxxxx_ryyyyyyyy"), Val: []byte("value1"), @@ -581,7 +591,7 @@ func TestClassifyAndAppend(t *testing.T) { Val: []byte("value2"), }, })) - require.Equal(t, indices, MakeRowsFromKvPairs([]common.KvPair{ + require.Equal(t, indices, lkv.MakeRowsFromKvPairs([]common.KvPair{ { Key: []byte("txxxxxxxx_izzzzzzzz"), Val: []byte("index1"), @@ -594,7 +604,7 @@ func TestClassifyAndAppend(t *testing.T) { type benchSQL2KVSuite struct { row []types.Datum colPerm []int - encoder Encoder + encoder lkv.Encoder logger log.Logger } @@ -634,9 +644,9 @@ func SetUpTest(b *testing.B) *benchSQL2KVSuite { tableInfo.State = model.StatePublic // Construct the corresponding KV encoder. - tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tableInfo) + tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tableInfo) require.NoError(b, err) - encoder, err := NewTableKVEncoder(tbl, &SessionOptions{SysVars: map[string]string{"tidb_row_format_version": "2"}}, nil, log.L()) + encoder, err := lkv.NewTableKVEncoder(tbl, &lkv.SessionOptions{SysVars: map[string]string{"tidb_row_format_version": "2"}}, nil, log.L()) require.NoError(b, err) logger := log.Logger{Logger: zap.NewNop()} diff --git a/br/pkg/lightning/backend/local/duplicate.go b/br/pkg/lightning/backend/local/duplicate.go index b44a6c680f670..6e56f5dae04c3 100644 --- a/br/pkg/lightning/backend/local/duplicate.go +++ b/br/pkg/lightning/backend/local/duplicate.go @@ -33,7 +33,7 @@ import ( "github.com/pingcap/tidb/br/pkg/lightning/errormanager" "github.com/pingcap/tidb/br/pkg/lightning/log" "github.com/pingcap/tidb/br/pkg/logutil" - "github.com/pingcap/tidb/br/pkg/restore" + "github.com/pingcap/tidb/br/pkg/restore/split" "github.com/pingcap/tidb/br/pkg/utils" "github.com/pingcap/tidb/distsql" tidbkv "github.com/pingcap/tidb/kv" @@ -299,7 +299,7 @@ type RemoteDupKVStream struct { func getDupDetectClient( ctx context.Context, - region *restore.RegionInfo, + region *split.RegionInfo, keyRange tidbkv.KeyRange, importClientFactory ImportClientFactory, ) (import_sstpb.ImportSST_DuplicateDetectClient, error) { @@ -331,7 +331,7 @@ func getDupDetectClient( // NewRemoteDupKVStream creates a new RemoteDupKVStream. func NewRemoteDupKVStream( ctx context.Context, - region *restore.RegionInfo, + region *split.RegionInfo, keyRange tidbkv.KeyRange, importClientFactory ImportClientFactory, ) (*RemoteDupKVStream, error) { @@ -393,7 +393,7 @@ func (s *RemoteDupKVStream) Close() error { type DuplicateManager struct { tbl table.Table tableName string - splitCli restore.SplitClient + splitCli split.SplitClient tikvCli *tikv.KVStore errorMgr *errormanager.ErrorManager decoder *kv.TableKVDecoder @@ -406,7 +406,7 @@ type DuplicateManager struct { func NewDuplicateManager( tbl table.Table, tableName string, - splitCli restore.SplitClient, + splitCli split.SplitClient, tikvCli *tikv.KVStore, errMgr *errormanager.ErrorManager, sessOpts *kv.SessionOptions, @@ -661,14 +661,14 @@ func (m *DuplicateManager) CollectDuplicateRowsFromDupDB(ctx context.Context, du func (m *DuplicateManager) splitKeyRangeByRegions( ctx context.Context, keyRange tidbkv.KeyRange, -) ([]*restore.RegionInfo, []tidbkv.KeyRange, error) { +) ([]*split.RegionInfo, []tidbkv.KeyRange, error) { rawStartKey := codec.EncodeBytes(nil, keyRange.StartKey) rawEndKey := codec.EncodeBytes(nil, keyRange.EndKey) - allRegions, err := restore.PaginateScanRegion(ctx, m.splitCli, rawStartKey, rawEndKey, 1024) + allRegions, err := split.PaginateScanRegion(ctx, m.splitCli, rawStartKey, rawEndKey, 1024) if err != nil { return nil, nil, errors.Trace(err) } - regions := make([]*restore.RegionInfo, 0, len(allRegions)) + regions := make([]*split.RegionInfo, 0, len(allRegions)) keyRanges := make([]tidbkv.KeyRange, 0, len(allRegions)) for _, region := range allRegions { startKey := keyRange.StartKey @@ -711,7 +711,7 @@ func (m *DuplicateManager) processRemoteDupTaskOnce( remainKeyRanges *pendingKeyRanges, ) (madeProgress bool, err error) { //nolint: prealloc - var regions []*restore.RegionInfo + var regions []*split.RegionInfo //nolint: prealloc var keyRanges []tidbkv.KeyRange diff --git a/br/pkg/lightning/backend/local/local.go b/br/pkg/lightning/backend/local/local.go index b0a807f2d7b76..20625224807c8 100644 --- a/br/pkg/lightning/backend/local/local.go +++ b/br/pkg/lightning/backend/local/local.go @@ -49,7 +49,7 @@ import ( "github.com/pingcap/tidb/br/pkg/logutil" "github.com/pingcap/tidb/br/pkg/membuf" "github.com/pingcap/tidb/br/pkg/pdutil" - split "github.com/pingcap/tidb/br/pkg/restore" + "github.com/pingcap/tidb/br/pkg/restore/split" "github.com/pingcap/tidb/br/pkg/utils" "github.com/pingcap/tidb/br/pkg/version" "github.com/pingcap/tidb/infoschema" diff --git a/br/pkg/lightning/backend/local/local_test.go b/br/pkg/lightning/backend/local/local_test.go index 45030633b49c9..9e04e0ccfd848 100644 --- a/br/pkg/lightning/backend/local/local_test.go +++ b/br/pkg/lightning/backend/local/local_test.go @@ -46,7 +46,7 @@ import ( "github.com/pingcap/tidb/br/pkg/membuf" "github.com/pingcap/tidb/br/pkg/mock" "github.com/pingcap/tidb/br/pkg/pdutil" - "github.com/pingcap/tidb/br/pkg/restore" + "github.com/pingcap/tidb/br/pkg/restore/split" "github.com/pingcap/tidb/br/pkg/utils" tidbkv "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx/stmtctx" @@ -424,11 +424,11 @@ func TestLocalWriterWithIngestUnsort(t *testing.T) { } type mockSplitClient struct { - restore.SplitClient + split.SplitClient } -func (c *mockSplitClient) GetRegion(ctx context.Context, key []byte) (*restore.RegionInfo, error) { - return &restore.RegionInfo{ +func (c *mockSplitClient) GetRegion(ctx context.Context, key []byte) (*split.RegionInfo, error) { + return &split.RegionInfo{ Leader: &metapb.Peer{Id: 1}, Region: &metapb.Region{ Id: 1, @@ -451,7 +451,7 @@ func TestIsIngestRetryable(t *testing.T) { }, } ctx := context.Background() - region := &restore.RegionInfo{ + region := &split.RegionInfo{ Leader: &metapb.Peer{Id: 1}, Region: &metapb.Region{ Id: 1, diff --git a/br/pkg/lightning/backend/local/localhelper.go b/br/pkg/lightning/backend/local/localhelper.go index eae62b813e531..8e0cd12723612 100644 --- a/br/pkg/lightning/backend/local/localhelper.go +++ b/br/pkg/lightning/backend/local/localhelper.go @@ -34,7 +34,7 @@ import ( "github.com/pingcap/tidb/br/pkg/lightning/common" "github.com/pingcap/tidb/br/pkg/lightning/log" "github.com/pingcap/tidb/br/pkg/logutil" - split "github.com/pingcap/tidb/br/pkg/restore" + "github.com/pingcap/tidb/br/pkg/restore/split" "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/mathutil" "go.uber.org/multierr" diff --git a/br/pkg/lightning/backend/local/localhelper_test.go b/br/pkg/lightning/backend/local/localhelper_test.go index 8d3d367443ac8..fcdc49078a2b4 100644 --- a/br/pkg/lightning/backend/local/localhelper_test.go +++ b/br/pkg/lightning/backend/local/localhelper_test.go @@ -30,7 +30,7 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/tidb/br/pkg/lightning/glue" "github.com/pingcap/tidb/br/pkg/lightning/log" - "github.com/pingcap/tidb/br/pkg/restore" + "github.com/pingcap/tidb/br/pkg/restore/split" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/sessionctx/stmtctx" @@ -44,14 +44,13 @@ import ( func init() { // Reduce the time cost for test cases. - restore.ScanRegionAttemptTimes = 2 splitRetryTimes = 2 } type testClient struct { mu sync.RWMutex stores map[uint64]*metapb.Store - regions map[uint64]*restore.RegionInfo + regions map[uint64]*split.RegionInfo regionsInfo *pdtypes.RegionTree // For now it's only used in ScanRegions nextRegionID uint64 splitCount atomic.Int32 @@ -60,7 +59,7 @@ type testClient struct { func newTestClient( stores map[uint64]*metapb.Store, - regions map[uint64]*restore.RegionInfo, + regions map[uint64]*split.RegionInfo, nextRegionID uint64, hook clientHook, ) *testClient { @@ -78,11 +77,11 @@ func newTestClient( } // ScatterRegions scatters regions in a batch. -func (c *testClient) ScatterRegions(ctx context.Context, regionInfo []*restore.RegionInfo) error { +func (c *testClient) ScatterRegions(ctx context.Context, regionInfo []*split.RegionInfo) error { return nil } -func (c *testClient) GetAllRegions() map[uint64]*restore.RegionInfo { +func (c *testClient) GetAllRegions() map[uint64]*split.RegionInfo { c.mu.RLock() defer c.mu.RUnlock() return c.regions @@ -98,7 +97,7 @@ func (c *testClient) GetStore(ctx context.Context, storeID uint64) (*metapb.Stor return store, nil } -func (c *testClient) GetRegion(ctx context.Context, key []byte) (*restore.RegionInfo, error) { +func (c *testClient) GetRegion(ctx context.Context, key []byte) (*split.RegionInfo, error) { c.mu.RLock() defer c.mu.RUnlock() for _, region := range c.regions { @@ -110,7 +109,7 @@ func (c *testClient) GetRegion(ctx context.Context, key []byte) (*restore.Region return nil, errors.Errorf("region not found: key=%s", string(key)) } -func (c *testClient) GetRegionByID(ctx context.Context, regionID uint64) (*restore.RegionInfo, error) { +func (c *testClient) GetRegionByID(ctx context.Context, regionID uint64) (*split.RegionInfo, error) { c.mu.RLock() defer c.mu.RUnlock() region, ok := c.regions[regionID] @@ -122,12 +121,12 @@ func (c *testClient) GetRegionByID(ctx context.Context, regionID uint64) (*resto func (c *testClient) SplitRegion( ctx context.Context, - regionInfo *restore.RegionInfo, + regionInfo *split.RegionInfo, key []byte, -) (*restore.RegionInfo, error) { +) (*split.RegionInfo, error) { c.mu.Lock() defer c.mu.Unlock() - var target *restore.RegionInfo + var target *split.RegionInfo splitKey := codec.EncodeBytes([]byte{}, key) for _, region := range c.regions { if bytes.Compare(splitKey, region.Region.StartKey) >= 0 && @@ -138,7 +137,7 @@ func (c *testClient) SplitRegion( if target == nil { return nil, errors.Errorf("region not found: key=%s", string(key)) } - newRegion := &restore.RegionInfo{ + newRegion := &split.RegionInfo{ Region: &metapb.Region{ Peers: target.Region.Peers, Id: c.nextRegionID, @@ -161,8 +160,8 @@ func (c *testClient) SplitRegion( } func (c *testClient) BatchSplitRegionsWithOrigin( - ctx context.Context, regionInfo *restore.RegionInfo, keys [][]byte, -) (*restore.RegionInfo, []*restore.RegionInfo, error) { + ctx context.Context, regionInfo *split.RegionInfo, keys [][]byte, +) (*split.RegionInfo, []*split.RegionInfo, error) { c.mu.Lock() defer c.mu.Unlock() c.splitCount.Inc() @@ -180,7 +179,7 @@ func (c *testClient) BatchSplitRegionsWithOrigin( default: } - newRegions := make([]*restore.RegionInfo, 0) + newRegions := make([]*split.RegionInfo, 0) target, ok := c.regions[regionInfo.Region.Id] if !ok { return nil, nil, errors.New("region not found") @@ -203,7 +202,7 @@ func (c *testClient) BatchSplitRegionsWithOrigin( if bytes.Compare(key, startKey) <= 0 || bytes.Compare(key, target.Region.EndKey) >= 0 { continue } - newRegion := &restore.RegionInfo{ + newRegion := &split.RegionInfo{ Region: &metapb.Region{ Peers: target.Region.Peers, Id: c.nextRegionID, @@ -236,13 +235,13 @@ func (c *testClient) BatchSplitRegionsWithOrigin( } func (c *testClient) BatchSplitRegions( - ctx context.Context, regionInfo *restore.RegionInfo, keys [][]byte, -) ([]*restore.RegionInfo, error) { + ctx context.Context, regionInfo *split.RegionInfo, keys [][]byte, +) ([]*split.RegionInfo, error) { _, newRegions, err := c.BatchSplitRegionsWithOrigin(ctx, regionInfo, keys) return newRegions, err } -func (c *testClient) ScatterRegion(ctx context.Context, regionInfo *restore.RegionInfo) error { +func (c *testClient) ScatterRegion(ctx context.Context, regionInfo *split.RegionInfo) error { return nil } @@ -252,15 +251,15 @@ func (c *testClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.Ge }, nil } -func (c *testClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*restore.RegionInfo, error) { +func (c *testClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*split.RegionInfo, error) { if c.hook != nil { key, endKey, limit = c.hook.BeforeScanRegions(ctx, key, endKey, limit) } infos := c.regionsInfo.ScanRange(key, endKey, limit) - regions := make([]*restore.RegionInfo, 0, len(infos)) + regions := make([]*split.RegionInfo, 0, len(infos)) for _, info := range infos { - regions = append(regions, &restore.RegionInfo{ + regions = append(regions, &split.RegionInfo{ Region: info.Meta, Leader: info.Leader, }) @@ -289,7 +288,7 @@ func (c *testClient) SetStoresLabel(ctx context.Context, stores []uint64, labelK return nil } -func cloneRegion(region *restore.RegionInfo) *restore.RegionInfo { +func cloneRegion(region *split.RegionInfo) *split.RegionInfo { r := &metapb.Region{} if region.Region != nil { b, _ := region.Region.Marshal() @@ -301,7 +300,7 @@ func cloneRegion(region *restore.RegionInfo) *restore.RegionInfo { b, _ := region.Region.Marshal() _ = l.Unmarshal(b) } - return &restore.RegionInfo{Region: r, Leader: l} + return &split.RegionInfo{Region: r, Leader: l} } // For keys ["", "aay", "bba", "bbh", "cca", ""], the key ranges of @@ -312,7 +311,7 @@ func initTestClient(keys [][]byte, hook clientHook) *testClient { Id: 1, StoreId: 1, } - regions := make(map[uint64]*restore.RegionInfo) + regions := make(map[uint64]*split.RegionInfo) for i := uint64(1); i < uint64(len(keys)); i++ { startKey := keys[i-1] if len(startKey) != 0 { @@ -322,7 +321,7 @@ func initTestClient(keys [][]byte, hook clientHook) *testClient { if len(endKey) != 0 { endKey = codec.EncodeBytes([]byte{}, endKey) } - regions[i] = &restore.RegionInfo{ + regions[i] = &split.RegionInfo{ Region: &metapb.Region{ Id: i, Peers: peers, @@ -339,7 +338,7 @@ func initTestClient(keys [][]byte, hook clientHook) *testClient { return newTestClient(stores, regions, uint64(len(keys)), hook) } -func checkRegionRanges(t *testing.T, regions []*restore.RegionInfo, keys [][]byte) { +func checkRegionRanges(t *testing.T, regions []*split.RegionInfo, keys [][]byte) { for i, r := range regions { _, regionStart, _ := codec.DecodeBytes(r.Region.StartKey, []byte{}) _, regionEnd, _ := codec.DecodeBytes(r.Region.EndKey, []byte{}) @@ -349,21 +348,21 @@ func checkRegionRanges(t *testing.T, regions []*restore.RegionInfo, keys [][]byt } type clientHook interface { - BeforeSplitRegion(ctx context.Context, regionInfo *restore.RegionInfo, keys [][]byte) (*restore.RegionInfo, [][]byte) - AfterSplitRegion(context.Context, *restore.RegionInfo, [][]byte, []*restore.RegionInfo, error) ([]*restore.RegionInfo, error) + BeforeSplitRegion(ctx context.Context, regionInfo *split.RegionInfo, keys [][]byte) (*split.RegionInfo, [][]byte) + AfterSplitRegion(context.Context, *split.RegionInfo, [][]byte, []*split.RegionInfo, error) ([]*split.RegionInfo, error) BeforeScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]byte, []byte, int) - AfterScanRegions([]*restore.RegionInfo, error) ([]*restore.RegionInfo, error) + AfterScanRegions([]*split.RegionInfo, error) ([]*split.RegionInfo, error) } type noopHook struct{} -func (h *noopHook) BeforeSplitRegion(ctx context.Context, regionInfo *restore.RegionInfo, keys [][]byte) (*restore.RegionInfo, [][]byte) { +func (h *noopHook) BeforeSplitRegion(ctx context.Context, regionInfo *split.RegionInfo, keys [][]byte) (*split.RegionInfo, [][]byte) { delayTime := rand.Int31n(10) + 1 time.Sleep(time.Duration(delayTime) * time.Millisecond) return regionInfo, keys } -func (h *noopHook) AfterSplitRegion(c context.Context, r *restore.RegionInfo, keys [][]byte, res []*restore.RegionInfo, err error) ([]*restore.RegionInfo, error) { +func (h *noopHook) AfterSplitRegion(c context.Context, r *split.RegionInfo, keys [][]byte, res []*split.RegionInfo, err error) ([]*split.RegionInfo, error) { return res, err } @@ -371,7 +370,7 @@ func (h *noopHook) BeforeScanRegions(ctx context.Context, key, endKey []byte, li return key, endKey, limit } -func (h *noopHook) AfterScanRegions(res []*restore.RegionInfo, err error) ([]*restore.RegionInfo, error) { +func (h *noopHook) AfterScanRegions(res []*split.RegionInfo, err error) ([]*split.RegionInfo, error) { return res, err } @@ -425,7 +424,7 @@ func doTestBatchSplitRegionByRanges(ctx context.Context, t *testing.T, hook clie // current region ranges: [, aay), [aay, bba), [bba, bbh), [bbh, cca), [cca, ) rangeStart := codec.EncodeBytes([]byte{}, []byte("b")) rangeEnd := codec.EncodeBytes([]byte{}, []byte("c")) - regions, err := restore.PaginateScanRegion(ctx, client, rangeStart, rangeEnd, 5) + regions, err := split.PaginateScanRegion(ctx, client, rangeStart, rangeEnd, 5) require.NoError(t, err) // regions is: [aay, bba), [bba, bbh), [bbh, cca) checkRegionRanges(t, regions, [][]byte{[]byte("aay"), []byte("bba"), []byte("bbh"), []byte("cca")}) @@ -451,7 +450,7 @@ func doTestBatchSplitRegionByRanges(ctx context.Context, t *testing.T, hook clie splitHook.check(t, client) // check split ranges - regions, err = restore.PaginateScanRegion(ctx, client, rangeStart, rangeEnd, 5) + regions, err = split.PaginateScanRegion(ctx, client, rangeStart, rangeEnd, 5) require.NoError(t, err) result := [][]byte{ []byte("b"), []byte("ba"), []byte("bb"), []byte("bba"), []byte("bbh"), []byte("bc"), @@ -505,7 +504,7 @@ type scanRegionEmptyHook struct { cnt int } -func (h *scanRegionEmptyHook) AfterScanRegions(res []*restore.RegionInfo, err error) ([]*restore.RegionInfo, error) { +func (h *scanRegionEmptyHook) AfterScanRegions(res []*split.RegionInfo, err error) ([]*split.RegionInfo, error) { h.cnt++ // skip the first call if h.cnt == 1 { @@ -522,7 +521,7 @@ type splitRegionEpochNotMatchHook struct { noopHook } -func (h *splitRegionEpochNotMatchHook) BeforeSplitRegion(ctx context.Context, regionInfo *restore.RegionInfo, keys [][]byte) (*restore.RegionInfo, [][]byte) { +func (h *splitRegionEpochNotMatchHook) BeforeSplitRegion(ctx context.Context, regionInfo *split.RegionInfo, keys [][]byte) (*split.RegionInfo, [][]byte) { regionInfo, keys = h.noopHook.BeforeSplitRegion(ctx, regionInfo, keys) regionInfo = cloneRegion(regionInfo) // decrease the region epoch, so split region will fail @@ -540,7 +539,7 @@ type splitRegionEpochNotMatchHookRandom struct { cnt atomic.Int32 } -func (h *splitRegionEpochNotMatchHookRandom) BeforeSplitRegion(ctx context.Context, regionInfo *restore.RegionInfo, keys [][]byte) (*restore.RegionInfo, [][]byte) { +func (h *splitRegionEpochNotMatchHookRandom) BeforeSplitRegion(ctx context.Context, regionInfo *split.RegionInfo, keys [][]byte) (*split.RegionInfo, [][]byte) { regionInfo, keys = h.noopHook.BeforeSplitRegion(ctx, regionInfo, keys) if h.cnt.Inc() != 0 { return regionInfo, keys @@ -561,7 +560,7 @@ type splitRegionNoValidKeyHook struct { errorCnt atomic.Int32 } -func (h *splitRegionNoValidKeyHook) BeforeSplitRegion(ctx context.Context, regionInfo *restore.RegionInfo, keys [][]byte) (*restore.RegionInfo, [][]byte) { +func (h *splitRegionNoValidKeyHook) BeforeSplitRegion(ctx context.Context, regionInfo *split.RegionInfo, keys [][]byte) (*split.RegionInfo, [][]byte) { regionInfo, keys = h.noopHook.BeforeSplitRegion(ctx, regionInfo, keys) if h.errorCnt.Inc() <= h.returnErrTimes { // clean keys to trigger "no valid keys" error @@ -607,7 +606,7 @@ func TestSplitAndScatterRegionInBatches(t *testing.T) { rangeStart := codec.EncodeBytes([]byte{}, []byte("a")) rangeEnd := codec.EncodeBytes([]byte{}, []byte("b")) - regions, err := restore.PaginateScanRegion(ctx, client, rangeStart, rangeEnd, 5) + regions, err := split.PaginateScanRegion(ctx, client, rangeStart, rangeEnd, 5) require.NoError(t, err) result := [][]byte{[]byte("a"), []byte("a00"), []byte("a01"), []byte("a02"), []byte("a03"), []byte("a04"), []byte("a05"), []byte("a06"), []byte("a07"), []byte("a08"), []byte("a09"), []byte("a10"), []byte("a11"), @@ -622,7 +621,7 @@ type reportAfterSplitHook struct { ch chan<- struct{} } -func (h *reportAfterSplitHook) AfterSplitRegion(ctx context.Context, region *restore.RegionInfo, keys [][]byte, resultRegions []*restore.RegionInfo, err error) ([]*restore.RegionInfo, error) { +func (h *reportAfterSplitHook) AfterSplitRegion(ctx context.Context, region *split.RegionInfo, keys [][]byte, resultRegions []*split.RegionInfo, err error) ([]*split.RegionInfo, error) { h.ch <- struct{}{} return resultRegions, err } @@ -705,7 +704,7 @@ func doTestBatchSplitByRangesWithClusteredIndex(t *testing.T, hook clientHook) { startKey := codec.EncodeBytes([]byte{}, rangeKeys[0]) endKey := codec.EncodeBytes([]byte{}, rangeKeys[len(rangeKeys)-1]) // check split ranges - regions, err := restore.PaginateScanRegion(ctx, client, startKey, endKey, 5) + regions, err := split.PaginateScanRegion(ctx, client, startKey, endKey, 5) require.NoError(t, err) require.Equal(t, len(ranges)+1, len(regions)) @@ -733,14 +732,14 @@ func TestNeedSplit(t *testing.T) { keys := []int64{10, 100, 500, 1000, 999999, -1} start := tablecodec.EncodeRowKeyWithHandle(tableID, kv.IntHandle(0)) regionStart := codec.EncodeBytes([]byte{}, start) - regions := make([]*restore.RegionInfo, 0) + regions := make([]*split.RegionInfo, 0) for _, end := range keys { var regionEndKey []byte if end >= 0 { endKey := tablecodec.EncodeRowKeyWithHandle(tableID, kv.IntHandle(end)) regionEndKey = codec.EncodeBytes([]byte{}, endKey) } - region := &restore.RegionInfo{ + region := &split.RegionInfo{ Region: &metapb.Region{ Id: 1, Peers: peers, diff --git a/br/pkg/lightning/config/config.go b/br/pkg/lightning/config/config.go index 0066895568550..f88e7693d6664 100644 --- a/br/pkg/lightning/config/config.go +++ b/br/pkg/lightning/config/config.go @@ -861,8 +861,39 @@ func (cfg *Config) Adjust(ctx context.Context) error { zap.ByteString("invalid-char-replacement", []byte(cfg.Mydumper.DataInvalidCharReplace))) } + mustHaveInternalConnections, err := cfg.AdjustCommon() + if err != nil { + return err + } + + // mydumper.filter and black-white-list cannot co-exist. + if cfg.HasLegacyBlackWhiteList() { + log.L().Warn("the config `black-white-list` has been deprecated, please replace with `mydumper.filter`") + if !common.StringSliceEqual(cfg.Mydumper.Filter, DefaultFilter) { + return common.ErrInvalidConfig.GenWithStack("`mydumper.filter` and `black-white-list` cannot be simultaneously defined") + } + } + + for _, rule := range cfg.Routes { + if !cfg.Mydumper.CaseSensitive { + rule.ToLower() + } + if err := rule.Valid(); err != nil { + return common.ErrInvalidConfig.Wrap(err).GenWithStack("file route rule is invalid") + } + } + + if err := cfg.CheckAndAdjustTiDBPort(ctx, mustHaveInternalConnections); err != nil { + return err + } + cfg.AdjustMydumper() + cfg.AdjustCheckPoint() + return cfg.CheckAndAdjustFilePath() +} + +func (cfg *Config) AdjustCommon() (bool, error) { if cfg.TikvImporter.Backend == "" { - return common.ErrInvalidConfig.GenWithStack("tikv-importer.backend must not be empty!") + return false, common.ErrInvalidConfig.GenWithStack("tikv-importer.backend must not be empty!") } cfg.TikvImporter.Backend = strings.ToLower(cfg.TikvImporter.Backend) mustHaveInternalConnections := true @@ -881,7 +912,7 @@ func (cfg *Config) Adjust(ctx context.Context) error { } cfg.DefaultVarsForImporterAndLocalBackend() default: - return common.ErrInvalidConfig.GenWithStack("unsupported `tikv-importer.backend` (%s)", cfg.TikvImporter.Backend) + return mustHaveInternalConnections, common.ErrInvalidConfig.GenWithStack("unsupported `tikv-importer.backend` (%s)", cfg.TikvImporter.Backend) } // TODO calculate these from the machine's free memory. @@ -894,7 +925,7 @@ func (cfg *Config) Adjust(ctx context.Context) error { if cfg.TikvImporter.Backend == BackendLocal { if err := cfg.CheckAndAdjustForLocalBackend(); err != nil { - return err + return mustHaveInternalConnections, err } } else { cfg.TikvImporter.DuplicateResolution = DupeResAlgNone @@ -905,7 +936,7 @@ func (cfg *Config) Adjust(ctx context.Context) error { switch cfg.TikvImporter.OnDuplicate { case ReplaceOnDup, IgnoreOnDup, ErrorOnDup: default: - return common.ErrInvalidConfig.GenWithStack( + return mustHaveInternalConnections, common.ErrInvalidConfig.GenWithStack( "unsupported `tikv-importer.on-duplicate` (%s)", cfg.TikvImporter.OnDuplicate) } } @@ -913,36 +944,13 @@ func (cfg *Config) Adjust(ctx context.Context) error { var err error cfg.TiDB.SQLMode, err = mysql.GetSQLMode(cfg.TiDB.StrSQLMode) if err != nil { - return common.ErrInvalidConfig.Wrap(err).GenWithStack("`mydumper.tidb.sql_mode` must be a valid SQL_MODE") + return mustHaveInternalConnections, common.ErrInvalidConfig.Wrap(err).GenWithStack("`mydumper.tidb.sql_mode` must be a valid SQL_MODE") } if err := cfg.CheckAndAdjustSecurity(); err != nil { - return err - } - - // mydumper.filter and black-white-list cannot co-exist. - if cfg.HasLegacyBlackWhiteList() { - log.L().Warn("the config `black-white-list` has been deprecated, please replace with `mydumper.filter`") - if !common.StringSliceEqual(cfg.Mydumper.Filter, DefaultFilter) { - return common.ErrInvalidConfig.GenWithStack("`mydumper.filter` and `black-white-list` cannot be simultaneously defined") - } + return mustHaveInternalConnections, err } - - for _, rule := range cfg.Routes { - if !cfg.Mydumper.CaseSensitive { - rule.ToLower() - } - if err := rule.Valid(); err != nil { - return common.ErrInvalidConfig.Wrap(err).GenWithStack("file route rule is invalid") - } - } - - if err := cfg.CheckAndAdjustTiDBPort(ctx, mustHaveInternalConnections); err != nil { - return err - } - cfg.AdjustMydumper() - cfg.AdjustCheckPoint() - return cfg.CheckAndAdjustFilePath() + return mustHaveInternalConnections, err } func (cfg *Config) CheckAndAdjustForLocalBackend() error { diff --git a/br/pkg/lightning/lightning.go b/br/pkg/lightning/lightning.go index 4138623830bba..5b3796adaac3a 100644 --- a/br/pkg/lightning/lightning.go +++ b/br/pkg/lightning/lightning.go @@ -51,6 +51,8 @@ import ( "github.com/pingcap/tidb/br/pkg/storage" "github.com/pingcap/tidb/br/pkg/utils" "github.com/pingcap/tidb/br/pkg/version/build" + _ "github.com/pingcap/tidb/expression" // get rid of `import cycle`: just init expression.RewriteAstExpr,and called at package `backend.kv` + _ "github.com/pingcap/tidb/planner/core" "github.com/pingcap/tidb/util/promutil" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" diff --git a/br/pkg/mock/backend.go b/br/pkg/mock/backend.go index 04896d4a8efd1..f5d55a971d1ff 100644 --- a/br/pkg/mock/backend.go +++ b/br/pkg/mock/backend.go @@ -347,6 +347,20 @@ func (mr *MockEngineWriterMockRecorder) AppendRows(arg0, arg1, arg2, arg3 interf return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendRows", reflect.TypeOf((*MockEngineWriter)(nil).AppendRows), arg0, arg1, arg2, arg3) } +// AppendRow mocks base method. +func (m *MockEngineWriter) AppendRow(arg0 context.Context, arg1 string, arg2 []string, arg3 kv.Rows) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AppendRow", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// AppendRow indicates an expected call of AppendRows. +func (mr *MockEngineWriterMockRecorder) AppendRow(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendRow", reflect.TypeOf((*MockEngineWriter)(nil).AppendRows), arg0, arg1, arg2, arg3) +} + // Close mocks base method. func (m *MockEngineWriter) Close(arg0 context.Context) (backend.ChunkFlushStatus, error) { m.ctrl.T.Helper() diff --git a/br/pkg/restore/client.go b/br/pkg/restore/client.go index d67adb90dffe5..61798de656178 100644 --- a/br/pkg/restore/client.go +++ b/br/pkg/restore/client.go @@ -25,13 +25,14 @@ import ( "github.com/pingcap/log" "github.com/pingcap/tidb/br/pkg/backup" "github.com/pingcap/tidb/br/pkg/checksum" - "github.com/pingcap/tidb/br/pkg/conn" + "github.com/pingcap/tidb/br/pkg/conn/util" berrors "github.com/pingcap/tidb/br/pkg/errors" "github.com/pingcap/tidb/br/pkg/glue" "github.com/pingcap/tidb/br/pkg/logutil" "github.com/pingcap/tidb/br/pkg/metautil" "github.com/pingcap/tidb/br/pkg/pdutil" "github.com/pingcap/tidb/br/pkg/redact" + "github.com/pingcap/tidb/br/pkg/restore/split" "github.com/pingcap/tidb/br/pkg/rtree" "github.com/pingcap/tidb/br/pkg/storage" "github.com/pingcap/tidb/br/pkg/stream" @@ -76,7 +77,7 @@ const ( // Client sends requests to restore files. type Client struct { pdClient pd.Client - toolClient SplitClient + toolClient split.SplitClient fileImporter FileImporter rawKVClient *RawKVBatchClient workerPool *utils.WorkerPool @@ -174,7 +175,7 @@ func NewRestoreClient( ) *Client { return &Client{ pdClient: pdClient, - toolClient: NewSplitClient(pdClient, tlsConf, isRawKv), + toolClient: split.NewSplitClient(pdClient, tlsConf, isRawKv), tlsConf: tlsConf, keepaliveConf: keepaliveConf, switchCh: make(chan struct{}), @@ -335,7 +336,7 @@ func (rc *Client) SetStorage(ctx context.Context, backend *backuppb.StorageBacke } func (rc *Client) InitClients(backend *backuppb.StorageBackend, isRawKvMode bool) { - metaClient := NewSplitClient(rc.pdClient, rc.tlsConf, isRawKvMode) + metaClient := split.NewSplitClient(rc.pdClient, rc.tlsConf, isRawKvMode) importCli := NewImportClient(metaClient, rc.tlsConf, rc.keepaliveConf) rc.fileImporter = NewFileImporter(metaClient, importCli, backend, isRawKvMode) } @@ -989,7 +990,7 @@ func (rc *Client) ResetSpeedLimit(ctx context.Context) error { func (rc *Client) setSpeedLimit(ctx context.Context, rateLimit uint64) error { if !rc.hasSpeedLimited { - stores, err := conn.GetAllTiKVStores(ctx, rc.pdClient, conn.SkipTiFlash) + stores, err := util.GetAllTiKVStores(ctx, rc.pdClient, util.SkipTiFlash) if err != nil { return errors.Trace(err) } @@ -1203,7 +1204,7 @@ func (rc *Client) SwitchToNormalMode(ctx context.Context) error { } func (rc *Client) switchTiKVMode(ctx context.Context, mode import_sstpb.SwitchMode) error { - stores, err := conn.GetAllTiKVStores(ctx, rc.pdClient, conn.SkipTiFlash) + stores, err := util.GetAllTiKVStores(ctx, rc.pdClient, util.SkipTiFlash) if err != nil { return errors.Trace(err) } @@ -1635,7 +1636,7 @@ func (rc *Client) PreCheckTableTiFlashReplica( tables []*metautil.Table, skipTiflash bool, ) error { - tiFlashStores, err := conn.GetAllTiKVStores(ctx, rc.pdClient, conn.TiFlashOnly) + tiFlashStores, err := util.GetAllTiKVStores(ctx, rc.pdClient, util.TiFlashOnly) if err != nil { return errors.Trace(err) } diff --git a/br/pkg/restore/import.go b/br/pkg/restore/import.go index 63ac21f4b2e1c..4751b73398ec1 100644 --- a/br/pkg/restore/import.go +++ b/br/pkg/restore/import.go @@ -20,8 +20,10 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" "github.com/pingcap/tidb/br/pkg/conn" + "github.com/pingcap/tidb/br/pkg/conn/util" berrors "github.com/pingcap/tidb/br/pkg/errors" "github.com/pingcap/tidb/br/pkg/logutil" + "github.com/pingcap/tidb/br/pkg/restore/split" "github.com/pingcap/tidb/br/pkg/summary" "github.com/pingcap/tidb/br/pkg/utils" pd "github.com/tikv/pd/client" @@ -88,7 +90,7 @@ type ImporterClient interface { type importClient struct { mu sync.Mutex - metaClient SplitClient + metaClient split.SplitClient clients map[uint64]import_sstpb.ImportSSTClient tlsConf *tls.Config @@ -96,7 +98,7 @@ type importClient struct { } // NewImportClient returns a new ImporterClient. -func NewImportClient(metaClient SplitClient, tlsConf *tls.Config, keepaliveConf keepalive.ClientParameters) ImporterClient { +func NewImportClient(metaClient split.SplitClient, tlsConf *tls.Config, keepaliveConf keepalive.ClientParameters) ImporterClient { return &importClient{ metaClient: metaClient, clients: make(map[uint64]import_sstpb.ImportSSTClient), @@ -235,7 +237,7 @@ func (ic *importClient) SupportMultiIngest(ctx context.Context, stores []uint64) // FileImporter used to import a file to TiKV. type FileImporter struct { - metaClient SplitClient + metaClient split.SplitClient importClient ImporterClient backend *backuppb.StorageBackend @@ -247,7 +249,7 @@ type FileImporter struct { // NewFileImporter returns a new file importClient. func NewFileImporter( - metaClient SplitClient, + metaClient split.SplitClient, importClient ImporterClient, backend *backuppb.StorageBackend, isRawKvMode bool, @@ -262,7 +264,7 @@ func NewFileImporter( // CheckMultiIngestSupport checks whether all stores support multi-ingest func (importer *FileImporter) CheckMultiIngestSupport(ctx context.Context, pdClient pd.Client) error { - allStores, err := conn.GetAllTiKVStores(ctx, pdClient, conn.SkipTiFlash) + allStores, err := util.GetAllTiKVStores(ctx, pdClient, util.SkipTiFlash) if err != nil { return errors.Trace(err) } @@ -334,7 +336,7 @@ func (importer *FileImporter) ImportKVFileForRegion( rule *RewriteRules, startTS uint64, restoreTS uint64, - info *RegionInfo, + info *split.RegionInfo, ) RPCResult { // Try to download file. result := importer.downloadAndApplyKVFile(ctx, file, rule, info, startTS, restoreTS) @@ -359,7 +361,7 @@ func (importer *FileImporter) ImportKVFileForRegion( } func (importer *FileImporter) ClearFiles(ctx context.Context, pdClient pd.Client, prefix string) error { - allStores, err := conn.GetAllTiKVStoresWithRetry(ctx, pdClient, conn.SkipTiFlash) + allStores, err := conn.GetAllTiKVStoresWithRetry(ctx, pdClient, util.SkipTiFlash) if err != nil { return errors.Trace(err) } @@ -401,7 +403,7 @@ func (importer *FileImporter) ImportKVFiles( // This RetryState will retry 48 time, for 5 min - 6 min. rs := utils.InitialRetryState(48, 100*time.Millisecond, 8*time.Second) ctl := OverRegionsInRange(startKey, endKey, importer.metaClient, &rs) - err = ctl.Run(ctx, func(ctx context.Context, r *RegionInfo) RPCResult { + err = ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) RPCResult { return importer.ImportKVFileForRegion(ctx, file, rule, startTS, restoreTS, r) }) @@ -437,7 +439,7 @@ func (importer *FileImporter) ImportSSTFiles( defer cancel() // Scan regions covered by the file range regionInfos, errScanRegion := PaginateScanRegion( - tctx, importer.metaClient, startKey, endKey, ScanRegionPaginationLimit) + tctx, importer.metaClient, startKey, endKey, split.ScanRegionPaginationLimit) if errScanRegion != nil { return errors.Trace(errScanRegion) } @@ -506,7 +508,7 @@ func (importer *FileImporter) setDownloadSpeedLimit(ctx context.Context, storeID func (importer *FileImporter) download( ctx context.Context, - regionInfo *RegionInfo, + regionInfo *split.RegionInfo, files []*backuppb.File, rewriteRules *RewriteRules, cipher *backuppb.CipherInfo, @@ -560,7 +562,7 @@ func (importer *FileImporter) download( func (importer *FileImporter) downloadSST( ctx context.Context, - regionInfo *RegionInfo, + regionInfo *split.RegionInfo, file *backuppb.File, rewriteRules *RewriteRules, cipher *backuppb.CipherInfo, @@ -632,7 +634,7 @@ func (importer *FileImporter) downloadSST( func (importer *FileImporter) downloadRawKVSST( ctx context.Context, - regionInfo *RegionInfo, + regionInfo *split.RegionInfo, file *backuppb.File, cipher *backuppb.CipherInfo, apiVersion kvrpcpb.APIVersion, @@ -700,7 +702,7 @@ func (importer *FileImporter) downloadRawKVSST( func (importer *FileImporter) ingest( ctx context.Context, - info *RegionInfo, + info *split.RegionInfo, downloadMetas []*import_sstpb.SSTMeta, ) error { for { @@ -715,9 +717,9 @@ func (importer *FileImporter) ingest( return nil case errPb.NotLeader != nil: // If error is `NotLeader`, update the region info and retry - var newInfo *RegionInfo + var newInfo *split.RegionInfo if newLeader := errPb.GetNotLeader().GetLeader(); newLeader != nil { - newInfo = &RegionInfo{ + newInfo = &split.RegionInfo{ Leader: newLeader, Region: info.Region, } @@ -762,7 +764,7 @@ func (importer *FileImporter) ingest( func (importer *FileImporter) ingestSSTs( ctx context.Context, sstMetas []*import_sstpb.SSTMeta, - regionInfo *RegionInfo, + regionInfo *split.RegionInfo, ) (*import_sstpb.IngestResponse, error) { leader := regionInfo.Leader if leader == nil { @@ -801,7 +803,7 @@ func (importer *FileImporter) downloadAndApplyKVFile( ctx context.Context, file *backuppb.DataFileInfo, rules *RewriteRules, - regionInfo *RegionInfo, + regionInfo *split.RegionInfo, startTS uint64, restoreTS uint64, ) RPCResult { diff --git a/br/pkg/restore/import_retry.go b/br/pkg/restore/import_retry.go index 17c706c9e4444..c8e702e0108b3 100644 --- a/br/pkg/restore/import_retry.go +++ b/br/pkg/restore/import_retry.go @@ -12,6 +12,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" berrors "github.com/pingcap/tidb/br/pkg/errors" "github.com/pingcap/tidb/br/pkg/logutil" + "github.com/pingcap/tidb/br/pkg/restore/split" "github.com/pingcap/tidb/br/pkg/utils" "github.com/tikv/client-go/v2/kv" "go.uber.org/multierr" @@ -19,12 +20,12 @@ import ( "google.golang.org/grpc/status" ) -type RegionFunc func(ctx context.Context, r *RegionInfo) RPCResult +type RegionFunc func(ctx context.Context, r *split.RegionInfo) RPCResult type OverRegionsInRangeController struct { start []byte end []byte - metaClient SplitClient + metaClient split.SplitClient errors error rs *utils.RetryState @@ -33,7 +34,7 @@ type OverRegionsInRangeController struct { // OverRegionsInRange creates a controller that cloud be used to scan regions in a range and // apply a function over these regions. // You can then call the `Run` method for applying some functions. -func OverRegionsInRange(start, end []byte, metaClient SplitClient, retryStatus *utils.RetryState) OverRegionsInRangeController { +func OverRegionsInRange(start, end []byte, metaClient split.SplitClient, retryStatus *utils.RetryState) OverRegionsInRangeController { // IMPORTANT: we record the start/end key with TimeStamp. // but scanRegion will drop the TimeStamp and the end key is exclusive. // if we do not use PrefixNextKey. we might scan fewer regions than we expected. @@ -49,12 +50,12 @@ func OverRegionsInRange(start, end []byte, metaClient SplitClient, retryStatus * } } -func (o *OverRegionsInRangeController) onError(ctx context.Context, result RPCResult, region *RegionInfo) { +func (o *OverRegionsInRangeController) onError(ctx context.Context, result RPCResult, region *split.RegionInfo) { o.errors = multierr.Append(o.errors, errors.Annotatef(&result, "execute over region %v failed", region.Region)) // TODO: Maybe handle some of region errors like `epoch not match`? } -func (o *OverRegionsInRangeController) tryFindLeader(ctx context.Context, region *RegionInfo) (*metapb.Peer, error) { +func (o *OverRegionsInRangeController) tryFindLeader(ctx context.Context, region *split.RegionInfo) (*metapb.Peer, error) { var leader *metapb.Peer failed := false leaderRs := utils.InitialRetryState(4, 5*time.Second, 10*time.Second) @@ -83,7 +84,7 @@ func (o *OverRegionsInRangeController) tryFindLeader(ctx context.Context, region } // handleInRegionError handles the error happens internal in the region. Update the region info, and perform a suitable backoff. -func (o *OverRegionsInRangeController) handleInRegionError(ctx context.Context, result RPCResult, region *RegionInfo) (cont bool) { +func (o *OverRegionsInRangeController) handleInRegionError(ctx context.Context, result RPCResult, region *split.RegionInfo) (cont bool) { if nl := result.StoreError.GetNotLeader(); nl != nil { if nl.Leader != nil { region.Leader = nl.Leader @@ -129,7 +130,7 @@ func (o *OverRegionsInRangeController) runOverRegions(ctx context.Context, f Reg // Scan regions covered by the file range regionInfos, errScanRegion := PaginateScanRegion( - ctx, o.metaClient, o.start, o.end, ScanRegionPaginationLimit) + ctx, o.metaClient, o.start, o.end, split.ScanRegionPaginationLimit) if errScanRegion != nil { return errors.Trace(errScanRegion) } @@ -147,7 +148,7 @@ func (o *OverRegionsInRangeController) runOverRegions(ctx context.Context, f Reg } // runInRegion executes the function in the region, and returns `cont = false` if no need for trying for next region. -func (o *OverRegionsInRangeController) runInRegion(ctx context.Context, f RegionFunc, region *RegionInfo) (cont bool, err error) { +func (o *OverRegionsInRangeController) runInRegion(ctx context.Context, f RegionFunc, region *split.RegionInfo) (cont bool, err error) { if !o.rs.ShouldRetry() { return false, o.errors } diff --git a/br/pkg/restore/import_retry_test.go b/br/pkg/restore/import_retry_test.go index 4349d9ffc2435..7fb09d537b23d 100644 --- a/br/pkg/restore/import_retry_test.go +++ b/br/pkg/restore/import_retry_test.go @@ -16,6 +16,7 @@ import ( "github.com/pingcap/kvproto/pkg/import_sstpb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/tidb/br/pkg/restore" + "github.com/pingcap/tidb/br/pkg/restore/split" "github.com/pingcap/tidb/br/pkg/utils" "github.com/pingcap/tidb/store/pdtypes" "github.com/pingcap/tidb/util/codec" @@ -33,7 +34,7 @@ func assertDecode(t *testing.T, key []byte) []byte { return decoded } -func assertRegions(t *testing.T, regions []*restore.RegionInfo, keys ...string) { +func assertRegions(t *testing.T, regions []*split.RegionInfo, keys ...string) { require.Equal(t, len(regions)+1, len(keys), "%+v\nvs\n%+v", regions, keys) last := keys[0] for i, r := range regions { @@ -54,32 +55,32 @@ func TestScanSuccess(t *testing.T) { // make exclusive to inclusive. ctl := restore.OverRegionsInRange([]byte("aa"), []byte("aay"), cli, &rs) - collectedRegions := []*restore.RegionInfo{} - ctl.Run(ctx, func(ctx context.Context, r *restore.RegionInfo) restore.RPCResult { + collectedRegions := []*split.RegionInfo{} + ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) restore.RPCResult { collectedRegions = append(collectedRegions, r) return restore.RPCResultOK() }) assertRegions(t, collectedRegions, "", "aay", "bba") ctl = restore.OverRegionsInRange([]byte("aaz"), []byte("bb"), cli, &rs) - collectedRegions = []*restore.RegionInfo{} - ctl.Run(ctx, func(ctx context.Context, r *restore.RegionInfo) restore.RPCResult { + collectedRegions = []*split.RegionInfo{} + ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) restore.RPCResult { collectedRegions = append(collectedRegions, r) return restore.RPCResultOK() }) assertRegions(t, collectedRegions, "aay", "bba", "bbh", "cca") ctl = restore.OverRegionsInRange([]byte("aa"), []byte("cc"), cli, &rs) - collectedRegions = []*restore.RegionInfo{} - ctl.Run(ctx, func(ctx context.Context, r *restore.RegionInfo) restore.RPCResult { + collectedRegions = []*split.RegionInfo{} + ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) restore.RPCResult { collectedRegions = append(collectedRegions, r) return restore.RPCResultOK() }) assertRegions(t, collectedRegions, "", "aay", "bba", "bbh", "cca", "") ctl = restore.OverRegionsInRange([]byte("aa"), []byte(""), cli, &rs) - collectedRegions = []*restore.RegionInfo{} - ctl.Run(ctx, func(ctx context.Context, r *restore.RegionInfo) restore.RPCResult { + collectedRegions = []*split.RegionInfo{} + ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) restore.RPCResult { collectedRegions = append(collectedRegions, r) return restore.RPCResultOK() }) @@ -101,10 +102,10 @@ func TestNotLeader(t *testing.T) { }, } // record the regions we didn't touch. - meetRegions := []*restore.RegionInfo{} + meetRegions := []*split.RegionInfo{} // record all regions we meet with id == 2. - idEqualsTo2Regions := []*restore.RegionInfo{} - err := ctl.Run(ctx, func(ctx context.Context, r *restore.RegionInfo) restore.RPCResult { + idEqualsTo2Regions := []*split.RegionInfo{} + err := ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) restore.RPCResult { if r.Region.Id == 2 { idEqualsTo2Regions = append(idEqualsTo2Regions, r) } @@ -126,7 +127,7 @@ func TestNotLeader(t *testing.T) { assertRegions(t, meetRegions, "", "aay", "bba", "bbh", "cca", "") } -func printRegion(name string, infos []*restore.RegionInfo) { +func printRegion(name string, infos []*split.RegionInfo) { fmt.Printf(">>>>> %s <<<<<\n", name) for _, info := range infos { fmt.Printf("[%04d] %s ~ %s\n", info.Region.Id, hex.EncodeToString(info.Region.StartKey), hex.EncodeToString(info.Region.EndKey)) @@ -154,7 +155,7 @@ func TestEpochNotMatch(t *testing.T) { require.NoError(t, err) require.Len(t, regions, 2) left, right := regions[0], regions[1] - info := restore.RegionInfo{ + info := split.RegionInfo{ Region: &metapb.Region{ StartKey: left.Region.StartKey, EndKey: right.Region.EndKey, @@ -177,10 +178,10 @@ func TestEpochNotMatch(t *testing.T) { CurrentRegions: []*metapb.Region{info.Region}, }, }} - firstRunRegions := []*restore.RegionInfo{} - secondRunRegions := []*restore.RegionInfo{} + firstRunRegions := []*split.RegionInfo{} + secondRunRegions := []*split.RegionInfo{} isSecondRun := false - err = ctl.Run(ctx, func(ctx context.Context, r *restore.RegionInfo) restore.RPCResult { + err = ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) restore.RPCResult { if !isSecondRun && r.Region.Id == left.Region.Id { mergeRegion() isSecondRun = true @@ -214,7 +215,7 @@ func TestRegionSplit(t *testing.T) { require.Len(t, regions, 1) target := regions[0] - newRegions := []*restore.RegionInfo{ + newRegions := []*split.RegionInfo{ { Region: &metapb.Region{ Id: 42, @@ -253,10 +254,10 @@ func TestRegionSplit(t *testing.T) { }, }, }} - firstRunRegions := []*restore.RegionInfo{} - secondRunRegions := []*restore.RegionInfo{} + firstRunRegions := []*split.RegionInfo{} + secondRunRegions := []*split.RegionInfo{} isSecondRun := false - err = ctl.Run(ctx, func(ctx context.Context, r *restore.RegionInfo) restore.RPCResult { + err = ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) restore.RPCResult { if !isSecondRun && r.Region.Id == target.Region.Id { splitRegion() isSecondRun = true @@ -298,7 +299,7 @@ func TestRetryBackoff(t *testing.T) { }, }} isSecondRun := false - err = ctl.Run(ctx, func(ctx context.Context, r *restore.RegionInfo) restore.RPCResult { + err = ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) restore.RPCResult { if !isSecondRun && r.Region.Id == left.Region.Id { isSecondRun = true return restore.RPCResultFromPBError(epochNotLeader) @@ -337,8 +338,8 @@ func TestPaginateScanLeader(t *testing.T) { cli.InjectErr = true cli.InjectTimes = int32(envInt("PAGINATE_SCAN_LEADER_FAILURE_COUNT", 2)) - collectedRegions := []*restore.RegionInfo{} - ctl.Run(ctx, func(ctx context.Context, r *restore.RegionInfo) restore.RPCResult { + collectedRegions := []*split.RegionInfo{} + ctl.Run(ctx, func(ctx context.Context, r *split.RegionInfo) restore.RPCResult { collectedRegions = append(collectedRegions, r) return restore.RPCResultOK() }) diff --git a/br/pkg/restore/range.go b/br/pkg/restore/range.go index 81881e78e8182..72a76105dd440 100644 --- a/br/pkg/restore/range.go +++ b/br/pkg/restore/range.go @@ -3,11 +3,8 @@ package restore import ( - "bytes" - "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/import_sstpb" - "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" berrors "github.com/pingcap/tidb/br/pkg/errors" "github.com/pingcap/tidb/br/pkg/logutil" @@ -73,22 +70,6 @@ func SortRanges(ranges []rtree.Range, rewriteRules *RewriteRules) ([]rtree.Range return sortedRanges, nil } -// RegionInfo includes a region and the leader of the region. -type RegionInfo struct { - Region *metapb.Region - Leader *metapb.Peer - PendingPeers []*metapb.Peer - DownPeers []*metapb.Peer -} - -// ContainsInterior returns whether the region contains the given key, and also -// that the key does not fall on the boundary (start key) of the region. -func (region *RegionInfo) ContainsInterior(key []byte) bool { - return bytes.Compare(key, region.Region.GetStartKey()) > 0 && - (len(region.Region.GetEndKey()) == 0 || - bytes.Compare(key, region.Region.GetEndKey()) < 0) -} - // RewriteRules contains rules for rewriting keys of tables. type RewriteRules struct { Data []*import_sstpb.RewriteRule diff --git a/br/pkg/restore/split.go b/br/pkg/restore/split.go index a0aa6810ae195..01943a1db9a2a 100644 --- a/br/pkg/restore/split.go +++ b/br/pkg/restore/split.go @@ -19,6 +19,7 @@ import ( berrors "github.com/pingcap/tidb/br/pkg/errors" "github.com/pingcap/tidb/br/pkg/logutil" "github.com/pingcap/tidb/br/pkg/redact" + "github.com/pingcap/tidb/br/pkg/restore/split" "github.com/pingcap/tidb/br/pkg/rtree" "github.com/pingcap/tidb/br/pkg/utils" "github.com/pingcap/tidb/util/codec" @@ -28,39 +29,17 @@ import ( "google.golang.org/grpc/status" ) -// Constants for split retry machinery. -const ( - SplitRetryTimes = 32 - SplitRetryInterval = 50 * time.Millisecond - SplitMaxRetryInterval = time.Second - - SplitCheckMaxRetryTimes = 64 - SplitCheckInterval = 8 * time.Millisecond - SplitMaxCheckInterval = time.Second - - ScatterWaitMaxRetryTimes = 64 - ScatterWaitInterval = 50 * time.Millisecond - ScatterMaxWaitInterval = time.Second - ScatterWaitUpperInterval = 180 * time.Second - - ScanRegionPaginationLimit = 128 - - RejectStoreCheckRetryTimes = 64 - RejectStoreCheckInterval = 100 * time.Millisecond - RejectStoreMaxCheckInterval = 2 * time.Second -) - var ( ScanRegionAttemptTimes = 60 ) // RegionSplitter is a executor of region split by rules. type RegionSplitter struct { - client SplitClient + client split.SplitClient } // NewRegionSplitter returns a new RegionSplitter. -func NewRegionSplitter(client SplitClient) *RegionSplitter { +func NewRegionSplitter(client split.SplitClient) *RegionSplitter { return &RegionSplitter{ client: client, } @@ -100,11 +79,11 @@ func (rs *RegionSplitter) Split( } minKey := codec.EncodeBytes(nil, sortedRanges[0].StartKey) maxKey := codec.EncodeBytes(nil, sortedRanges[len(sortedRanges)-1].EndKey) - interval := SplitRetryInterval - scatterRegions := make([]*RegionInfo, 0) + interval := split.SplitRetryInterval + scatterRegions := make([]*split.RegionInfo, 0) SplitRegions: - for i := 0; i < SplitRetryTimes; i++ { - regions, errScan := PaginateScanRegion(ctx, rs.client, minKey, maxKey, ScanRegionPaginationLimit) + for i := 0; i < split.SplitRetryTimes; i++ { + regions, errScan := PaginateScanRegion(ctx, rs.client, minKey, maxKey, split.ScanRegionPaginationLimit) if errScan != nil { if berrors.ErrPDBatchScanRegion.Equal(errScan) { log.Warn("inconsistent region info get.", logutil.ShortError(errScan)) @@ -114,13 +93,13 @@ SplitRegions: return errors.Trace(errScan) } splitKeyMap := getSplitKeys(rewriteRules, sortedRanges, regions, isRawKv) - regionMap := make(map[uint64]*RegionInfo) + regionMap := make(map[uint64]*split.RegionInfo) for _, region := range regions { regionMap[region.Region.GetId()] = region } for regionID, keys := range splitKeyMap { log.Info("get split keys for region", zap.Int("len", len(keys)), zap.Uint64("region", regionID)) - var newRegions []*RegionInfo + var newRegions []*split.RegionInfo region := regionMap[regionID] log.Info("split regions", logutil.Region(region.Region), logutil.Keys(keys), rtree.ZapRanges(ranges)) @@ -139,8 +118,8 @@ SplitRegions: return errors.Trace(errSplit) } interval = 2 * interval - if interval > SplitMaxRetryInterval { - interval = SplitMaxRetryInterval + if interval > split.SplitMaxRetryInterval { + interval = split.SplitMaxRetryInterval } time.Sleep(interval) log.Warn("split regions failed, retry", @@ -170,7 +149,7 @@ SplitRegions: scatterCount := 0 for _, region := range scatterRegions { rs.waitForScatterRegion(ctx, region) - if time.Since(startTime) > ScatterWaitUpperInterval { + if time.Since(startTime) > split.ScatterWaitUpperInterval { break } scatterCount++ @@ -233,8 +212,8 @@ func (rs *RegionSplitter) isScatterRegionFinished(ctx context.Context, regionID } func (rs *RegionSplitter) waitForSplit(ctx context.Context, regionID uint64) { - interval := SplitCheckInterval - for i := 0; i < SplitCheckMaxRetryTimes; i++ { + interval := split.SplitCheckInterval + for i := 0; i < split.SplitCheckMaxRetryTimes; i++ { ok, err := rs.hasHealthyRegion(ctx, regionID) if err != nil { log.Warn("wait for split failed", zap.Error(err)) @@ -244,8 +223,8 @@ func (rs *RegionSplitter) waitForSplit(ctx context.Context, regionID uint64) { break } interval = 2 * interval - if interval > SplitMaxCheckInterval { - interval = SplitMaxCheckInterval + if interval > split.SplitMaxCheckInterval { + interval = split.SplitMaxCheckInterval } time.Sleep(interval) } @@ -255,10 +234,10 @@ type retryTimeKey struct{} var retryTimes = new(retryTimeKey) -func (rs *RegionSplitter) waitForScatterRegion(ctx context.Context, regionInfo *RegionInfo) { - interval := ScatterWaitInterval +func (rs *RegionSplitter) waitForScatterRegion(ctx context.Context, regionInfo *split.RegionInfo) { + interval := split.ScatterWaitInterval regionID := regionInfo.Region.GetId() - for i := 0; i < ScatterWaitMaxRetryTimes; i++ { + for i := 0; i < split.ScatterWaitMaxRetryTimes; i++ { ctx1 := context.WithValue(ctx, retryTimes, i) ok, err := rs.isScatterRegionFinished(ctx1, regionID) if err != nil { @@ -270,18 +249,18 @@ func (rs *RegionSplitter) waitForScatterRegion(ctx context.Context, regionInfo * break } interval = 2 * interval - if interval > ScatterMaxWaitInterval { - interval = ScatterMaxWaitInterval + if interval > split.ScatterMaxWaitInterval { + interval = split.ScatterMaxWaitInterval } time.Sleep(interval) } } func (rs *RegionSplitter) splitAndScatterRegions( - ctx context.Context, regionInfo *RegionInfo, keys [][]byte, -) ([]*RegionInfo, error) { + ctx context.Context, regionInfo *split.RegionInfo, keys [][]byte, +) ([]*split.RegionInfo, error) { if len(keys) == 0 { - return []*RegionInfo{regionInfo}, nil + return []*split.RegionInfo{regionInfo}, nil } newRegions, err := rs.client.BatchSplitRegions(ctx, regionInfo, keys) @@ -308,8 +287,8 @@ func (rs *RegionSplitter) splitAndScatterRegions( // ScatterRegionsWithBackoffer scatter the region with some backoffer. // This function is for testing the retry mechanism. // For a real cluster, directly use ScatterRegions would be fine. -func (rs *RegionSplitter) ScatterRegionsWithBackoffer(ctx context.Context, newRegions []*RegionInfo, backoffer utils.Backoffer) { - newRegionSet := make(map[uint64]*RegionInfo, len(newRegions)) +func (rs *RegionSplitter) ScatterRegionsWithBackoffer(ctx context.Context, newRegions []*split.RegionInfo, backoffer utils.Backoffer) { + newRegionSet := make(map[uint64]*split.RegionInfo, len(newRegions)) for _, newRegion := range newRegions { newRegionSet[newRegion.Region.Id] = newRegion } @@ -338,7 +317,7 @@ func (rs *RegionSplitter) ScatterRegionsWithBackoffer(ctx context.Context, newRe // if all region are failed to scatter, the short error might also be verbose... logutil.ShortError(err), logutil.AbbreviatedArray("failed-regions", newRegionSet, func(i interface{}) []string { - m := i.(map[uint64]*RegionInfo) + m := i.(map[uint64]*split.RegionInfo) result := make([]string, 0, len(m)) for id := range m { result = append(result, strconv.Itoa(int(id))) @@ -372,7 +351,7 @@ func isUnsupportedError(err error) bool { } // ScatterRegions scatter the regions. -func (rs *RegionSplitter) ScatterRegions(ctx context.Context, newRegions []*RegionInfo) { +func (rs *RegionSplitter) ScatterRegions(ctx context.Context, newRegions []*split.RegionInfo) { for _, region := range newRegions { // Wait for a while until the regions successfully split. rs.waitForSplit(ctx, region.Region.Id) @@ -400,7 +379,7 @@ func (rs *RegionSplitter) ScatterRegions(ctx context.Context, newRegions []*Regi } } -func CheckRegionConsistency(startKey, endKey []byte, regions []*RegionInfo) error { +func CheckRegionConsistency(startKey, endKey []byte, regions []*split.RegionInfo) error { // current pd can't guarantee the consistency of returned regions if len(regions) == 0 { return errors.Annotatef(berrors.ErrPDBatchScanRegion, "scan region return empty result, startKey: %s, endKey: %s", @@ -431,14 +410,14 @@ func CheckRegionConsistency(startKey, endKey []byte, regions []*RegionInfo) erro // return all regions at once. // It reduces max gRPC message size. func PaginateScanRegion( - ctx context.Context, client SplitClient, startKey, endKey []byte, limit int, -) ([]*RegionInfo, error) { + ctx context.Context, client split.SplitClient, startKey, endKey []byte, limit int, +) ([]*split.RegionInfo, error) { if len(endKey) != 0 && bytes.Compare(startKey, endKey) > 0 { return nil, errors.Annotatef(berrors.ErrRestoreInvalidRange, "startKey > endKey, startKey: %s, endkey: %s", hex.EncodeToString(startKey), hex.EncodeToString(endKey)) } - var regions []*RegionInfo + var regions []*split.RegionInfo var err error // we don't need to return multierr. since there only 3 times retry. // in most case 3 times retry have the same error. so we just return the last error. @@ -446,10 +425,10 @@ func PaginateScanRegion( // because it's not easy to check multierr equals normal error. // see https://github.com/pingcap/tidb/issues/33419. _ = utils.WithRetry(ctx, func() error { - regions = []*RegionInfo{} + regions = []*split.RegionInfo{} scanStartKey := startKey for { - var batch []*RegionInfo + var batch []*split.RegionInfo batch, err = client.ScanRegions(ctx, scanStartKey, endKey, limit) if err != nil { err = errors.Annotatef(berrors.ErrPDBatchScanRegion, "scan regions from start-key:%s, err: %s", @@ -513,7 +492,7 @@ func (b *scanRegionBackoffer) Attempt() int { // getSplitKeys checks if the regions should be split by the end key of // the ranges, groups the split keys by region id. -func getSplitKeys(rewriteRules *RewriteRules, ranges []rtree.Range, regions []*RegionInfo, isRawKv bool) map[uint64][][]byte { +func getSplitKeys(rewriteRules *RewriteRules, ranges []rtree.Range, regions []*split.RegionInfo, isRawKv bool) map[uint64][][]byte { splitKeyMap := make(map[uint64][][]byte) checkKeys := make([][]byte, 0) for _, rg := range ranges { @@ -536,7 +515,7 @@ func getSplitKeys(rewriteRules *RewriteRules, ranges []rtree.Range, regions []*R } // NeedSplit checks whether a key is necessary to split, if true returns the split region. -func NeedSplit(splitKey []byte, regions []*RegionInfo, isRawKv bool) *RegionInfo { +func NeedSplit(splitKey []byte, regions []*split.RegionInfo, isRawKv bool) *split.RegionInfo { // If splitKey is the max key. if len(splitKey) == 0 { return nil diff --git a/br/pkg/restore/split/region.go b/br/pkg/restore/split/region.go new file mode 100644 index 0000000000000..13d161e6476ab --- /dev/null +++ b/br/pkg/restore/split/region.go @@ -0,0 +1,25 @@ +// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. + +package split + +import ( + "bytes" + + "github.com/pingcap/kvproto/pkg/metapb" +) + +// RegionInfo includes a region and the leader of the region. +type RegionInfo struct { + Region *metapb.Region + Leader *metapb.Peer + PendingPeers []*metapb.Peer + DownPeers []*metapb.Peer +} + +// ContainsInterior returns whether the region contains the given key, and also +// that the key does not fall on the boundary (start key) of the region. +func (region *RegionInfo) ContainsInterior(key []byte) bool { + return bytes.Compare(key, region.Region.GetStartKey()) > 0 && + (len(region.Region.GetEndKey()) == 0 || + bytes.Compare(key, region.Region.GetEndKey()) < 0) +} diff --git a/br/pkg/restore/split/split.go b/br/pkg/restore/split/split.go new file mode 100644 index 0000000000000..c80b48cd25eb5 --- /dev/null +++ b/br/pkg/restore/split/split.go @@ -0,0 +1,27 @@ +// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. + +package split + +import "time" + +// Constants for split retry machinery. +const ( + SplitRetryTimes = 32 + SplitRetryInterval = 50 * time.Millisecond + SplitMaxRetryInterval = time.Second + + SplitCheckMaxRetryTimes = 64 + SplitCheckInterval = 8 * time.Millisecond + SplitMaxCheckInterval = time.Second + + ScatterWaitMaxRetryTimes = 64 + ScatterWaitInterval = 50 * time.Millisecond + ScatterMaxWaitInterval = time.Second + ScatterWaitUpperInterval = 180 * time.Second + + ScanRegionPaginationLimit = 128 + + RejectStoreCheckRetryTimes = 64 + RejectStoreCheckInterval = 100 * time.Millisecond + RejectStoreMaxCheckInterval = 2 * time.Second +) diff --git a/br/pkg/restore/split/split_client.go b/br/pkg/restore/split/split_client.go new file mode 100644 index 0000000000000..71cafea8bf2b2 --- /dev/null +++ b/br/pkg/restore/split/split_client.go @@ -0,0 +1,725 @@ +// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. + +package split + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "path" + "strconv" + "strings" + "sync" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/errorpb" + "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/kvproto/pkg/tikvpb" + "github.com/pingcap/log" + "github.com/pingcap/tidb/br/pkg/conn/util" + berrors "github.com/pingcap/tidb/br/pkg/errors" + "github.com/pingcap/tidb/br/pkg/httputil" + "github.com/pingcap/tidb/br/pkg/logutil" + "github.com/pingcap/tidb/br/pkg/redact" + "github.com/pingcap/tidb/br/pkg/utils" + "github.com/pingcap/tidb/store/pdtypes" + pd "github.com/tikv/pd/client" + "go.uber.org/multierr" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" +) + +const ( + splitRegionMaxRetryTime = 4 +) + +// SplitClient is an external client used by RegionSplitter. +type SplitClient interface { + // GetStore gets a store by a store id. + GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error) + // GetRegion gets a region which includes a specified key. + GetRegion(ctx context.Context, key []byte) (*RegionInfo, error) + // GetRegionByID gets a region by a region id. + GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error) + // SplitRegion splits a region from a key, if key is not included in the region, it will return nil. + // note: the key should not be encoded + SplitRegion(ctx context.Context, regionInfo *RegionInfo, key []byte) (*RegionInfo, error) + // BatchSplitRegions splits a region from a batch of keys. + // note: the keys should not be encoded + BatchSplitRegions(ctx context.Context, regionInfo *RegionInfo, keys [][]byte) ([]*RegionInfo, error) + // BatchSplitRegionsWithOrigin splits a region from a batch of keys and return the original region and split new regions + BatchSplitRegionsWithOrigin(ctx context.Context, regionInfo *RegionInfo, keys [][]byte) (*RegionInfo, []*RegionInfo, error) + // ScatterRegion scatters a specified region. + ScatterRegion(ctx context.Context, regionInfo *RegionInfo) error + // ScatterRegions scatters regions in a batch. + ScatterRegions(ctx context.Context, regionInfo []*RegionInfo) error + // GetOperator gets the status of operator of the specified region. + GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) + // ScanRegions gets a list of regions, starts from the region that contains key. + // Limit limits the maximum number of regions returned. + ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*RegionInfo, error) + // GetPlacementRule loads a placement rule from PD. + GetPlacementRule(ctx context.Context, groupID, ruleID string) (pdtypes.Rule, error) + // SetPlacementRule insert or update a placement rule to PD. + SetPlacementRule(ctx context.Context, rule pdtypes.Rule) error + // DeletePlacementRule removes a placement rule from PD. + DeletePlacementRule(ctx context.Context, groupID, ruleID string) error + // SetStoresLabel add or update specified label of stores. If labelValue + // is empty, it clears the label. + SetStoresLabel(ctx context.Context, stores []uint64, labelKey, labelValue string) error +} + +func checkRegionConsistency(startKey, endKey []byte, regions []*RegionInfo) error { + // current pd can't guarantee the consistency of returned regions + if len(regions) == 0 { + return errors.Annotatef(berrors.ErrPDBatchScanRegion, "scan region return empty result, startKey: %s, endkey: %s", + redact.Key(startKey), redact.Key(endKey)) + } + + if bytes.Compare(regions[0].Region.StartKey, startKey) > 0 { + return errors.Annotatef(berrors.ErrPDBatchScanRegion, "first region's startKey > startKey, startKey: %s, regionStartKey: %s", + redact.Key(startKey), redact.Key(regions[0].Region.StartKey)) + } else if len(regions[len(regions)-1].Region.EndKey) != 0 && bytes.Compare(regions[len(regions)-1].Region.EndKey, endKey) < 0 { + return errors.Annotatef(berrors.ErrPDBatchScanRegion, "last region's endKey < startKey, startKey: %s, regionStartKey: %s", + redact.Key(endKey), redact.Key(regions[len(regions)-1].Region.EndKey)) + } + + cur := regions[0] + for _, r := range regions[1:] { + if !bytes.Equal(cur.Region.EndKey, r.Region.StartKey) { + return errors.Annotatef(berrors.ErrPDBatchScanRegion, "region endKey not equal to next region startKey, endKey: %s, startKey: %s", + redact.Key(cur.Region.EndKey), redact.Key(r.Region.StartKey)) + } + cur = r + } + + return nil +} + +// PaginateScanRegion scan regions with a limit pagination and +// return all regions at once. +// It reduces max gRPC message size. +func PaginateScanRegion( + ctx context.Context, client SplitClient, startKey, endKey []byte, limit int, +) ([]*RegionInfo, error) { + if len(endKey) != 0 && bytes.Compare(startKey, endKey) >= 0 { + return nil, errors.Annotatef(berrors.ErrRestoreInvalidRange, "startKey >= endKey, startKey: %s, endkey: %s", + hex.EncodeToString(startKey), hex.EncodeToString(endKey)) + } + + var regions []*RegionInfo + err := utils.WithRetry(ctx, func() error { + regions = []*RegionInfo{} + scanStartKey := startKey + for { + batch, err := client.ScanRegions(ctx, scanStartKey, endKey, limit) + if err != nil { + return errors.Trace(err) + } + regions = append(regions, batch...) + if len(batch) < limit { + // No more region + break + } + scanStartKey = batch[len(batch)-1].Region.GetEndKey() + if len(scanStartKey) == 0 || + (len(endKey) > 0 && bytes.Compare(scanStartKey, endKey) >= 0) { + // All key space have scanned + break + } + } + if err := checkRegionConsistency(startKey, endKey, regions); err != nil { + log.Warn("failed to scan region, retrying", logutil.ShortError(err)) + return err + } + return nil + }, newScanRegionBackoffer()) + + return regions, err +} + +type scanRegionBackoffer struct { + attempt int +} + +func newScanRegionBackoffer() utils.Backoffer { + return &scanRegionBackoffer{ + attempt: 3, + } +} + +// NextBackoff returns a duration to wait before retrying again +func (b *scanRegionBackoffer) NextBackoff(err error) time.Duration { + if berrors.ErrPDBatchScanRegion.Equal(err) { + // 500ms * 3 could be enough for splitting remain regions in the hole. + b.attempt-- + return 500 * time.Millisecond + } + b.attempt = 0 + return 0 +} + +// Attempt returns the remain attempt times +func (b *scanRegionBackoffer) Attempt() int { + return b.attempt +} + +// pdClient is a wrapper of pd client, can be used by RegionSplitter. +type pdClient struct { + mu sync.Mutex + client pd.Client + tlsConf *tls.Config + storeCache map[uint64]*metapb.Store + + // FIXME when config changed during the lifetime of pdClient, + // this may mislead the scatter. + needScatterVal bool + needScatterInit sync.Once + + isRawKv bool +} + +// NewSplitClient returns a client used by RegionSplitter. +func NewSplitClient(client pd.Client, tlsConf *tls.Config, isRawKv bool) SplitClient { + cli := &pdClient{ + client: client, + tlsConf: tlsConf, + storeCache: make(map[uint64]*metapb.Store), + isRawKv: isRawKv, + } + return cli +} + +func (c *pdClient) needScatter(ctx context.Context) bool { + c.needScatterInit.Do(func() { + var err error + c.needScatterVal, err = c.checkNeedScatter(ctx) + if err != nil { + log.Warn("failed to check whether need to scatter, use permissive strategy: always scatter", logutil.ShortError(err)) + c.needScatterVal = true + } + if !c.needScatterVal { + log.Info("skipping scatter because the replica number isn't less than store count.") + } + }) + return c.needScatterVal +} + +// ScatterRegions scatters regions in a batch. +func (c *pdClient) ScatterRegions(ctx context.Context, regionInfo []*RegionInfo) error { + c.mu.Lock() + defer c.mu.Unlock() + regionsID := make([]uint64, 0, len(regionInfo)) + for _, v := range regionInfo { + regionsID = append(regionsID, v.Region.Id) + } + resp, err := c.client.ScatterRegions(ctx, regionsID) + if err != nil { + return err + } + if pbErr := resp.GetHeader().GetError(); pbErr.GetType() != pdpb.ErrorType_OK { + return errors.Annotatef(berrors.ErrPDInvalidResponse, "pd returns error during batch scattering: %s", pbErr) + } + return nil +} + +func (c *pdClient) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error) { + c.mu.Lock() + defer c.mu.Unlock() + store, ok := c.storeCache[storeID] + if ok { + return store, nil + } + store, err := c.client.GetStore(ctx, storeID) + if err != nil { + return nil, errors.Trace(err) + } + c.storeCache[storeID] = store + return store, nil +} + +func (c *pdClient) GetRegion(ctx context.Context, key []byte) (*RegionInfo, error) { + region, err := c.client.GetRegion(ctx, key) + if err != nil { + return nil, errors.Trace(err) + } + if region == nil { + return nil, nil + } + return &RegionInfo{ + Region: region.Meta, + Leader: region.Leader, + }, nil +} + +func (c *pdClient) GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error) { + region, err := c.client.GetRegionByID(ctx, regionID) + if err != nil { + return nil, errors.Trace(err) + } + if region == nil { + return nil, nil + } + return &RegionInfo{ + Region: region.Meta, + Leader: region.Leader, + PendingPeers: region.PendingPeers, + DownPeers: region.DownPeers, + }, nil +} + +func (c *pdClient) SplitRegion(ctx context.Context, regionInfo *RegionInfo, key []byte) (*RegionInfo, error) { + var peer *metapb.Peer + if regionInfo.Leader != nil { + peer = regionInfo.Leader + } else { + if len(regionInfo.Region.Peers) == 0 { + return nil, errors.Annotate(berrors.ErrRestoreNoPeer, "region does not have peer") + } + peer = regionInfo.Region.Peers[0] + } + storeID := peer.GetStoreId() + store, err := c.GetStore(ctx, storeID) + if err != nil { + return nil, errors.Trace(err) + } + conn, err := grpc.Dial(store.GetAddress(), grpc.WithInsecure()) + if err != nil { + return nil, errors.Trace(err) + } + defer conn.Close() + + client := tikvpb.NewTikvClient(conn) + resp, err := client.SplitRegion(ctx, &kvrpcpb.SplitRegionRequest{ + Context: &kvrpcpb.Context{ + RegionId: regionInfo.Region.Id, + RegionEpoch: regionInfo.Region.RegionEpoch, + Peer: peer, + }, + SplitKey: key, + }) + if err != nil { + return nil, errors.Trace(err) + } + if resp.RegionError != nil { + log.Error("fail to split region", + logutil.Region(regionInfo.Region), + logutil.Key("key", key), + zap.Stringer("regionErr", resp.RegionError)) + return nil, errors.Annotatef(berrors.ErrRestoreSplitFailed, "err=%v", resp.RegionError) + } + + // BUG: Left is deprecated, it may be nil even if split is succeed! + // Assume the new region is the left one. + newRegion := resp.GetLeft() + if newRegion == nil { + regions := resp.GetRegions() + for _, r := range regions { + if bytes.Equal(r.GetStartKey(), regionInfo.Region.GetStartKey()) { + newRegion = r + break + } + } + } + if newRegion == nil { + return nil, errors.Annotate(berrors.ErrRestoreSplitFailed, "new region is nil") + } + var leader *metapb.Peer + // Assume the leaders will be at the same store. + if regionInfo.Leader != nil { + for _, p := range newRegion.GetPeers() { + if p.GetStoreId() == regionInfo.Leader.GetStoreId() { + leader = p + break + } + } + } + return &RegionInfo{ + Region: newRegion, + Leader: leader, + }, nil +} + +func splitRegionWithFailpoint( + ctx context.Context, + regionInfo *RegionInfo, + peer *metapb.Peer, + client tikvpb.TikvClient, + keys [][]byte, + isRawKv bool, +) (*kvrpcpb.SplitRegionResponse, error) { + failpoint.Inject("not-leader-error", func(injectNewLeader failpoint.Value) { + log.Debug("failpoint not-leader-error injected.") + resp := &kvrpcpb.SplitRegionResponse{ + RegionError: &errorpb.Error{ + NotLeader: &errorpb.NotLeader{ + RegionId: regionInfo.Region.Id, + }, + }, + } + if injectNewLeader.(bool) { + resp.RegionError.NotLeader.Leader = regionInfo.Leader + } + failpoint.Return(resp, nil) + }) + failpoint.Inject("somewhat-retryable-error", func() { + log.Debug("failpoint somewhat-retryable-error injected.") + failpoint.Return(&kvrpcpb.SplitRegionResponse{ + RegionError: &errorpb.Error{ + ServerIsBusy: &errorpb.ServerIsBusy{}, + }, + }, nil) + }) + return client.SplitRegion(ctx, &kvrpcpb.SplitRegionRequest{ + Context: &kvrpcpb.Context{ + RegionId: regionInfo.Region.Id, + RegionEpoch: regionInfo.Region.RegionEpoch, + Peer: peer, + }, + SplitKeys: keys, + IsRawKv: isRawKv, + }) +} + +func (c *pdClient) sendSplitRegionRequest( + ctx context.Context, regionInfo *RegionInfo, keys [][]byte, +) (*kvrpcpb.SplitRegionResponse, error) { + var splitErrors error + for i := 0; i < splitRegionMaxRetryTime; i++ { + retry, result, err := sendSplitRegionRequest(c, ctx, regionInfo, keys, &splitErrors, i) + if retry { + continue + } + if err != nil { + return nil, multierr.Append(splitErrors, err) + } + if result != nil { + return result, nil + } + return nil, errors.Trace(splitErrors) + } + return nil, errors.Trace(splitErrors) +} + +func sendSplitRegionRequest(c *pdClient, ctx context.Context, regionInfo *RegionInfo, keys [][]byte, splitErrors *error, retry int) (bool, *kvrpcpb.SplitRegionResponse, error) { + var peer *metapb.Peer + // scanRegions may return empty Leader in https://github.com/tikv/pd/blob/v4.0.8/server/grpc_service.go#L524 + // so wee also need check Leader.Id != 0 + if regionInfo.Leader != nil && regionInfo.Leader.Id != 0 { + peer = regionInfo.Leader + } else { + if len(regionInfo.Region.Peers) == 0 { + return false, nil, + errors.Annotatef(berrors.ErrRestoreNoPeer, "region[%d] doesn't have any peer", regionInfo.Region.GetId()) + } + peer = regionInfo.Region.Peers[0] + } + storeID := peer.GetStoreId() + store, err := c.GetStore(ctx, storeID) + if err != nil { + return false, nil, err + } + opt := grpc.WithInsecure() + if c.tlsConf != nil { + opt = grpc.WithTransportCredentials(credentials.NewTLS(c.tlsConf)) + } + conn, err := grpc.Dial(store.GetAddress(), opt) + if err != nil { + return false, nil, err + } + defer conn.Close() + client := tikvpb.NewTikvClient(conn) + resp, err := splitRegionWithFailpoint(ctx, regionInfo, peer, client, keys, c.isRawKv) + if err != nil { + return false, nil, err + } + if resp.RegionError != nil { + log.Warn("fail to split region", + logutil.Region(regionInfo.Region), + zap.Stringer("regionErr", resp.RegionError)) + *splitErrors = multierr.Append(*splitErrors, + errors.Annotatef(berrors.ErrRestoreSplitFailed, "split region failed: err=%v", resp.RegionError)) + if nl := resp.RegionError.NotLeader; nl != nil { + if leader := nl.GetLeader(); leader != nil { + regionInfo.Leader = leader + } else { + newRegionInfo, findLeaderErr := c.GetRegionByID(ctx, nl.RegionId) + if findLeaderErr != nil { + return false, nil, findLeaderErr + } + if !checkRegionEpoch(newRegionInfo, regionInfo) { + return false, nil, berrors.ErrKVEpochNotMatch + } + log.Info("find new leader", zap.Uint64("new leader", newRegionInfo.Leader.Id)) + regionInfo = newRegionInfo + } + log.Info("split region meet not leader error, retrying", + zap.Int("retry times", retry), + zap.Uint64("regionID", regionInfo.Region.Id), + zap.Any("new leader", regionInfo.Leader), + ) + return true, nil, nil + } + // TODO: we don't handle RegionNotMatch and RegionNotFound here, + // because I think we don't have enough information to retry. + // But maybe we can handle them here by some information the error itself provides. + if resp.RegionError.ServerIsBusy != nil || + resp.RegionError.StaleCommand != nil { + log.Warn("a error occurs on split region", + zap.Int("retry times", retry), + zap.Uint64("regionID", regionInfo.Region.Id), + zap.String("error", resp.RegionError.Message), + zap.Any("error verbose", resp.RegionError), + ) + return true, nil, nil + } + return false, nil, nil + } + return false, resp, nil +} + +func (c *pdClient) BatchSplitRegionsWithOrigin( + ctx context.Context, regionInfo *RegionInfo, keys [][]byte, +) (*RegionInfo, []*RegionInfo, error) { + resp, err := c.sendSplitRegionRequest(ctx, regionInfo, keys) + if err != nil { + return nil, nil, errors.Trace(err) + } + + regions := resp.GetRegions() + newRegionInfos := make([]*RegionInfo, 0, len(regions)) + var originRegion *RegionInfo + for _, region := range regions { + var leader *metapb.Peer + + // Assume the leaders will be at the same store. + if regionInfo.Leader != nil { + for _, p := range region.GetPeers() { + if p.GetStoreId() == regionInfo.Leader.GetStoreId() { + leader = p + break + } + } + } + // original region + if region.GetId() == regionInfo.Region.GetId() { + originRegion = &RegionInfo{ + Region: region, + Leader: leader, + } + continue + } + newRegionInfos = append(newRegionInfos, &RegionInfo{ + Region: region, + Leader: leader, + }) + } + return originRegion, newRegionInfos, nil +} + +func (c *pdClient) BatchSplitRegions( + ctx context.Context, regionInfo *RegionInfo, keys [][]byte, +) ([]*RegionInfo, error) { + _, newRegions, err := c.BatchSplitRegionsWithOrigin(ctx, regionInfo, keys) + return newRegions, err +} + +func (c *pdClient) getStoreCount(ctx context.Context) (int, error) { + stores, err := util.GetAllTiKVStores(ctx, c.client, util.SkipTiFlash) + if err != nil { + return 0, err + } + return len(stores), err +} + +func (c *pdClient) getMaxReplica(ctx context.Context) (int, error) { + api := c.getPDAPIAddr() + configAPI := api + "/pd/api/v1/config/replicate" + req, err := http.NewRequestWithContext(ctx, "GET", configAPI, nil) + if err != nil { + return 0, errors.Trace(err) + } + res, err := httputil.NewClient(c.tlsConf).Do(req) + if err != nil { + return 0, errors.Trace(err) + } + defer func() { + if err = res.Body.Close(); err != nil { + log.Error("Response fail to close", zap.Error(err)) + } + }() + var conf pdtypes.ReplicationConfig + if err := json.NewDecoder(res.Body).Decode(&conf); err != nil { + return 0, errors.Trace(err) + } + return int(conf.MaxReplicas), nil +} + +func (c *pdClient) checkNeedScatter(ctx context.Context) (bool, error) { + storeCount, err := c.getStoreCount(ctx) + if err != nil { + return false, err + } + maxReplica, err := c.getMaxReplica(ctx) + if err != nil { + return false, err + } + log.Info("checking whether need to scatter", zap.Int("store", storeCount), zap.Int("max-replica", maxReplica)) + // Skipping scatter may lead to leader unbalanced, + // currently, we skip scatter only when: + // 1. max-replica > store-count (Probably a misconfigured or playground cluster.) + // 2. store-count == 1 (No meaning for scattering.) + // We can still omit scatter when `max-replica == store-count`, if we create a BalanceLeader operator here, + // however, there isn't evidence for transform leader is much faster than scattering empty regions. + return storeCount >= maxReplica && storeCount > 1, nil +} + +func (c *pdClient) ScatterRegion(ctx context.Context, regionInfo *RegionInfo) error { + if !c.needScatter(ctx) { + return nil + } + return c.client.ScatterRegion(ctx, regionInfo.Region.GetId()) +} + +func (c *pdClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) { + return c.client.GetOperator(ctx, regionID) +} + +func (c *pdClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*RegionInfo, error) { + failpoint.Inject("no-leader-error", func(_ failpoint.Value) { + logutil.CL(ctx).Debug("failpoint no-leader-error injected.") + failpoint.Return(nil, status.Error(codes.Unavailable, "not leader")) + }) + + regions, err := c.client.ScanRegions(ctx, key, endKey, limit) + if err != nil { + return nil, errors.Trace(err) + } + regionInfos := make([]*RegionInfo, 0, len(regions)) + for _, region := range regions { + regionInfos = append(regionInfos, &RegionInfo{ + Region: region.Meta, + Leader: region.Leader, + }) + } + return regionInfos, nil +} + +func (c *pdClient) GetPlacementRule(ctx context.Context, groupID, ruleID string) (pdtypes.Rule, error) { + var rule pdtypes.Rule + addr := c.getPDAPIAddr() + if addr == "" { + return rule, errors.Annotate(berrors.ErrRestoreSplitFailed, "failed to add stores labels: no leader") + } + req, err := http.NewRequestWithContext(ctx, "GET", addr+path.Join("/pd/api/v1/config/rule", groupID, ruleID), nil) + if err != nil { + return rule, errors.Trace(err) + } + res, err := httputil.NewClient(c.tlsConf).Do(req) + if err != nil { + return rule, errors.Trace(err) + } + defer func() { + if err = res.Body.Close(); err != nil { + log.Error("Response fail to close", zap.Error(err)) + } + }() + b, err := io.ReadAll(res.Body) + if err != nil { + return rule, errors.Trace(err) + } + err = json.Unmarshal(b, &rule) + if err != nil { + return rule, errors.Trace(err) + } + return rule, nil +} + +func (c *pdClient) SetPlacementRule(ctx context.Context, rule pdtypes.Rule) error { + addr := c.getPDAPIAddr() + if addr == "" { + return errors.Annotate(berrors.ErrPDLeaderNotFound, "failed to add stores labels") + } + m, _ := json.Marshal(rule) + req, err := http.NewRequestWithContext(ctx, "POST", addr+path.Join("/pd/api/v1/config/rule"), bytes.NewReader(m)) + if err != nil { + return errors.Trace(err) + } + res, err := httputil.NewClient(c.tlsConf).Do(req) + if err != nil { + return errors.Trace(err) + } + return errors.Trace(res.Body.Close()) +} + +func (c *pdClient) DeletePlacementRule(ctx context.Context, groupID, ruleID string) error { + addr := c.getPDAPIAddr() + if addr == "" { + return errors.Annotate(berrors.ErrPDLeaderNotFound, "failed to add stores labels") + } + req, err := http.NewRequestWithContext(ctx, "DELETE", addr+path.Join("/pd/api/v1/config/rule", groupID, ruleID), nil) + if err != nil { + return errors.Trace(err) + } + res, err := httputil.NewClient(c.tlsConf).Do(req) + if err != nil { + return errors.Trace(err) + } + return errors.Trace(res.Body.Close()) +} + +func (c *pdClient) SetStoresLabel( + ctx context.Context, stores []uint64, labelKey, labelValue string, +) error { + b := []byte(fmt.Sprintf(`{"%s": "%s"}`, labelKey, labelValue)) + addr := c.getPDAPIAddr() + if addr == "" { + return errors.Annotate(berrors.ErrPDLeaderNotFound, "failed to add stores labels") + } + httpCli := httputil.NewClient(c.tlsConf) + for _, id := range stores { + req, err := http.NewRequestWithContext( + ctx, "POST", + addr+path.Join("/pd/api/v1/store", strconv.FormatUint(id, 10), "label"), + bytes.NewReader(b), + ) + if err != nil { + return errors.Trace(err) + } + res, err := httpCli.Do(req) + if err != nil { + return errors.Trace(err) + } + err = res.Body.Close() + if err != nil { + return errors.Trace(err) + } + } + return nil +} + +func (c *pdClient) getPDAPIAddr() string { + addr := c.client.GetLeaderAddr() + if addr != "" && !strings.HasPrefix(addr, "http") { + addr = "http://" + addr + } + return strings.TrimRight(addr, "/") +} + +func checkRegionEpoch(_new, _old *RegionInfo) bool { + return _new.Region.GetId() == _old.Region.GetId() && + _new.Region.GetRegionEpoch().GetVersion() == _old.Region.GetRegionEpoch().GetVersion() && + _new.Region.GetRegionEpoch().GetConfVer() == _old.Region.GetRegionEpoch().GetConfVer() +} diff --git a/br/pkg/restore/split_client.go b/br/pkg/restore/split_client.go index 2e2a89a45d0ed..0cba105e771a8 100755 --- a/br/pkg/restore/split_client.go +++ b/br/pkg/restore/split_client.go @@ -3,624 +3,14 @@ package restore import ( - "bytes" - "context" - "crypto/tls" - "encoding/json" - "fmt" - "io" - "net/http" - "path" - "strconv" "strings" - "sync" "time" - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/kvproto/pkg/errorpb" - "github.com/pingcap/kvproto/pkg/kvrpcpb" - "github.com/pingcap/kvproto/pkg/metapb" - "github.com/pingcap/kvproto/pkg/pdpb" - "github.com/pingcap/kvproto/pkg/tikvpb" - "github.com/pingcap/log" - "github.com/pingcap/tidb/br/pkg/conn" - berrors "github.com/pingcap/tidb/br/pkg/errors" - "github.com/pingcap/tidb/br/pkg/httputil" - "github.com/pingcap/tidb/br/pkg/logutil" - "github.com/pingcap/tidb/store/pdtypes" - pd "github.com/tikv/pd/client" - "go.uber.org/multierr" - "go.uber.org/zap" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" + "github.com/pingcap/tidb/br/pkg/restore/split" "google.golang.org/grpc/status" ) -const ( - splitRegionMaxRetryTime = 4 -) - -// SplitClient is an external client used by RegionSplitter. -type SplitClient interface { - // GetStore gets a store by a store id. - GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error) - // GetRegion gets a region which includes a specified key. - GetRegion(ctx context.Context, key []byte) (*RegionInfo, error) - // GetRegionByID gets a region by a region id. - GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error) - // SplitRegion splits a region from a key, if key is not included in the region, it will return nil. - // note: the key should not be encoded - SplitRegion(ctx context.Context, regionInfo *RegionInfo, key []byte) (*RegionInfo, error) - // BatchSplitRegions splits a region from a batch of keys. - // note: the keys should not be encoded - BatchSplitRegions(ctx context.Context, regionInfo *RegionInfo, keys [][]byte) ([]*RegionInfo, error) - // BatchSplitRegionsWithOrigin splits a region from a batch of keys and return the original region and split new regions - BatchSplitRegionsWithOrigin(ctx context.Context, regionInfo *RegionInfo, keys [][]byte) (*RegionInfo, []*RegionInfo, error) - // ScatterRegion scatters a specified region. - ScatterRegion(ctx context.Context, regionInfo *RegionInfo) error - // ScatterRegions scatters regions in a batch. - ScatterRegions(ctx context.Context, regionInfo []*RegionInfo) error - // GetOperator gets the status of operator of the specified region. - GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) - // ScanRegions gets a list of regions, starts from the region that contains key. - // Limit limits the maximum number of regions returned. - ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*RegionInfo, error) - // GetPlacementRule loads a placement rule from PD. - GetPlacementRule(ctx context.Context, groupID, ruleID string) (pdtypes.Rule, error) - // SetPlacementRule insert or update a placement rule to PD. - SetPlacementRule(ctx context.Context, rule pdtypes.Rule) error - // DeletePlacementRule removes a placement rule from PD. - DeletePlacementRule(ctx context.Context, groupID, ruleID string) error - // SetStoresLabel add or update specified label of stores. If labelValue - // is empty, it clears the label. - SetStoresLabel(ctx context.Context, stores []uint64, labelKey, labelValue string) error -} - -// pdClient is a wrapper of pd client, can be used by RegionSplitter. -type pdClient struct { - mu sync.Mutex - client pd.Client - tlsConf *tls.Config - storeCache map[uint64]*metapb.Store - - // FIXME when config changed during the lifetime of pdClient, - // this may mislead the scatter. - needScatterVal bool - needScatterInit sync.Once - - isRawKv bool -} - -// NewSplitClient returns a client used by RegionSplitter. -func NewSplitClient(client pd.Client, tlsConf *tls.Config, isRawKv bool) SplitClient { - cli := &pdClient{ - client: client, - tlsConf: tlsConf, - storeCache: make(map[uint64]*metapb.Store), - isRawKv: isRawKv, - } - return cli -} - -func (c *pdClient) needScatter(ctx context.Context) bool { - c.needScatterInit.Do(func() { - var err error - c.needScatterVal, err = c.checkNeedScatter(ctx) - if err != nil { - log.Warn("failed to check whether need to scatter, use permissive strategy: always scatter", logutil.ShortError(err)) - c.needScatterVal = true - } - if !c.needScatterVal { - log.Info("skipping scatter because the replica number isn't less than store count.") - } - }) - return c.needScatterVal -} - -// ScatterRegions scatters regions in a batch. -func (c *pdClient) ScatterRegions(ctx context.Context, regionInfo []*RegionInfo) error { - c.mu.Lock() - defer c.mu.Unlock() - regionsID := make([]uint64, 0, len(regionInfo)) - for _, v := range regionInfo { - regionsID = append(regionsID, v.Region.Id) - } - resp, err := c.client.ScatterRegions(ctx, regionsID) - if err != nil { - return err - } - if pbErr := resp.GetHeader().GetError(); pbErr.GetType() != pdpb.ErrorType_OK { - return errors.Annotatef(berrors.ErrPDInvalidResponse, "pd returns error during batch scattering: %s", pbErr) - } - return nil -} - -func (c *pdClient) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error) { - c.mu.Lock() - defer c.mu.Unlock() - store, ok := c.storeCache[storeID] - if ok { - return store, nil - } - store, err := c.client.GetStore(ctx, storeID) - if err != nil { - return nil, errors.Trace(err) - } - c.storeCache[storeID] = store - return store, nil -} - -func (c *pdClient) GetRegion(ctx context.Context, key []byte) (*RegionInfo, error) { - region, err := c.client.GetRegion(ctx, key) - if err != nil { - return nil, errors.Trace(err) - } - if region == nil { - return nil, nil - } - return &RegionInfo{ - Region: region.Meta, - Leader: region.Leader, - }, nil -} - -func (c *pdClient) GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error) { - region, err := c.client.GetRegionByID(ctx, regionID) - if err != nil { - return nil, errors.Trace(err) - } - if region == nil { - return nil, nil - } - return &RegionInfo{ - Region: region.Meta, - Leader: region.Leader, - PendingPeers: region.PendingPeers, - DownPeers: region.DownPeers, - }, nil -} - -func (c *pdClient) SplitRegion(ctx context.Context, regionInfo *RegionInfo, key []byte) (*RegionInfo, error) { - var peer *metapb.Peer - if regionInfo.Leader != nil { - peer = regionInfo.Leader - } else { - if len(regionInfo.Region.Peers) == 0 { - return nil, errors.Annotate(berrors.ErrRestoreNoPeer, "region does not have peer") - } - peer = regionInfo.Region.Peers[0] - } - storeID := peer.GetStoreId() - store, err := c.GetStore(ctx, storeID) - if err != nil { - return nil, errors.Trace(err) - } - conn, err := grpc.Dial(store.GetAddress(), grpc.WithInsecure()) - if err != nil { - return nil, errors.Trace(err) - } - defer conn.Close() - - client := tikvpb.NewTikvClient(conn) - resp, err := client.SplitRegion(ctx, &kvrpcpb.SplitRegionRequest{ - Context: &kvrpcpb.Context{ - RegionId: regionInfo.Region.Id, - RegionEpoch: regionInfo.Region.RegionEpoch, - Peer: peer, - }, - SplitKey: key, - }) - if err != nil { - return nil, errors.Trace(err) - } - if resp.RegionError != nil { - log.Error("fail to split region", - logutil.Region(regionInfo.Region), - logutil.Key("key", key), - zap.Stringer("regionErr", resp.RegionError)) - return nil, errors.Annotatef(berrors.ErrRestoreSplitFailed, "err=%v", resp.RegionError) - } - - // BUG: Left is deprecated, it may be nil even if split is succeed! - // Assume the new region is the left one. - newRegion := resp.GetLeft() - if newRegion == nil { - regions := resp.GetRegions() - for _, r := range regions { - if bytes.Equal(r.GetStartKey(), regionInfo.Region.GetStartKey()) { - newRegion = r - break - } - } - } - if newRegion == nil { - return nil, errors.Annotate(berrors.ErrRestoreSplitFailed, "new region is nil") - } - var leader *metapb.Peer - // Assume the leaders will be at the same store. - if regionInfo.Leader != nil { - for _, p := range newRegion.GetPeers() { - if p.GetStoreId() == regionInfo.Leader.GetStoreId() { - leader = p - break - } - } - } - return &RegionInfo{ - Region: newRegion, - Leader: leader, - }, nil -} - -func splitRegionWithFailpoint( - ctx context.Context, - regionInfo *RegionInfo, - peer *metapb.Peer, - client tikvpb.TikvClient, - keys [][]byte, - isRawKv bool, -) (*kvrpcpb.SplitRegionResponse, error) { - failpoint.Inject("not-leader-error", func(injectNewLeader failpoint.Value) { - log.Debug("failpoint not-leader-error injected.") - resp := &kvrpcpb.SplitRegionResponse{ - RegionError: &errorpb.Error{ - NotLeader: &errorpb.NotLeader{ - RegionId: regionInfo.Region.Id, - }, - }, - } - if injectNewLeader.(bool) { - resp.RegionError.NotLeader.Leader = regionInfo.Leader - } - failpoint.Return(resp, nil) - }) - failpoint.Inject("somewhat-retryable-error", func() { - log.Debug("failpoint somewhat-retryable-error injected.") - failpoint.Return(&kvrpcpb.SplitRegionResponse{ - RegionError: &errorpb.Error{ - ServerIsBusy: &errorpb.ServerIsBusy{}, - }, - }, nil) - }) - return client.SplitRegion(ctx, &kvrpcpb.SplitRegionRequest{ - Context: &kvrpcpb.Context{ - RegionId: regionInfo.Region.Id, - RegionEpoch: regionInfo.Region.RegionEpoch, - Peer: peer, - }, - SplitKeys: keys, - IsRawKv: isRawKv, - }) -} - -func (c *pdClient) sendSplitRegionRequest( - ctx context.Context, regionInfo *RegionInfo, keys [][]byte, -) (*kvrpcpb.SplitRegionResponse, error) { - var splitErrors error - for i := 0; i < splitRegionMaxRetryTime; i++ { - retry, result, err := sendSplitRegionRequest(c, ctx, regionInfo, keys, &splitErrors, i) - if retry { - continue - } - if err != nil { - return nil, multierr.Append(splitErrors, err) - } - if result != nil { - return result, nil - } - return nil, errors.Trace(splitErrors) - } - return nil, errors.Trace(splitErrors) -} - -func sendSplitRegionRequest(c *pdClient, ctx context.Context, regionInfo *RegionInfo, keys [][]byte, splitErrors *error, retry int) (bool, *kvrpcpb.SplitRegionResponse, error) { - var peer *metapb.Peer - // scanRegions may return empty Leader in https://github.com/tikv/pd/blob/v4.0.8/server/grpc_service.go#L524 - // so wee also need check Leader.Id != 0 - if regionInfo.Leader != nil && regionInfo.Leader.Id != 0 { - peer = regionInfo.Leader - } else { - if len(regionInfo.Region.Peers) == 0 { - return false, nil, - errors.Annotatef(berrors.ErrRestoreNoPeer, "region[%d] doesn't have any peer", regionInfo.Region.GetId()) - } - peer = regionInfo.Region.Peers[0] - } - storeID := peer.GetStoreId() - store, err := c.GetStore(ctx, storeID) - if err != nil { - return false, nil, err - } - opt := grpc.WithInsecure() - if c.tlsConf != nil { - opt = grpc.WithTransportCredentials(credentials.NewTLS(c.tlsConf)) - } - conn, err := grpc.Dial(store.GetAddress(), opt) - if err != nil { - return false, nil, err - } - defer conn.Close() - client := tikvpb.NewTikvClient(conn) - resp, err := splitRegionWithFailpoint(ctx, regionInfo, peer, client, keys, c.isRawKv) - if err != nil { - return false, nil, err - } - if resp.RegionError != nil { - log.Warn("fail to split region", - logutil.Region(regionInfo.Region), - zap.Stringer("regionErr", resp.RegionError)) - *splitErrors = multierr.Append(*splitErrors, - errors.Annotatef(berrors.ErrRestoreSplitFailed, "split region failed: err=%v", resp.RegionError)) - if nl := resp.RegionError.NotLeader; nl != nil { - if leader := nl.GetLeader(); leader != nil { - regionInfo.Leader = leader - } else { - newRegionInfo, findLeaderErr := c.GetRegionByID(ctx, nl.RegionId) - if findLeaderErr != nil { - return false, nil, findLeaderErr - } - if !checkRegionEpoch(newRegionInfo, regionInfo) { - return false, nil, berrors.ErrKVEpochNotMatch - } - log.Info("find new leader", zap.Uint64("new leader", newRegionInfo.Leader.Id)) - regionInfo = newRegionInfo - } - log.Info("split region meet not leader error, retrying", - zap.Int("retry times", retry), - zap.Uint64("regionID", regionInfo.Region.Id), - zap.Any("new leader", regionInfo.Leader), - ) - return true, nil, nil - } - // TODO: we don't handle RegionNotMatch and RegionNotFound here, - // because I think we don't have enough information to retry. - // But maybe we can handle them here by some information the error itself provides. - if resp.RegionError.ServerIsBusy != nil || - resp.RegionError.StaleCommand != nil { - log.Warn("a error occurs on split region", - zap.Int("retry times", retry), - zap.Uint64("regionID", regionInfo.Region.Id), - zap.String("error", resp.RegionError.Message), - zap.Any("error verbose", resp.RegionError), - ) - return true, nil, nil - } - return false, nil, nil - } - return false, resp, nil -} - -func (c *pdClient) BatchSplitRegionsWithOrigin( - ctx context.Context, regionInfo *RegionInfo, keys [][]byte, -) (*RegionInfo, []*RegionInfo, error) { - resp, err := c.sendSplitRegionRequest(ctx, regionInfo, keys) - if err != nil { - return nil, nil, errors.Trace(err) - } - - regions := resp.GetRegions() - newRegionInfos := make([]*RegionInfo, 0, len(regions)) - var originRegion *RegionInfo - for _, region := range regions { - var leader *metapb.Peer - - // Assume the leaders will be at the same store. - if regionInfo.Leader != nil { - for _, p := range region.GetPeers() { - if p.GetStoreId() == regionInfo.Leader.GetStoreId() { - leader = p - break - } - } - } - // original region - if region.GetId() == regionInfo.Region.GetId() { - originRegion = &RegionInfo{ - Region: region, - Leader: leader, - } - continue - } - newRegionInfos = append(newRegionInfos, &RegionInfo{ - Region: region, - Leader: leader, - }) - } - return originRegion, newRegionInfos, nil -} - -func (c *pdClient) BatchSplitRegions( - ctx context.Context, regionInfo *RegionInfo, keys [][]byte, -) ([]*RegionInfo, error) { - _, newRegions, err := c.BatchSplitRegionsWithOrigin(ctx, regionInfo, keys) - return newRegions, err -} - -func (c *pdClient) getStoreCount(ctx context.Context) (int, error) { - stores, err := conn.GetAllTiKVStores(ctx, c.client, conn.SkipTiFlash) - if err != nil { - return 0, err - } - return len(stores), err -} - -func (c *pdClient) getMaxReplica(ctx context.Context) (int, error) { - api := c.getPDAPIAddr() - configAPI := api + "/pd/api/v1/config/replicate" - req, err := http.NewRequestWithContext(ctx, "GET", configAPI, nil) - if err != nil { - return 0, errors.Trace(err) - } - res, err := httputil.NewClient(c.tlsConf).Do(req) - if err != nil { - return 0, errors.Trace(err) - } - defer func() { - if err = res.Body.Close(); err != nil { - log.Error("Response fail to close", zap.Error(err)) - } - }() - var conf pdtypes.ReplicationConfig - if err := json.NewDecoder(res.Body).Decode(&conf); err != nil { - return 0, errors.Trace(err) - } - return int(conf.MaxReplicas), nil -} - -func (c *pdClient) checkNeedScatter(ctx context.Context) (bool, error) { - storeCount, err := c.getStoreCount(ctx) - if err != nil { - return false, err - } - maxReplica, err := c.getMaxReplica(ctx) - if err != nil { - return false, err - } - log.Info("checking whether need to scatter", zap.Int("store", storeCount), zap.Int("max-replica", maxReplica)) - // Skipping scatter may lead to leader unbalanced, - // currently, we skip scatter only when: - // 1. max-replica > store-count (Probably a misconfigured or playground cluster.) - // 2. store-count == 1 (No meaning for scattering.) - // We can still omit scatter when `max-replica == store-count`, if we create a BalanceLeader operator here, - // however, there isn't evidence for transform leader is much faster than scattering empty regions. - return storeCount >= maxReplica && storeCount > 1, nil -} - -func (c *pdClient) ScatterRegion(ctx context.Context, regionInfo *RegionInfo) error { - if !c.needScatter(ctx) { - return nil - } - return c.client.ScatterRegion(ctx, regionInfo.Region.GetId()) -} - -func (c *pdClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error) { - return c.client.GetOperator(ctx, regionID) -} - -func (c *pdClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*RegionInfo, error) { - failpoint.Inject("no-leader-error", func(_ failpoint.Value) { - logutil.CL(ctx).Debug("failpoint no-leader-error injected.") - failpoint.Return(nil, status.Error(codes.Unavailable, "not leader")) - }) - - regions, err := c.client.ScanRegions(ctx, key, endKey, limit) - if err != nil { - return nil, errors.Trace(err) - } - regionInfos := make([]*RegionInfo, 0, len(regions)) - for _, region := range regions { - regionInfos = append(regionInfos, &RegionInfo{ - Region: region.Meta, - Leader: region.Leader, - }) - } - return regionInfos, nil -} - -func (c *pdClient) GetPlacementRule(ctx context.Context, groupID, ruleID string) (pdtypes.Rule, error) { - var rule pdtypes.Rule - addr := c.getPDAPIAddr() - if addr == "" { - return rule, errors.Annotate(berrors.ErrRestoreSplitFailed, "failed to add stores labels: no leader") - } - req, err := http.NewRequestWithContext(ctx, "GET", addr+path.Join("/pd/api/v1/config/rule", groupID, ruleID), nil) - if err != nil { - return rule, errors.Trace(err) - } - res, err := httputil.NewClient(c.tlsConf).Do(req) - if err != nil { - return rule, errors.Trace(err) - } - defer func() { - if err = res.Body.Close(); err != nil { - log.Error("Response fail to close", zap.Error(err)) - } - }() - b, err := io.ReadAll(res.Body) - if err != nil { - return rule, errors.Trace(err) - } - err = json.Unmarshal(b, &rule) - if err != nil { - return rule, errors.Trace(err) - } - return rule, nil -} - -func (c *pdClient) SetPlacementRule(ctx context.Context, rule pdtypes.Rule) error { - addr := c.getPDAPIAddr() - if addr == "" { - return errors.Annotate(berrors.ErrPDLeaderNotFound, "failed to add stores labels") - } - m, _ := json.Marshal(rule) - req, err := http.NewRequestWithContext(ctx, "POST", addr+path.Join("/pd/api/v1/config/rule"), bytes.NewReader(m)) - if err != nil { - return errors.Trace(err) - } - res, err := httputil.NewClient(c.tlsConf).Do(req) - if err != nil { - return errors.Trace(err) - } - return errors.Trace(res.Body.Close()) -} - -func (c *pdClient) DeletePlacementRule(ctx context.Context, groupID, ruleID string) error { - addr := c.getPDAPIAddr() - if addr == "" { - return errors.Annotate(berrors.ErrPDLeaderNotFound, "failed to add stores labels") - } - req, err := http.NewRequestWithContext(ctx, "DELETE", addr+path.Join("/pd/api/v1/config/rule", groupID, ruleID), nil) - if err != nil { - return errors.Trace(err) - } - res, err := httputil.NewClient(c.tlsConf).Do(req) - if err != nil { - return errors.Trace(err) - } - return errors.Trace(res.Body.Close()) -} - -func (c *pdClient) SetStoresLabel( - ctx context.Context, stores []uint64, labelKey, labelValue string, -) error { - b := []byte(fmt.Sprintf(`{"%s": "%s"}`, labelKey, labelValue)) - addr := c.getPDAPIAddr() - if addr == "" { - return errors.Annotate(berrors.ErrPDLeaderNotFound, "failed to add stores labels") - } - httpCli := httputil.NewClient(c.tlsConf) - for _, id := range stores { - req, err := http.NewRequestWithContext( - ctx, "POST", - addr+path.Join("/pd/api/v1/store", strconv.FormatUint(id, 10), "label"), - bytes.NewReader(b), - ) - if err != nil { - return errors.Trace(err) - } - res, err := httpCli.Do(req) - if err != nil { - return errors.Trace(err) - } - err = res.Body.Close() - if err != nil { - return errors.Trace(err) - } - } - return nil -} - -func (c *pdClient) getPDAPIAddr() string { - addr := c.client.GetLeaderAddr() - if addr != "" && !strings.HasPrefix(addr, "http") { - addr = "http://" + addr - } - return strings.TrimRight(addr, "/") -} - -func checkRegionEpoch(_new, _old *RegionInfo) bool { +func checkRegionEpoch(_new, _old *split.RegionInfo) bool { return _new.Region.GetId() == _old.Region.GetId() && _new.Region.GetRegionEpoch().GetVersion() == _old.Region.GetRegionEpoch().GetVersion() && _new.Region.GetRegionEpoch().GetConfVer() == _old.Region.GetRegionEpoch().GetConfVer() diff --git a/br/pkg/restore/split_test.go b/br/pkg/restore/split_test.go index 1e37a7975b205..f8681ffc9378c 100644 --- a/br/pkg/restore/split_test.go +++ b/br/pkg/restore/split_test.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/tidb/br/pkg/glue" "github.com/pingcap/tidb/br/pkg/logutil" "github.com/pingcap/tidb/br/pkg/restore" + "github.com/pingcap/tidb/br/pkg/restore/split" "github.com/pingcap/tidb/br/pkg/rtree" "github.com/pingcap/tidb/br/pkg/utils" "github.com/pingcap/tidb/store/pdtypes" @@ -32,10 +33,10 @@ import ( type TestClient struct { mu sync.RWMutex stores map[uint64]*metapb.Store - regions map[uint64]*restore.RegionInfo + regions map[uint64]*split.RegionInfo regionsInfo *pdtypes.RegionTree // For now it's only used in ScanRegions nextRegionID uint64 - injectInScatter func(*restore.RegionInfo) error + injectInScatter func(*split.RegionInfo) error supportBatchScatter bool scattered map[uint64]bool @@ -45,7 +46,7 @@ type TestClient struct { func NewTestClient( stores map[uint64]*metapb.Store, - regions map[uint64]*restore.RegionInfo, + regions map[uint64]*split.RegionInfo, nextRegionID uint64, ) *TestClient { regionsInfo := &pdtypes.RegionTree{} @@ -58,7 +59,7 @@ func NewTestClient( regionsInfo: regionsInfo, nextRegionID: nextRegionID, scattered: map[uint64]bool{}, - injectInScatter: func(*restore.RegionInfo) error { return nil }, + injectInScatter: func(*split.RegionInfo) error { return nil }, } } @@ -67,11 +68,11 @@ func (c *TestClient) InstallBatchScatterSupport() { } // ScatterRegions scatters regions in a batch. -func (c *TestClient) ScatterRegions(ctx context.Context, regionInfo []*restore.RegionInfo) error { +func (c *TestClient) ScatterRegions(ctx context.Context, regionInfo []*split.RegionInfo) error { if !c.supportBatchScatter { return status.Error(codes.Unimplemented, "Ah, yep") } - regions := map[uint64]*restore.RegionInfo{} + regions := map[uint64]*split.RegionInfo{} for _, region := range regionInfo { regions[region.Region.Id] = region } @@ -92,7 +93,7 @@ func (c *TestClient) ScatterRegions(ctx context.Context, regionInfo []*restore.R return nil } -func (c *TestClient) GetAllRegions() map[uint64]*restore.RegionInfo { +func (c *TestClient) GetAllRegions() map[uint64]*split.RegionInfo { c.mu.RLock() defer c.mu.RUnlock() return c.regions @@ -108,7 +109,7 @@ func (c *TestClient) GetStore(ctx context.Context, storeID uint64) (*metapb.Stor return store, nil } -func (c *TestClient) GetRegion(ctx context.Context, key []byte) (*restore.RegionInfo, error) { +func (c *TestClient) GetRegion(ctx context.Context, key []byte) (*split.RegionInfo, error) { c.mu.RLock() defer c.mu.RUnlock() for _, region := range c.regions { @@ -120,7 +121,7 @@ func (c *TestClient) GetRegion(ctx context.Context, key []byte) (*restore.Region return nil, errors.Errorf("region not found: key=%s", string(key)) } -func (c *TestClient) GetRegionByID(ctx context.Context, regionID uint64) (*restore.RegionInfo, error) { +func (c *TestClient) GetRegionByID(ctx context.Context, regionID uint64) (*split.RegionInfo, error) { c.mu.RLock() defer c.mu.RUnlock() region, ok := c.regions[regionID] @@ -132,12 +133,12 @@ func (c *TestClient) GetRegionByID(ctx context.Context, regionID uint64) (*resto func (c *TestClient) SplitRegion( ctx context.Context, - regionInfo *restore.RegionInfo, + regionInfo *split.RegionInfo, key []byte, -) (*restore.RegionInfo, error) { +) (*split.RegionInfo, error) { c.mu.Lock() defer c.mu.Unlock() - var target *restore.RegionInfo + var target *split.RegionInfo splitKey := codec.EncodeBytes([]byte{}, key) for _, region := range c.regions { if bytes.Compare(splitKey, region.Region.StartKey) >= 0 && @@ -148,7 +149,7 @@ func (c *TestClient) SplitRegion( if target == nil { return nil, errors.Errorf("region not found: key=%s", string(key)) } - newRegion := &restore.RegionInfo{ + newRegion := &split.RegionInfo{ Region: &metapb.Region{ Peers: target.Region.Peers, Id: c.nextRegionID, @@ -164,14 +165,14 @@ func (c *TestClient) SplitRegion( } func (c *TestClient) BatchSplitRegionsWithOrigin( - ctx context.Context, regionInfo *restore.RegionInfo, keys [][]byte, -) (*restore.RegionInfo, []*restore.RegionInfo, error) { + ctx context.Context, regionInfo *split.RegionInfo, keys [][]byte, +) (*split.RegionInfo, []*split.RegionInfo, error) { c.mu.Lock() defer c.mu.Unlock() - newRegions := make([]*restore.RegionInfo, 0) - var region *restore.RegionInfo + newRegions := make([]*split.RegionInfo, 0) + var region *split.RegionInfo for _, key := range keys { - var target *restore.RegionInfo + var target *split.RegionInfo splitKey := codec.EncodeBytes([]byte{}, key) for _, region := range c.regions { if region.ContainsInterior(splitKey) { @@ -181,7 +182,7 @@ func (c *TestClient) BatchSplitRegionsWithOrigin( if target == nil { continue } - newRegion := &restore.RegionInfo{ + newRegion := &split.RegionInfo{ Region: &metapb.Region{ Peers: target.Region.Peers, Id: c.nextRegionID, @@ -200,13 +201,13 @@ func (c *TestClient) BatchSplitRegionsWithOrigin( } func (c *TestClient) BatchSplitRegions( - ctx context.Context, regionInfo *restore.RegionInfo, keys [][]byte, -) ([]*restore.RegionInfo, error) { + ctx context.Context, regionInfo *split.RegionInfo, keys [][]byte, +) ([]*split.RegionInfo, error) { _, newRegions, err := c.BatchSplitRegionsWithOrigin(ctx, regionInfo, keys) return newRegions, err } -func (c *TestClient) ScatterRegion(ctx context.Context, regionInfo *restore.RegionInfo) error { +func (c *TestClient) ScatterRegion(ctx context.Context, regionInfo *split.RegionInfo) error { return c.injectInScatter(regionInfo) } @@ -216,16 +217,16 @@ func (c *TestClient) GetOperator(ctx context.Context, regionID uint64) (*pdpb.Ge }, nil } -func (c *TestClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*restore.RegionInfo, error) { +func (c *TestClient) ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*split.RegionInfo, error) { if c.InjectErr && c.InjectTimes > 0 { c.InjectTimes -= 1 return nil, status.Error(codes.Unavailable, "not leader") } infos := c.regionsInfo.ScanRange(key, endKey, limit) - regions := make([]*restore.RegionInfo, 0, len(infos)) + regions := make([]*split.RegionInfo, 0, len(infos)) for _, info := range infos { - regions = append(regions, &restore.RegionInfo{ + regions = append(regions, &split.RegionInfo{ Region: info.Meta, Leader: info.Leader, }) @@ -296,12 +297,12 @@ func TestScatterFinishInTime(t *testing.T) { t.Fail() } - regionInfos := make([]*restore.RegionInfo, 0, len(regions)) + regionInfos := make([]*split.RegionInfo, 0, len(regions)) for _, info := range regions { regionInfos = append(regionInfos, info) } failed := map[uint64]int{} - client.injectInScatter = func(r *restore.RegionInfo) error { + client.injectInScatter = func(r *split.RegionInfo) error { failed[r.Region.Id]++ if failed[r.Region.Id] > 7 { return nil @@ -351,13 +352,13 @@ func runTestSplitAndScatterWith(t *testing.T, client *TestClient) { t.Log("get wrong result") t.Fail() } - regionInfos := make([]*restore.RegionInfo, 0, len(regions)) + regionInfos := make([]*split.RegionInfo, 0, len(regions)) for _, info := range regions { regionInfos = append(regionInfos, info) } scattered := map[uint64]bool{} const alwaysFailedRegionID = 1 - client.injectInScatter = func(regionInfo *restore.RegionInfo) error { + client.injectInScatter = func(regionInfo *split.RegionInfo) error { if _, ok := scattered[regionInfo.Region.Id]; !ok || regionInfo.Region.Id == alwaysFailedRegionID { scattered[regionInfo.Region.Id] = false return status.Errorf(codes.Unknown, "region %d is not fully replicated", regionInfo.Region.Id) @@ -383,7 +384,7 @@ func initTestClient() *TestClient { StoreId: 1, } keys := [6]string{"", "aay", "bba", "bbh", "cca", ""} - regions := make(map[uint64]*restore.RegionInfo) + regions := make(map[uint64]*split.RegionInfo) for i := uint64(1); i < 6; i++ { startKey := []byte(keys[i-1]) if len(startKey) != 0 { @@ -393,7 +394,7 @@ func initTestClient() *TestClient { if len(endKey) != 0 { endKey = codec.EncodeBytes([]byte{}, endKey) } - regions[i] = &restore.RegionInfo{ + regions[i] = &split.RegionInfo{ Leader: &metapb.Peer{ Id: i, }, @@ -452,7 +453,7 @@ func initRewriteRules() *restore.RewriteRules { // expected regions after split: // [, aay), [aay, bba), [bba, bbf), [bbf, bbh), [bbh, bbj), // [bbj, cca), [cca, xxe), [xxe, xxz), [xxz, ) -func validateRegions(regions map[uint64]*restore.RegionInfo) bool { +func validateRegions(regions map[uint64]*split.RegionInfo) bool { keys := [...]string{"", "aay", "bba", "bbf", "bbh", "bbj", "cca", "xxe", "xxz", ""} if len(regions) != len(keys)-1 { return false @@ -487,7 +488,7 @@ func TestNeedSplit(t *testing.T) { return codec.EncodeBytes([]byte{}, in) } - regions := []*restore.RegionInfo{ + regions := []*split.RegionInfo{ { Region: &metapb.Region{ StartKey: encode([]byte("b")), @@ -515,19 +516,19 @@ func TestRegionConsistency(t *testing.T) { startKey []byte endKey []byte err string - regions []*restore.RegionInfo + regions []*split.RegionInfo }{ { codec.EncodeBytes([]byte{}, []byte("a")), codec.EncodeBytes([]byte{}, []byte("a")), "scan region return empty result, startKey: (.*?), endKey: (.*?)", - []*restore.RegionInfo{}, + []*split.RegionInfo{}, }, { codec.EncodeBytes([]byte{}, []byte("a")), codec.EncodeBytes([]byte{}, []byte("a")), "first region's startKey > startKey, startKey: (.*?), regionStartKey: (.*?)", - []*restore.RegionInfo{ + []*split.RegionInfo{ { Region: &metapb.Region{ StartKey: codec.EncodeBytes([]byte{}, []byte("b")), @@ -540,7 +541,7 @@ func TestRegionConsistency(t *testing.T) { codec.EncodeBytes([]byte{}, []byte("b")), codec.EncodeBytes([]byte{}, []byte("e")), "last region's endKey < endKey, endKey: (.*?), regionEndKey: (.*?)", - []*restore.RegionInfo{ + []*split.RegionInfo{ { Region: &metapb.Region{ StartKey: codec.EncodeBytes([]byte{}, []byte("b")), @@ -553,7 +554,7 @@ func TestRegionConsistency(t *testing.T) { codec.EncodeBytes([]byte{}, []byte("c")), codec.EncodeBytes([]byte{}, []byte("e")), "region endKey not equal to next region startKey(.*?)", - []*restore.RegionInfo{ + []*split.RegionInfo{ { Region: &metapb.Region{ StartKey: codec.EncodeBytes([]byte{}, []byte("b")), diff --git a/br/pkg/restore/util.go b/br/pkg/restore/util.go index 584bbe81d566d..7dcd594bc347b 100644 --- a/br/pkg/restore/util.go +++ b/br/pkg/restore/util.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/tidb/br/pkg/glue" "github.com/pingcap/tidb/br/pkg/logutil" "github.com/pingcap/tidb/br/pkg/redact" + "github.com/pingcap/tidb/br/pkg/restore/split" "github.com/pingcap/tidb/br/pkg/rtree" "github.com/pingcap/tidb/br/pkg/utils" "github.com/pingcap/tidb/parser/model" @@ -481,7 +482,7 @@ func SplitRanges( updateCh glue.Progress, isRawKv bool, ) error { - splitter := NewRegionSplitter(NewSplitClient(client.GetPDClient(), client.GetTLSConfig(), isRawKv)) + splitter := NewRegionSplitter(split.NewSplitClient(client.GetPDClient(), client.GetTLSConfig(), isRawKv)) return splitter.Split(ctx, ranges, rewriteRules, isRawKv, func(keys [][]byte) { for range keys { diff --git a/br/pkg/restore/util_test.go b/br/pkg/restore/util_test.go index fbb7f48894d1d..8b23a72127520 100644 --- a/br/pkg/restore/util_test.go +++ b/br/pkg/restore/util_test.go @@ -13,6 +13,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" berrors "github.com/pingcap/tidb/br/pkg/errors" "github.com/pingcap/tidb/br/pkg/restore" + "github.com/pingcap/tidb/br/pkg/restore/split" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/codec" "github.com/stretchr/testify/require" @@ -180,12 +181,12 @@ func TestPaginateScanRegion(t *testing.T) { Id: 1, } - makeRegions := func(num uint64) (map[uint64]*restore.RegionInfo, []*restore.RegionInfo) { - regionsMap := make(map[uint64]*restore.RegionInfo, num) - regions := make([]*restore.RegionInfo, 0, num) + makeRegions := func(num uint64) (map[uint64]*split.RegionInfo, []*split.RegionInfo) { + regionsMap := make(map[uint64]*split.RegionInfo, num) + regions := make([]*split.RegionInfo, 0, num) endKey := make([]byte, 8) for i := uint64(0); i < num-1; i++ { - ri := &restore.RegionInfo{ + ri := &split.RegionInfo{ Region: &metapb.Region{ Id: i + 1, Peers: peers, @@ -210,7 +211,7 @@ func TestPaginateScanRegion(t *testing.T) { } else { endKey = codec.EncodeBytes([]byte{}, endKey) } - ri := &restore.RegionInfo{ + ri := &split.RegionInfo{ Region: &metapb.Region{ Id: num, Peers: peers, @@ -225,9 +226,9 @@ func TestPaginateScanRegion(t *testing.T) { } ctx := context.Background() - regionMap := make(map[uint64]*restore.RegionInfo) - var regions []*restore.RegionInfo - var batch []*restore.RegionInfo + regionMap := make(map[uint64]*split.RegionInfo) + var regions []*split.RegionInfo + var batch []*split.RegionInfo require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/br/pkg/restore/scanRegionBackoffer", "return(true)")) _, err := restore.PaginateScanRegion(ctx, NewTestClient(stores, regionMap, 0), []byte{}, []byte{}, 3) require.Error(t, err) diff --git a/br/pkg/task/common.go b/br/pkg/task/common.go index d4cf94c1a0f69..11fe8988a3a58 100644 --- a/br/pkg/task/common.go +++ b/br/pkg/task/common.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/kvproto/pkg/encryptionpb" "github.com/pingcap/log" "github.com/pingcap/tidb/br/pkg/conn" + "github.com/pingcap/tidb/br/pkg/conn/util" berrors "github.com/pingcap/tidb/br/pkg/errors" "github.com/pingcap/tidb/br/pkg/glue" "github.com/pingcap/tidb/br/pkg/metautil" @@ -590,7 +591,7 @@ func NewMgr(ctx context.Context, // Is it necessary to remove `StoreBehavior`? return conn.NewMgr( - ctx, g, pdAddress, tlsConf, securityOption, keepalive, conn.SkipTiFlash, + ctx, g, pdAddress, tlsConf, securityOption, keepalive, util.SkipTiFlash, checkRequirements, needDomain, versionCheckerType, ) }