diff --git a/executor/builder.go b/executor/builder.go index 8db1bdcdb0adb..7f7921a231197 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -169,7 +169,7 @@ func (b *executorBuilder) build(p plannercore.Plan) Executor { case *plannercore.PhysicalIndexLookUpReader: return b.buildIndexLookUpReader(v) case *plannercore.SplitRegion: - return b.buildSplitIndexRegion(v) + return b.buildSplitRegion(v) default: b.err = ErrUnknownPlan.GenWithStack("Unknown Plan %T", p) return nil @@ -1324,17 +1324,33 @@ func (b *executorBuilder) buildUnionAll(v *plannercore.PhysicalUnionAll) Executo return e } -func (b *executorBuilder) buildSplitIndexRegion(v *plannercore.SplitRegion) Executor { +func (b *executorBuilder) buildSplitRegion(v *plannercore.SplitRegion) Executor { base := newBaseExecutor(b.ctx, nil, v.ExplainID()) base.initCap = chunk.ZeroCapacity - return &SplitIndexRegionExec{ + if v.IndexInfo != nil { + return &SplitIndexRegionExec{ + baseExecutor: base, + tableInfo: v.TableInfo, + indexInfo: v.IndexInfo, + lower: v.Lower, + upper: v.Upper, + num: v.Num, + valueLists: v.ValueLists, + } + } + if len(v.ValueLists) > 0 { + return &SplitTableRegionExec{ + baseExecutor: base, + tableInfo: v.TableInfo, + valueLists: v.ValueLists, + } + } + return &SplitTableRegionExec{ baseExecutor: base, tableInfo: v.TableInfo, - indexInfo: v.IndexInfo, - lower: v.Lower, - upper: v.Upper, + lower: v.Lower[0], + upper: v.Upper[0], num: v.Num, - valueLists: v.ValueLists, } } diff --git a/executor/executor_test.go b/executor/executor_test.go index 5398d7fdec47d..0d443f84badbb 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -3518,7 +3518,7 @@ func (s *testSuite) TestUnsignedFeedback(c *C) { c.Assert(result.Rows()[2][3], Equals, "table:t, range:[0,+inf], keep order:false") } -func (s *testSuite) TestSplitIndexRegion(c *C) { +func (s *testSuite) TestSplitRegion(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") tk.MustExec("drop table if exists t") @@ -3529,26 +3529,27 @@ func (s *testSuite) TestSplitIndexRegion(c *C) { terr := errors.Cause(err).(*terror.Error) c.Assert(terr.Code(), Equals, terror.ErrCode(mysql.WarnDataTruncated)) + // Test for split index region. // Check min value is more than max value. tk.MustExec(`split table t index idx1 between (0) and (1000000000) regions 10`) _, err = tk.Exec(`split table t index idx1 between (2,'a') and (1,'c') regions 10`) c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "Split index region `idx1` lower value (2,a) should less than the upper value (1,c)") + c.Assert(err.Error(), Equals, "Split index `idx1` region lower value (2,a) should less than the upper value (1,c)") // Check min value is invalid. _, err = tk.Exec(`split table t index idx1 between () and (1) regions 10`) c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "Split index region `idx1` lower value count should more than 0") + c.Assert(err.Error(), Equals, "Split index `idx1` region lower value count should more than 0") // Check max value is invalid. _, err = tk.Exec(`split table t index idx1 between (1) and () regions 10`) c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "Split index region `idx1` upper value count should more than 0") + c.Assert(err.Error(), Equals, "Split index `idx1` region upper value count should more than 0") // Check pre-split region num is too large. _, err = tk.Exec(`split table t index idx1 between (0) and (1000000000) regions 10000`) c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "Split index region num is exceed the limit 1000") + c.Assert(err.Error(), Equals, "Split index region num exceeded the limit 1000") // Check pre-split region num 0 is invalid. _, err = tk.Exec(`split table t index idx1 between (0) and (1000000000) regions 0`) @@ -3558,7 +3559,47 @@ func (s *testSuite) TestSplitIndexRegion(c *C) { // Test truncate error msg. _, err = tk.Exec(`split table t index idx1 between ("aa") and (1000000000) regions 0`) c.Assert(err, NotNil) - c.Assert(err.Error(), Equals, "[types:1265]Incorrect value: 'aa' for index column 'b'") + c.Assert(err.Error(), Equals, "[types:1265]Incorrect value: 'aa' for column 'b'") + + // Test for split table region. + tk.MustExec(`split table t between (0) and (1000000000) regions 10`) + // Check the ower value is more than the upper value. + _, err = tk.Exec(`split table t between (2) and (1) regions 10`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split table `t` region lower value 2 should less than the upper value 1") + + // Check the lower value is invalid. + _, err = tk.Exec(`split table t between () and (1) regions 10`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split table region lower value count should be 1") + + // Check upper value is invalid. + _, err = tk.Exec(`split table t between (1) and () regions 10`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split table region upper value count should be 1") + + // Check pre-split region num is too large. + _, err = tk.Exec(`split table t between (0) and (1000000000) regions 10000`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split table region num exceeded the limit 1000") + + // Check pre-split region num 0 is invalid. + _, err = tk.Exec(`split table t between (0) and (1000000000) regions 0`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split table region num should more than 0") + + // Test truncate error msg. + _, err = tk.Exec(`split table t between ("aa") and (1000000000) regions 10`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "[types:1265]Incorrect value: 'aa' for column '_tidb_rowid'") + + // Test split table region step is too small. + _, err = tk.Exec(`split table t between (0) and (100) regions 10`) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Split table `t` region step value should more than 1000, step 10 is invalid") + + // Test split region by syntax + tk.MustExec(`split table t by (0),(1000),(1000000)`) } type testOOMSuite struct { diff --git a/executor/split.go b/executor/split.go index 369a0ed58cad2..ade0682692379 100644 --- a/executor/split.go +++ b/executor/split.go @@ -22,6 +22,7 @@ import ( "github.com/cznic/mathutil" "github.com/pingcap/errors" "github.com/pingcap/parser/model" + "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/tablecodec" @@ -126,9 +127,9 @@ func (e *SplitIndexRegionExec) getSplitIdxKeys() ([][]byte, error) { lowerStr, err1 := datumSliceToString(e.lower) upperStr, err2 := datumSliceToString(e.upper) if err1 != nil || err2 != nil { - return nil, errors.Errorf("Split index region `%v` lower value %v should less than the upper value %v", e.indexInfo.Name, e.lower, e.upper) + return nil, errors.Errorf("Split index `%v` region lower value %v should less than the upper value %v", e.indexInfo.Name, e.lower, e.upper) } - return nil, errors.Errorf("Split index region `%v` lower value %v should less than the upper value %v", e.indexInfo.Name, lowerStr, upperStr) + return nil, errors.Errorf("Split index `%v` region lower value %v should less than the upper value %v", e.indexInfo.Name, lowerStr, upperStr) } return getValuesList(lowerIdxKey, upperIdxKey, e.num, idxKeys), nil } @@ -204,3 +205,106 @@ func datumSliceToString(ds []types.Datum) (string, error) { str += ")" return str, nil } + +// SplitTableRegionExec represents a split table regions executor. +type SplitTableRegionExec struct { + baseExecutor + + tableInfo *model.TableInfo + lower types.Datum + upper types.Datum + num int + valueLists [][]types.Datum +} + +// Next implements the Executor Next interface. +func (e *SplitTableRegionExec) Next(ctx context.Context, _ *chunk.Chunk) error { + store := e.ctx.GetStore() + s, ok := store.(splitableStore) + if !ok { + return nil + } + splitKeys, err := e.getSplitTableKeys() + if err != nil { + return err + } + regionIDs := make([]uint64, 0, len(splitKeys)) + for _, key := range splitKeys { + regionID, err := s.SplitRegionAndScatter(key) + if err != nil { + logutil.Logger(context.Background()).Warn("split table region failed", + zap.String("table", e.tableInfo.Name.L), + zap.Error(err)) + continue + } + regionIDs = append(regionIDs, regionID) + } + if !e.ctx.GetSessionVars().WaitTableSplitFinish { + return nil + } + for _, regionID := range regionIDs { + err := s.WaitScatterRegionFinish(regionID) + if err != nil { + logutil.Logger(context.Background()).Warn("wait scatter region failed", + zap.Uint64("regionID", regionID), + zap.String("table", e.tableInfo.Name.L), + zap.Error(err)) + } + } + return nil +} + +var minRegionStepValue = uint64(1000) + +func (e *SplitTableRegionExec) getSplitTableKeys() ([][]byte, error) { + var keys [][]byte + if e.num > 0 { + keys = make([][]byte, 0, e.num) + } else { + keys = make([][]byte, 0, len(e.valueLists)) + } + recordPrefix := tablecodec.GenTableRecordPrefix(e.tableInfo.ID) + if len(e.valueLists) > 0 { + for _, v := range e.valueLists { + key := tablecodec.EncodeRecordKey(recordPrefix, v[0].GetInt64()) + keys = append(keys, key) + } + return keys, nil + } + isUnsigned := false + if e.tableInfo.PKIsHandle { + if pkCol := e.tableInfo.GetPkColInfo(); pkCol != nil { + isUnsigned = mysql.HasUnsignedFlag(pkCol.Flag) + } + } + var step uint64 + var lowerValue int64 + if isUnsigned { + lowerRecordID := e.lower.GetUint64() + upperRecordID := e.upper.GetUint64() + if upperRecordID <= lowerRecordID { + return nil, errors.Errorf("Split table `%s` region lower value %v should less than the upper value %v", e.tableInfo.Name, lowerRecordID, upperRecordID) + } + step = (upperRecordID - lowerRecordID) / uint64(e.num) + lowerValue = int64(lowerRecordID) + } else { + lowerRecordID := e.lower.GetInt64() + upperRecordID := e.upper.GetInt64() + if upperRecordID <= lowerRecordID { + return nil, errors.Errorf("Split table `%s` region lower value %v should less than the upper value %v", e.tableInfo.Name, lowerRecordID, upperRecordID) + } + step = uint64(upperRecordID-lowerRecordID) / uint64(e.num) + lowerValue = lowerRecordID + } + if step < minRegionStepValue { + return nil, errors.Errorf("Split table `%s` region step value should more than %v, step %v is invalid", e.tableInfo.Name, minRegionStepValue, step) + } + + recordID := lowerValue + for i := 1; i < e.num; i++ { + recordID += int64(step) + key := tablecodec.EncodeRecordKey(recordPrefix, recordID) + keys = append(keys, key) + } + return keys, nil +} diff --git a/executor/split_test.go b/executor/split_test.go index 7ca296af934c0..ccc82e7efca1d 100644 --- a/executor/split_test.go +++ b/executor/split_test.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/table/tables" + "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/mock" ) @@ -285,6 +286,81 @@ func (s *testSplitIndex) TestSplitIndex(c *C) { } } +func (s *testSplitIndex) TestSplitTable(c *C) { + tbInfo := &model.TableInfo{ + Name: model.NewCIStr("t1"), + ID: rand.Int63(), + Columns: []*model.ColumnInfo{ + { + Name: model.NewCIStr("c0"), + ID: 1, + Offset: 1, + DefaultValue: 0, + State: model.StatePublic, + FieldType: *types.NewFieldType(mysql.TypeLong), + }, + }, + } + defer func(originValue uint64) { + minRegionStepValue = originValue + }(minRegionStepValue) + minRegionStepValue = 10 + // range is 0 ~ 100, and split into 10 region. + // So 10 regions range is like below: + // region1: [-inf ~ 10) + // region2: [10 ~ 20) + // region3: [20 ~ 30) + // region4: [30 ~ 40) + // region5: [40 ~ 50) + // region6: [50 ~ 60) + // region7: [60 ~ 70) + // region8: [70 ~ 80) + // region9: [80 ~ 90 ) + // region10: [90 ~ +inf) + ctx := mock.NewContext() + e := &SplitTableRegionExec{ + baseExecutor: newBaseExecutor(ctx, nil, ""), + tableInfo: tbInfo, + lower: types.NewDatum(0), + upper: types.NewDatum(100), + num: 10, + } + valueList, err := e.getSplitTableKeys() + c.Assert(err, IsNil) + c.Assert(len(valueList), Equals, e.num-1) + + cases := []struct { + value int + lessEqualIdx int + }{ + {-1, -1}, + {0, -1}, + {1, -1}, + {10, 0}, + {11, 0}, + {20, 1}, + {21, 1}, + {31, 2}, + {41, 3}, + {51, 4}, + {61, 5}, + {71, 6}, + {81, 7}, + {91, 8}, + {100, 8}, + {1000, 8}, + } + + recordPrefix := tablecodec.GenTableRecordPrefix(e.tableInfo.ID) + for _, ca := range cases { + // test for minInt64 handle + key := tablecodec.EncodeRecordKey(recordPrefix, int64(ca.value)) + c.Assert(err, IsNil) + idx := searchLessEqualIdx(valueList, key) + c.Assert(idx, Equals, ca.lessEqualIdx, Commentf("%#v", ca)) + } +} + func searchLessEqualIdx(valueList [][]byte, value []byte) int { idx := -1 for i, v := range valueList { diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index 5826fea556643..a39aae71b63ac 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -1432,6 +1432,13 @@ func (b *planBuilder) buildLoadStats(ld *ast.LoadStatsStmt) Plan { const maxSplitRegionNum = 1000 func (b *planBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) { + if len(node.IndexName.L) != 0 { + return b.buildSplitIndexRegion(node) + } + return b.buildSplitTableRegion(node) +} + +func (b *planBuilder) buildSplitIndexRegion(node *ast.SplitRegionStmt) (Plan, error) { tblInfo := node.Table.TableInfo indexInfo := tblInfo.FindIndexByName(node.IndexName.L) if indexInfo == nil { @@ -1465,10 +1472,10 @@ func (b *planBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) // Split index regions by lower, upper value. checkLowerUpperValue := func(valuesItem []ast.ExprNode, name string) ([]types.Datum, error) { if len(valuesItem) == 0 { - return nil, errors.Errorf("Split index region `%v` %s value count should more than 0", indexInfo.Name, name) + return nil, errors.Errorf("Split index `%v` region %s value count should more than 0", indexInfo.Name, name) } if len(valuesItem) > len(indexInfo.Columns) { - return nil, errors.Errorf("Split index region `%v` Column count doesn't match value count at %v", indexInfo.Name, name) + return nil, errors.Errorf("Split index `%v` region column count doesn't match value count at %v", indexInfo.Name, name) } return b.convertValue2ColumnType(valuesItem, mockTablePlan, indexInfo, tblInfo) } @@ -1484,7 +1491,7 @@ func (b *planBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) p.Upper = upperValues if node.SplitOpt.Num > maxSplitRegionNum { - return nil, errors.Errorf("Split index region num is exceed the limit %v", maxSplitRegionNum) + return nil, errors.Errorf("Split index region num exceeded the limit %v", maxSplitRegionNum) } else if node.SplitOpt.Num < 1 { return nil, errors.Errorf("Split index region num should more than 0") } @@ -1495,43 +1502,110 @@ func (b *planBuilder) buildSplitRegion(node *ast.SplitRegionStmt) (Plan, error) func (b *planBuilder) convertValue2ColumnType(valuesItem []ast.ExprNode, mockTablePlan LogicalPlan, indexInfo *model.IndexInfo, tblInfo *model.TableInfo) ([]types.Datum, error) { values := make([]types.Datum, 0, len(valuesItem)) for j, valueItem := range valuesItem { - var expr expression.Expression - var err error - switch x := valueItem.(type) { - case *driver.ValueExpr: - expr = &expression.Constant{ - Value: x.Datum, - RetType: &x.Type, - } - default: - expr, _, err = b.rewrite(valueItem, mockTablePlan, nil, true) - if err != nil { - return nil, err - } - } - constant, ok := expr.(*expression.Constant) - if !ok { - return nil, errors.New("expect constant values") - } - value, err := constant.Eval(chunk.Row{}) + colOffset := indexInfo.Columns[j].Offset + value, err := b.convertValue(valueItem, mockTablePlan, tblInfo.Columns[colOffset]) if err != nil { return nil, err } - colOffset := indexInfo.Columns[j].Offset - value1, err := value.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &tblInfo.Columns[colOffset].FieldType) + values = append(values, value) + } + return values, nil +} + +func (b *planBuilder) convertValue(valueItem ast.ExprNode, mockTablePlan LogicalPlan, col *model.ColumnInfo) (d types.Datum, err error) { + var expr expression.Expression + switch x := valueItem.(type) { + case *driver.ValueExpr: + expr = &expression.Constant{ + Value: x.Datum, + RetType: &x.Type, + } + default: + expr, _, err = b.rewrite(valueItem, mockTablePlan, nil, true) if err != nil { - if !types.ErrTruncated.Equal(err) { - return nil, err + return d, err + } + } + constant, ok := expr.(*expression.Constant) + if !ok { + return d, errors.New("Expect constant values") + } + value, err := constant.Eval(chunk.Row{}) + if err != nil { + return d, err + } + d, err = value.ConvertTo(b.ctx.GetSessionVars().StmtCtx, &col.FieldType) + if err != nil { + if !types.ErrTruncated.Equal(err) { + return d, err + } + valStr, err1 := value.ToString() + if err1 != nil { + return d, err + } + return d, types.ErrTruncated.GenWithStack("Incorrect value: '%-.128s' for column '%.192s'", valStr, col.Name.O) + } + return d, nil +} + +func (b *planBuilder) buildSplitTableRegion(node *ast.SplitRegionStmt) (Plan, error) { + tblInfo := node.Table.TableInfo + var pkCol *model.ColumnInfo + if tblInfo.PKIsHandle { + if col := tblInfo.GetPkColInfo(); col != nil { + pkCol = col + } + } + if pkCol == nil { + pkCol = model.NewExtraHandleColInfo() + } + mockTablePlan := LogicalTableDual{}.init(b.ctx) + schema := expression.TableInfo2SchemaWithDBName(b.ctx, node.Table.Schema, tblInfo) + mockTablePlan.SetSchema(schema) + + p := &SplitRegion{ + TableInfo: tblInfo, + } + if len(node.SplitOpt.ValueLists) > 0 { + values := make([][]types.Datum, 0, len(node.SplitOpt.ValueLists)) + for i, valuesItem := range node.SplitOpt.ValueLists { + if len(valuesItem) > 1 { + return nil, ErrWrongValueCountOnRow.GenWithStackByArgs(i + 1) } - valStr, err1 := value.ToString() - if err1 != nil { + value, err := b.convertValue(valuesItem[0], mockTablePlan, pkCol) + if err != nil { return nil, err } - return nil, types.ErrTruncated.GenWithStack("Incorrect value: '%-.128s' for index column '%.192s'", valStr, tblInfo.Columns[colOffset].Name.O) + values = append(values, []types.Datum{value}) } - values = append(values, value1) + p.ValueLists = values + return p, nil } - return values, nil + + checkLowerUpperValue := func(valuesItem []ast.ExprNode, name string) (types.Datum, error) { + if len(valuesItem) != 1 { + return types.Datum{}, errors.Errorf("Split table region %s value count should be 1", name) + } + return b.convertValue(valuesItem[0], mockTablePlan, pkCol) + } + lowerValues, err := checkLowerUpperValue(node.SplitOpt.Lower, "lower") + if err != nil { + return nil, err + } + upperValue, err := checkLowerUpperValue(node.SplitOpt.Upper, "upper") + if err != nil { + return nil, err + } + p.Lower = []types.Datum{lowerValues} + p.Upper = []types.Datum{upperValue} + + if node.SplitOpt.Num > maxSplitRegionNum { + return nil, errors.Errorf("Split table region num exceeded the limit %v", maxSplitRegionNum) + } else if node.SplitOpt.Num < 1 { + return nil, errors.Errorf("Split table region num should more than 0") + } + p.Num = int(node.SplitOpt.Num) + return p, nil } func (b *planBuilder) buildDDL(node ast.DDLNode) (Plan, error) {