From f45e7a563de80fa7329eb466f79ec3fa6757ac11 Mon Sep 17 00:00:00 2001 From: Lingyu Song Date: Fri, 28 Jun 2019 22:26:09 +0800 Subject: [PATCH] meta: make auto increment id can be adjust. (#10978) --- ddl/db_test.go | 9 ++++++ executor/admin_test.go | 5 +++ executor/ddl_test.go | 5 +++ meta/autoid/autoid.go | 65 +++++++++++++++++++++++++++++--------- meta/autoid/autoid_test.go | 21 ++++++++++++ 5 files changed, 90 insertions(+), 15 deletions(-) diff --git a/ddl/db_test.go b/ddl/db_test.go index 4c8729adb9cbb..51d261f897d03 100644 --- a/ddl/db_test.go +++ b/ddl/db_test.go @@ -25,6 +25,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/charset" "github.com/pingcap/parser/model" @@ -2850,6 +2851,10 @@ func (s *testDBSuite) TestComment(c *C) { } func (s *testDBSuite) TestRebaseAutoID(c *C) { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil) + }() s.tk = testkit.NewTestKit(c, s.store) s.tk.MustExec("use " + s.schemaName) @@ -4657,6 +4662,10 @@ func (s *testDBSuite) TestCanceledJobTakeTime(c *C) { } func (s *testDBSuite) TestAlterShardRowIDBits(c *C) { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil) + }() s.tk = testkit.NewTestKit(c, s.store) tk := s.tk diff --git a/executor/admin_test.go b/executor/admin_test.go index ac5210e74a0b7..bf6b7ed30bdea 100644 --- a/executor/admin_test.go +++ b/executor/admin_test.go @@ -17,6 +17,7 @@ import ( "fmt" . "github.com/pingcap/check" + "github.com/pingcap/failpoint" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/meta/autoid" @@ -536,6 +537,10 @@ func (s *testSuite) TestAdminCheckPrimaryIndex(c *C) { } func (s *testSuite) TestAdminShowNextID(c *C) { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil) + }() step := int64(10) autoIDStep := autoid.GetStep() autoid.SetStep(step) diff --git a/executor/ddl_test.go b/executor/ddl_test.go index dfbb7997fcc3e..034fad694ab19 100644 --- a/executor/ddl_test.go +++ b/executor/ddl_test.go @@ -21,6 +21,7 @@ import ( "time" . "github.com/pingcap/check" + "github.com/pingcap/failpoint" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" "github.com/pingcap/parser/terror" @@ -260,6 +261,10 @@ func (s *testSuite) TestDefaultDBAfterDropCurDB(c *C) { } func (s *testSuite) TestRenameTable(c *C) { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil) + }() tk := testkit.NewTestKit(c, s.store) tk.MustExec("create database rename1") diff --git a/meta/autoid/autoid.go b/meta/autoid/autoid.go index 4d57b6b2fa174..e2fe504a3bc08 100644 --- a/meta/autoid/autoid.go +++ b/meta/autoid/autoid.go @@ -22,6 +22,7 @@ import ( "github.com/cznic/mathutil" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/parser/terror" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" @@ -30,6 +31,12 @@ import ( "go.uber.org/zap" ) +const ( + minStep = 1000 + maxStep = 2000000 + defaultConsumeTime = 10 * time.Second +) + // Test needs to change it, so it's a variable. var step = int64(30000) @@ -59,8 +66,10 @@ type allocator struct { end int64 store kv.Storage // dbID is current database's ID. - dbID int64 - isUnsigned bool + dbID int64 + isUnsigned bool + lastAllocTime time.Time + step int64 } // GetStep is only used by tests @@ -121,7 +130,7 @@ func (alloc *allocator) rebase4Unsigned(tableID int64, requiredBase uint64, allo uCurrentEnd := uint64(currentEnd) if allocIDs { newBase = mathutil.MaxUint64(uCurrentEnd, requiredBase) - newEnd = mathutil.MinUint64(math.MaxUint64-uint64(step), newBase) + uint64(step) + newEnd = mathutil.MinUint64(math.MaxUint64-uint64(alloc.step), newBase) + uint64(alloc.step) } else { if uCurrentEnd >= requiredBase { newBase = uCurrentEnd @@ -166,7 +175,7 @@ func (alloc *allocator) rebase4Signed(tableID, requiredBase int64, allocIDs bool } if allocIDs { newBase = mathutil.MaxInt64(currentEnd, requiredBase) - newEnd = mathutil.MinInt64(math.MaxInt64-step, newBase) + step + newEnd = mathutil.MinInt64(math.MaxInt64-alloc.step, newBase) + alloc.step } else { if currentEnd >= requiredBase { newBase = currentEnd @@ -212,6 +221,8 @@ func (alloc *allocator) alloc4Unsigned(tableID int64) (int64, error) { if alloc.base == alloc.end { // step var newBase, newEnd int64 startTime := time.Now() + consumeDur := startTime.Sub(alloc.lastAllocTime) + alloc.step = NextStep(alloc.step, consumeDur) err := kv.RunInNewTxn(alloc.store, true, func(txn kv.Transaction) error { m := meta.NewMeta(txn) var err1 error @@ -219,7 +230,7 @@ func (alloc *allocator) alloc4Unsigned(tableID int64) (int64, error) { if err1 != nil { return errors.Trace(err1) } - tmpStep := int64(mathutil.MinUint64(math.MaxUint64-uint64(newBase), uint64(step))) + tmpStep := int64(mathutil.MinUint64(math.MaxUint64-uint64(newBase), uint64(alloc.step))) newEnd, err1 = m.GenAutoTableID(alloc.dbID, tableID, tmpStep) return err1 }) @@ -227,6 +238,7 @@ func (alloc *allocator) alloc4Unsigned(tableID int64) (int64, error) { if err != nil { return 0, err } + alloc.lastAllocTime = time.Now() if uint64(newBase) == math.MaxUint64 { return 0, ErrAutoincReadFailed } @@ -245,6 +257,8 @@ func (alloc *allocator) alloc4Signed(tableID int64) (int64, error) { if alloc.base == alloc.end { // step var newBase, newEnd int64 startTime := time.Now() + consumeDur := startTime.Sub(alloc.lastAllocTime) + alloc.step = NextStep(alloc.step, consumeDur) err := kv.RunInNewTxn(alloc.store, true, func(txn kv.Transaction) error { m := meta.NewMeta(txn) var err1 error @@ -252,7 +266,7 @@ func (alloc *allocator) alloc4Signed(tableID int64) (int64, error) { if err1 != nil { return errors.Trace(err1) } - tmpStep := mathutil.MinInt64(math.MaxInt64-newBase, step) + tmpStep := mathutil.MinInt64(math.MaxInt64-newBase, alloc.step) newEnd, err1 = m.GenAutoTableID(alloc.dbID, tableID, tmpStep) return err1 }) @@ -260,6 +274,7 @@ func (alloc *allocator) alloc4Signed(tableID int64) (int64, error) { if err != nil { return 0, err } + alloc.lastAllocTime = time.Now() if newBase == math.MaxInt64 { return 0, ErrAutoincReadFailed } @@ -287,6 +302,35 @@ func (alloc *allocator) Alloc(tableID int64) (int64, error) { return alloc.alloc4Signed(tableID) } +// NextStep return new auto id step according to previous step and consuming time. +func NextStep(curStep int64, consumeDur time.Duration) int64 { + failpoint.Inject("mockAutoIDChange", func(val failpoint.Value) { + if val.(bool) { + failpoint.Return(step) + } + }) + + consumeRate := defaultConsumeTime.Seconds() / consumeDur.Seconds() + res := int64(float64(curStep) * consumeRate) + if res < minStep { + return minStep + } else if res > maxStep { + return maxStep + } + return res +} + +// NewAllocator returns a new auto increment id generator on the store. +func NewAllocator(store kv.Storage, dbID int64, isUnsigned bool) Allocator { + return &allocator{ + store: store, + dbID: dbID, + isUnsigned: isUnsigned, + step: step, + lastAllocTime: time.Now(), + } +} + var ( memID int64 memIDLock sync.Mutex @@ -340,15 +384,6 @@ func (alloc *memoryAllocator) Alloc(tableID int64) (int64, error) { return alloc.base, nil } -// NewAllocator returns a new auto increment id generator on the store. -func NewAllocator(store kv.Storage, dbID int64, isUnsigned bool) Allocator { - return &allocator{ - store: store, - dbID: dbID, - isUnsigned: isUnsigned, - } -} - // NewMemoryAllocator returns a new auto increment id generator in memory. func NewMemoryAllocator(dbID int64) Allocator { return &memoryAllocator{ diff --git a/meta/autoid/autoid_test.go b/meta/autoid/autoid_test.go index d87759412c6c3..c907277528594 100644 --- a/meta/autoid/autoid_test.go +++ b/meta/autoid/autoid_test.go @@ -21,6 +21,7 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" @@ -39,6 +40,11 @@ type testSuite struct { } func (*testSuite) TestT(c *C) { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil) + }() + store, err := mockstore.NewMockTikvStore() c.Assert(err, IsNil) defer store.Close() @@ -130,6 +136,11 @@ func (*testSuite) TestT(c *C) { } func (*testSuite) TestUnsignedAutoid(c *C) { + c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil) + defer func() { + c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil) + }() + store, err := mockstore.NewMockTikvStore() c.Assert(err, IsNil) defer store.Close() @@ -315,3 +326,13 @@ func (*testSuite) TestRollbackAlloc(c *C) { c.Assert(alloc.Base(), Equals, int64(0)) c.Assert(alloc.End(), Equals, int64(0)) } + +// TestNextStep tests generate next auto id step. +func (*testSuite) TestNextStep(c *C) { + nextStep := autoid.NextStep(2000000, 1*time.Nanosecond) + c.Assert(nextStep, Equals, int64(2000000)) + nextStep = autoid.NextStep(678910, 10*time.Second) + c.Assert(nextStep, Equals, int64(678910)) + nextStep = autoid.NextStep(50000, 10*time.Minute) + c.Assert(nextStep, Equals, int64(1000)) +}