mysql: autocorrect semi-sync timeout

radondb · Mar 12, 2020 · ad73417 · ad73417
1 parent 2c271e4
commit ad73417
Show file tree

Hide file tree

Showing 10 changed files with 108 additions and 23 deletions.
diff --git a/.codecov.yml b/.codecov.yml
@@ -0,0 +1,13 @@
+coverage:
+  precision: 0
+  round: down
+  range: "70...100"
+  status:
+    project:
+      default:
+        enabled: yes
+        threshold: 1%
+    patch:
+      default:
+        enabled: no
+        threshold: 1%
diff --git a/src/cli/cmd/raft_test.go b/src/cli/cmd/raft_test.go
@@ -14,6 +14,7 @@ import (
 	"raft"
 	"server"
 	"testing"
+	"time"
 	"xbase/common"
 	"xbase/xlog"
 
@@ -43,11 +44,12 @@ func TestCLIRaftCommand(t *testing.T) {
 		}
 	}
 
+	conf, err := GetConfig()
+
 	// 1. test disable raft to leader
 	{
 		// setting xenon is leader
 		{
-			conf, err := GetConfig()
 			ErrorOK(err)
 			conf.Server.Endpoint = leader
 			err = SaveConfig(conf)
@@ -89,6 +91,9 @@ func TestCLIRaftCommand(t *testing.T) {
 			_, err := executeCommand(cmd, "trytoleader")
 			assert.Nil(t, err)
 		}
+
+		// wait cli done, avoid enable cmd changes STOPPED to FOLLOWER
+		time.Sleep(time.Duration(conf.Raft.ElectionTimeout))
 	}
 
 	// 2. test add/remove ndoes to local

diff --git a/src/mysql/api.go b/src/mysql/api.go
@@ -425,13 +425,13 @@ func (m *Mysql) DisableSemiSyncMaster() error {
 	return m.mysqlHandler.DisableSemiSyncMaster(db)
 }
 
-// SetSemiSyncMasterDefault useed to set semi-sync master timeout = default.
-func (m *Mysql) SetSemiSyncMasterDefault() error {
+// SetSemiSyncMasterTimeout used to set semi-sync master timeout.
+func (m *Mysql) SetSemiSyncMasterTimeout(timeout uint64) error {
 	db, err := m.getDB()
 	if err != nil {
 		return err
 	}
-	return m.mysqlHandler.SetSemiSyncMasterDefault(db)
+	return m.mysqlHandler.SetSemiSyncMasterTimeout(db, timeout)
 }
 
 // CheckUserExists used to check the user exists or not.

diff --git a/src/mysql/api_test.go b/src/mysql/api_test.go
@@ -883,7 +883,7 @@ func TestSemiSyncMaster(t *testing.T) {
 	queryList := []string{
 		"SET GLOBAL rpl_semi_sync_master_enabled=ON",
 		"SET GLOBAL rpl_semi_sync_master_enabled=OFF",
-		"SET GLOBAL rpl_semi_sync_master_timeout=default",
+		"SET GLOBAL rpl_semi_sync_master_timeout=300000",
 	}
 
 	mock.ExpectExec(queryList[0]).WillReturnResult(sqlmock.NewResult(1, 1))
@@ -895,7 +895,7 @@ func TestSemiSyncMaster(t *testing.T) {
 	assert.Nil(t, err)
 
 	mock.ExpectExec(queryList[2]).WillReturnResult(sqlmock.NewResult(1, 1))
-	err = mysql.SetSemiSyncMasterDefault()
+	err = mysql.SetSemiSyncMasterTimeout(300000)
 	assert.Nil(t, err)
 }
 

diff --git a/src/mysql/mock.go b/src/mysql/mock.go
@@ -45,7 +45,7 @@ type MockGTID struct {
 	DisableSemiSyncMasterFn    func(*sql.DB) error
 	SelectSysVarFn             func(*sql.DB, string) (string, error)
 	SetSemiWaitSlaveCountFn    func(*sql.DB, int) error
-	SetSemiSyncMasterDefaultFn func(*sql.DB) error
+	SetSemiSyncMasterTimeoutFn func(*sql.DB, uint64) error
 
 	// Users
 	GetUserFn                     func(*sql.DB) ([]model.MysqlUser, error)
@@ -256,9 +256,9 @@ func (mogtid *MockGTID) DisableSemiSyncMaster(db *sql.DB) error {
 	return mogtid.DisableSemiSyncMasterFn(db)
 }
 
-// SetSemiSyncMasterDefault mock.
-func (mogtid *MockGTID) SetSemiSyncMasterDefault(db *sql.DB) error {
-	return mogtid.SetSemiSyncMasterDefaultFn(db)
+// SetSemiSyncMasterTimeout mock.
+func (mogtid *MockGTID) SetSemiSyncMasterTimeout(db *sql.DB, timeout uint64) error {
+	return mogtid.SetSemiSyncMasterTimeoutFn(db, timeout)
 }
 
 // DefaultSelectSysVar mock.
@@ -281,8 +281,8 @@ func (mogtid *MockGTID) SetSemiWaitSlaveCount(db *sql.DB, count int) error {
 	return mogtid.SetSemiWaitSlaveCountFn(db, count)
 }
 
-// SetSemiSyncMasterDefault mock
-func SetSemiSyncMasterDefault(db *sql.DB) error {
+// SetSemiSyncMasterTimeout mock
+func SetSemiSyncMasterTimeout(db *sql.DB, timeout uint64) error {
 	return nil
 }
 
@@ -409,7 +409,7 @@ func defaultMockGTID() *MockGTID {
 	mock.DisableSemiSyncMasterFn = DefaultDisableSemiSyncMaster
 	mock.SelectSysVarFn = DefaultSelectSysVar
 	mock.SetSemiWaitSlaveCountFn = DefaultSetSemiWaitSlaveCount
-	mock.SetSemiSyncMasterDefaultFn = SetSemiSyncMasterDefault
+	mock.SetSemiSyncMasterTimeoutFn = SetSemiSyncMasterTimeout
 
 	// Users.
 	mock.CheckUserExistsFn = DefaultCheckUserExists

diff --git a/src/mysql/mysql_handler.go b/src/mysql/mysql_handler.go
@@ -72,8 +72,8 @@ type MysqlHandler interface {
 	// disable master semi sync: don't wait slave ack
 	DisableSemiSyncMaster(db *sql.DB) error
 
-	// set semi-sync master-timeout = default
-	SetSemiSyncMasterDefault(db *sql.DB) error
+	// set semi-sync master-timeout
+	SetSemiSyncMasterTimeout(db *sql.DB, timeout uint64) error
 
 	//set rpl_semi_master_wait_for_slave_count
 	SetSemiWaitSlaveCount(db *sql.DB, count int) error

diff --git a/src/mysql/mysqlbase.go b/src/mysql/mysqlbase.go
@@ -271,9 +271,9 @@ func (my *MysqlBase) DisableSemiSyncMaster(db *sql.DB) error {
 	return ExecuteWithTimeout(db, reqTimeout, cmds)
 }
 
-// SetSemiSyncMasterDefault useed to set semi-sync master timeout = default
-func (my *MysqlBase) SetSemiSyncMasterDefault(db *sql.DB) error {
-	cmds := "SET GLOBAL rpl_semi_sync_master_timeout=default"
+// SetSemiSyncMasterTimeout useed to set semi-sync master timeout
+func (my *MysqlBase) SetSemiSyncMasterTimeout(db *sql.DB, timeout uint64) error {
+	cmds := fmt.Sprintf("SET GLOBAL rpl_semi_sync_master_timeout=%d", timeout)
 	return ExecuteWithTimeout(db, reqTimeout, cmds)
 }
 

diff --git a/src/mysql/mysqlbase_test.go b/src/mysql/mysqlbase_test.go
@@ -348,17 +348,17 @@ func TestMysqlBaseSemiMaster(t *testing.T) {
 	assert.Nil(t, err)
 }
 
-func TestMysqlBaseSemiMasterDefault(t *testing.T) {
+func TestMysqlBaseSemiMasterTimeout(t *testing.T) {
 	db, mock, err := sqlmock.New()
 	assert.Nil(t, err)
 	defer db.Close()
 
 	queryList := []string{
-		"SET GLOBAL rpl_semi_sync_master_timeout=default",
+		"SET GLOBAL rpl_semi_sync_master_timeout=300000",
 	}
 
 	mock.ExpectExec(queryList[0]).WillReturnResult(sqlmock.NewResult(1, 1))
-	err = mysqlbase.SetSemiSyncMasterDefault(db)
+	err = mysqlbase.SetSemiSyncMasterTimeout(db, 300000)
 	assert.Nil(t, err)
 }
 

diff --git a/src/raft/leader.go b/src/raft/leader.go
@@ -51,6 +51,11 @@ type Leader struct {
 	processPingRequestHandler func(*model.RaftRPCRequest) *model.RaftRPCResponse
 }
 
+const (
+	semisyncTimeoutFor2Nodes = 300000              // 5 minutes
+	semisyncTimeout          = 1000000000000000000 // for 3 or more nodes
+)
+
 // NewLeader creates new Leader.
 func NewLeader(r *Raft) *Leader {
 	L := &Leader{
@@ -488,8 +493,8 @@ func (r *Leader) checkSemiSync() {
 	min := 3
 	cur := r.getMembers()
 	if cur < min {
-		if err := r.mysql.SetSemiSyncMasterDefault(); err != nil {
-			r.ERROR("mysql.set.semi-sync.master.timeout.default.error[%v]", err)
+		if err := r.mysql.SetSemiSyncMasterTimeout(semisyncTimeoutFor2Nodes); err != nil {
+			r.ERROR("mysql.set.semi-sync.master.timeout.to.default.error[%v]", err)
 		}
 	} else {
 		if err := r.mysql.EnableSemiSyncMaster(); err != nil {
@@ -498,6 +503,9 @@ func (r *Leader) checkSemiSync() {
 		if err := r.mysql.SetSemiWaitSlaveCount((cur - 1) / 2); err != nil {
 			r.ERROR("mysql.set.semi.wait.slave.count.error[%v]", err)
 		}
+		if err := r.mysql.SetSemiSyncMasterTimeout(semisyncTimeout); err != nil {
+			r.ERROR("mysql.set.semi.sync.master.timeout.to.infinite.error[%v]", err)
+		}
 	}
 }
 

diff --git a/src/raft/raft_test.go b/src/raft/raft_test.go
@@ -1353,6 +1353,65 @@ func TestRaftLeaderPurgeBinlog(t *testing.T) {
 	}
 }
 
+// TEST EFFECTS:
+// test the leader check semi-sync
+//
+// TEST PROCESSES:
+// 1. set rafts GTID
+//    1.0 rafts[0]  with MockGTID_X1{Master_Log_File = "mysql-bin.000001", Read_Master_Log_Pos = 123}
+//    1.1 rafts[1]  with MockGTID_X3{Master_Log_File = "mysql-bin.000003", Read_Master_Log_Pos = 123}
+//    1.2 rafts[2]  with MockGTID_X5{Master_Log_File = "mysql-bin.000005", Read_Master_Log_Pos = 123}
+// 2. Start 3 rafts state as FOLLOWER
+// 3. wait rafts[2] elected as leader
+// 4. check rafts[2] skipCheckSemiSync
+// 5. Stop all rafts
+func TestRaftLeaderCheckSemiSync(t *testing.T) {
+	conf := config.DefaultRaftConfig()
+	conf.MetaDatadir = "/tmp/"
+
+	log := xlog.NewStdLog(xlog.Level(xlog.PANIC))
+	port := common.RandomPort(8000, 9000)
+	_, rafts, cleanup := MockRaftsWithConfig(log, conf, port, 3, -1)
+	defer cleanup()
+
+	// 1. set rafts GTID
+	//    1.0 rafts[0]  with MockGTIDB{Master_Log_File = "mysql-bin.000001", Read_Master_Log_Pos = 123}
+	//    1.1 rafts[1]  with MockGTIDB{Master_Log_File = "mysql-bin.000003", Read_Master_Log_Pos = 123}
+	//    1.2 rafts[2]  with MockGTIDC{Master_Log_File = "mysql-bin.000005", Read_Master_Log_Pos = 123}
+	{
+		rafts[0].mysql.SetMysqlHandler(mysql.NewMockGTIDX1())
+		rafts[1].mysql.SetMysqlHandler(mysql.NewMockGTIDX3())
+		rafts[2].mysql.SetMysqlHandler(mysql.NewMockGTIDX5())
+	}
+
+	// 2. Start 3 rafts state as FOLLOWER
+	for _, raft := range rafts {
+		raft.Start()
+	}
+
+	// leader
+	{
+		var got State
+		var whoisleader int
+
+		MockWaitLeaderEggs(rafts, 1)
+		MockWaitLeaderEggs(rafts, 0)
+		want := (LEADER + FOLLOWER + FOLLOWER)
+		for i, raft := range rafts {
+			got += raft.getState()
+			if raft.getState() == LEADER {
+				whoisleader = i
+			}
+		}
+
+		assert.Equal(t, want, got)
+		assert.Equal(t, 2, whoisleader)
+
+		// wait for check semi-sync to be invoked
+		time.Sleep(time.Millisecond * time.Duration(rafts[0].getElectionTimeout()*16))
+	}
+}
+
 // TEST EFFECTS:
 // test the follower change master to failed
 //