fix: automatic fix master-slave replication relationship after master…

… or slave service restarted (OpenAtomFoundation#2373, OpenAtomFoundation#2038, OpenAtomFoundation#1950, OpenAtomFoundation#1967, OpenAtomFoundation#2351))

codis/config/dashboard.toml

@@ -33,9 +33,10 @@ migration_async_numkeys = 500
 migration_timeout = "30s"
 
 # Set configs for redis sentinel.
-sentinel_check_server_state_interval = "5s"
-sentinel_check_master_failover_interval = "1s"
-sentinel_master_dead_check_times = 5
+sentinel_check_server_state_interval = "10s"
+sentinel_check_master_failover_interval = "2s"
+sentinel_master_dead_check_times = 10
+sentinel_check_offline_server_interval = "2s"
 sentinel_client_timeout = "10s"
 sentinel_quorum = 2
 sentinel_parallel_syncs = 1

codis/pkg/models/action.go

@@ -11,4 +11,7 @@ const (
 	ActionMigrating = "migrating"
 	ActionFinished  = "finished"
 	ActionSyncing   = "syncing"
+	ActionSynced    = "synced"
+
+	ActionSyncedFailed = "synced_failed"
 )

codis/pkg/models/group.go

@@ -25,6 +25,38 @@ func (g *Group) GetServersMap() map[string]*GroupServer {
 	return results
 }
 
+// SelectNewMaster choose a new master node in the group
+func (g *Group) SelectNewMaster() (string, int) {
+	var newMasterServer *GroupServer
+	var newMasterIndex = -1
+
+	for index, server := range g.Servers {
+		if index == 0 || server.State != GroupServerStateNormal {
+			continue
+		}
+
+		if newMasterServer == nil {
+			newMasterServer = server
+			newMasterIndex = index
+		} else if server.DbBinlogFileNum > newMasterServer.DbBinlogFileNum {
+			// Select the slave node with the latest offset as the master node
+			newMasterServer = server
+			newMasterIndex = index
+		} else if server.DbBinlogFileNum == newMasterServer.DbBinlogFileNum {
+			if server.DbBinlogOffset > newMasterServer.DbBinlogOffset {
+				newMasterServer = server
+				newMasterIndex = index
+			}
+		}
+	}
+
+	if newMasterServer == nil {
+		return "", newMasterIndex
+	}
+
+	return newMasterServer.Addr, newMasterIndex
+}
+
 type GroupServerState int8
 
 const (
@@ -33,6 +65,13 @@ const (
 	GroupServerStateOffline
 )
 
+type GroupServerRole string
+
+const (
+	RoleMaster GroupServerRole = "master"
+	RoleSlave  GroupServerRole = "slave"
+)
+
 type GroupServer struct {
 	Addr       string `json:"server"`
 	DataCenter string `json:"datacenter"`
@@ -43,9 +82,11 @@ type GroupServer struct {
 	} `json:"action"`
 
 	// master or slave
-	Role string `json:"role"`
+	Role GroupServerRole `json:"role"`
 	// If it is a master node, take the master_repl_offset field, otherwise take the slave_repl_offset field
-	ReplyOffset int `json:"reply_offset"`
+	DbBinlogFileNum uint64 `json:"binlog_file_num"` // db0
+	DbBinlogOffset  uint64 `json:"binlog_offset"`   // db0
+
 	// Monitoring status, 0 normal, 1 subjective offline, 2 actual offline
 	// If marked as 2 , no service is provided
 	State GroupServerState `json:"state"`

codis/pkg/topom/config.go

@@ -50,9 +50,10 @@ migration_async_numkeys = 500
 migration_timeout = "30s"
 
 # Set configs for redis sentinel.
-sentinel_check_server_state_interval = "5s"
-sentinel_check_master_failover_interval = "1s"
-sentinel_master_dead_check_times = 5
+sentinel_check_server_state_interval = "10s"
+sentinel_check_master_failover_interval = "2s"
+sentinel_master_dead_check_times = 10
+sentinel_check_offline_server_interval = "2s"
 sentinel_client_timeout = "10s"
 sentinel_quorum = 2
 sentinel_parallel_syncs = 1
@@ -86,6 +87,7 @@ type Config struct {
 	SentinelCheckServerStateInterval    timesize.Duration `toml:"sentinel_check_server_state_interval" json:"sentinel_client_timeout"`
 	SentinelCheckMasterFailoverInterval timesize.Duration `toml:"sentinel_check_master_failover_interval" json:"sentinel_check_master_failover_interval"`
 	SentinelMasterDeadCheckTimes        int8              `toml:"sentinel_master_dead_check_times" json:"sentinel_master_dead_check_times"`
+	SentinelCheckOfflineServerInterval  timesize.Duration `toml:"sentinel_check_offline_server_interval" json:"sentinel_check_offline_server_interval"`
 	SentinelClientTimeout               timesize.Duration `toml:"sentinel_client_timeout" json:"sentinel_client_timeout"`
 	SentinelQuorum                      int               `toml:"sentinel_quorum" json:"sentinel_quorum"`
 	SentinelParallelSyncs               int               `toml:"sentinel_parallel_syncs" json:"sentinel_parallel_syncs"`

codis/pkg/topom/context.go

@@ -40,7 +40,7 @@ func (ctx *context) getSlotMapping(sid int) (*models.SlotMapping, error) {
 }
 
 func (ctx *context) getSlotMappingsByGroupId(gid int) []*models.SlotMapping {
-	var slots = []*models.SlotMapping{}
+	var slots []*models.SlotMapping
 	for _, m := range ctx.slots {
 		if m.GroupId == gid || m.Action.TargetId == gid {
 			slots = append(slots, m)

codis/pkg/topom/topom.go

@@ -210,12 +210,12 @@ func (s *Topom) Start(routines bool) error {
 		}
 	}, nil, true, 0)
 
-	// Check the status of the pre-offline master every 1 second
+	// Check the status of the pre-offline master every 2 second
 	// to determine whether to automatically switch master and slave
 	gxruntime.GoUnterminated(func() {
 		for !s.IsClosed() {
 			if s.IsOnline() {
-				w, _ := s.CheckPreOffineMastersState(5 * time.Second)
+				w, _ := s.CheckPreOfflineMastersState(5 * time.Second)
 				if w != nil {
 					w.Wait()
 				}
@@ -224,6 +224,20 @@ func (s *Topom) Start(routines bool) error {
 		}
 	}, nil, true, 0)
 
+	// Check the status of the offline master and slave every 30 second
+	// to determine whether to automatically recover to right master-slave replication relationship
+	gxruntime.GoUnterminated(func() {
+		for !s.IsClosed() {
+			if s.IsOnline() {
+				w, _ := s.CheckOfflineMastersAndSlavesState(5 * time.Second)
+				if w != nil {
+					w.Wait()
+				}
+			}
+			time.Sleep(s.Config().SentinelCheckOfflineServerInterval.Duration())
+		}
+	}, nil, true, 0)
+
 	gxruntime.GoUnterminated(func() {
 		for !s.IsClosed() {
 			if s.IsOnline() {