Skip to content

Commit

Permalink
PlannedReparent: tests for Promote failure, re-run failed PRS, PRS wi…
Browse files Browse the repository at this point in the history 
…th current master

Signed-off-by: deepthi <[email protected]>
  • Loading branch information
@deepthi
deepthi committed Apr 14, 2020
1 parent bfe7931 commit eddbb58
Show file tree
Hide file tree
Showing 4 changed files with 407 additions and 168 deletions.
6 changes: 6 additions & 0 deletions go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ type FakeMysqlDaemon struct {
// PromoteResult is returned by Promote
PromoteResult mysql.Position

// PromoteError is used by Promote
PromoteError error

// SchemaFunc provides the return value for GetSchema.
// If not defined, the "Schema" field will be used instead, see below.
SchemaFunc func() (*tabletmanagerdatapb.SchemaDefinition, error)
Expand Down Expand Up @@ -350,6 +353,9 @@ func (fmd *FakeMysqlDaemon) WaitMasterPos(_ context.Context, pos mysql.Position)

// Promote is part of the MysqlDaemon interface
func (fmd *FakeMysqlDaemon) Promote(hookExtraEnv map[string]string) (mysql.Position, error) {
if fmd.PromoteError != nil {
return mysql.Position{}, fmd.PromoteError
}
return fmd.PromoteResult, nil
}

Expand Down
2 changes: 1 addition & 1 deletion go/vt/wrangler/reparent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -641,7 +641,7 @@ func (wr *Wrangler) plannedReparentShardLocked(ctx context.Context, ev *events.R

// Check we still have the topology lock.
if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil {
return fmt.Errorf("lost topology lock, aborting: %v", err)
return vterrors.Wrap(err, "lost topology lock, aborting")
}

// Create a cancelable context for the following RPCs.
Expand Down
107 changes: 65 additions & 42 deletions go/vt/wrangler/testlib/emergency_reparent_shard_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,11 @@ func TestEmergencyReparentShard(t *testing.T) {
vp := NewVtctlPipe(t, ts)
defer vp.Close()

// Create a master, a couple good slaves
// Create a master, a couple good replicas
oldMaster := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, nil)
newMaster := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, nil)
goodSlave1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil)
goodSlave2 := NewFakeTablet(t, wr, "cell2", 3, topodatapb.TabletType_REPLICA, nil)
goodReplica1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil)
goodReplica2 := NewFakeTablet(t, wr, "cell2", 3, topodatapb.TabletType_REPLICA, nil)

// new master
newMaster.FakeMysqlDaemon.ReadOnly = true
Expand Down Expand Up @@ -80,10 +80,10 @@ func TestEmergencyReparentShard(t *testing.T) {
oldMaster.StartActionLoop(t, wr)
defer oldMaster.StopActionLoop(t)

// good slave 1 is replicating
goodSlave1.FakeMysqlDaemon.ReadOnly = true
goodSlave1.FakeMysqlDaemon.Replicating = true
goodSlave1.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{
// good replica 1 is replicating
goodReplica1.FakeMysqlDaemon.ReadOnly = true
goodReplica1.FakeMysqlDaemon.Replicating = true
goodReplica1.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{
GTIDSet: mysql.MariadbGTIDSet{
mysql.MariadbGTID{
Domain: 2,
Expand All @@ -92,19 +92,19 @@ func TestEmergencyReparentShard(t *testing.T) {
},
},
}
goodSlave1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet)
goodSlave1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
goodReplica1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet)
goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"STOP SLAVE",
"FAKE SET MASTER",
"START SLAVE",
}
goodSlave1.StartActionLoop(t, wr)
defer goodSlave1.StopActionLoop(t)
goodReplica1.StartActionLoop(t, wr)
defer goodReplica1.StopActionLoop(t)

// good slave 2 is not replicating
goodSlave2.FakeMysqlDaemon.ReadOnly = true
goodSlave2.FakeMysqlDaemon.Replicating = false
goodSlave2.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{
// good replica 2 is not replicating
goodReplica2.FakeMysqlDaemon.ReadOnly = true
goodReplica2.FakeMysqlDaemon.Replicating = false
goodReplica2.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{
GTIDSet: mysql.MariadbGTIDSet{
mysql.MariadbGTID{
Domain: 2,
Expand All @@ -113,12 +113,12 @@ func TestEmergencyReparentShard(t *testing.T) {
},
},
}
goodSlave2.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet)
goodSlave2.StartActionLoop(t, wr)
goodSlave2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
goodReplica2.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet)
goodReplica2.StartActionLoop(t, wr)
goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"FAKE SET MASTER",
}
defer goodSlave2.StopActionLoop(t)
defer goodReplica2.StopActionLoop(t)

// run EmergencyReparentShard
if err := vp.Run([]string{"EmergencyReparentShard", "-wait_slave_timeout", "10s", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, topoproto.TabletAliasString(newMaster.Tablet.Alias)}); err != nil {
Expand All @@ -132,30 +132,30 @@ func TestEmergencyReparentShard(t *testing.T) {
if err := oldMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("oldMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
if err := goodSlave1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("goodSlave1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
if err := goodReplica1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("goodReplica1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
if err := goodSlave2.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("goodSlave2.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
if err := goodReplica2.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("goodReplica2.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
if newMaster.FakeMysqlDaemon.ReadOnly {
t.Errorf("newMaster.FakeMysqlDaemon.ReadOnly set")
}
// old master read-only flag doesn't matter, it is scrapped
if !goodSlave1.FakeMysqlDaemon.ReadOnly {
t.Errorf("goodSlave1.FakeMysqlDaemon.ReadOnly not set")
if !goodReplica1.FakeMysqlDaemon.ReadOnly {
t.Errorf("goodReplica1.FakeMysqlDaemon.ReadOnly not set")
}
if !goodSlave2.FakeMysqlDaemon.ReadOnly {
t.Errorf("goodSlave2.FakeMysqlDaemon.ReadOnly not set")
if !goodReplica2.FakeMysqlDaemon.ReadOnly {
t.Errorf("goodReplica2.FakeMysqlDaemon.ReadOnly not set")
}
if !goodSlave1.FakeMysqlDaemon.Replicating {
t.Errorf("goodSlave1.FakeMysqlDaemon.Replicating not set")
if !goodReplica1.FakeMysqlDaemon.Replicating {
t.Errorf("goodReplica1.FakeMysqlDaemon.Replicating not set")
}
if goodSlave2.FakeMysqlDaemon.Replicating {
t.Errorf("goodSlave2.FakeMysqlDaemon.Replicating set")
if goodReplica2.FakeMysqlDaemon.Replicating {
t.Errorf("goodReplica2.FakeMysqlDaemon.Replicating set")
}
checkSemiSyncEnabled(t, true, true, newMaster)
checkSemiSyncEnabled(t, false, true, goodSlave1, goodSlave2)
checkSemiSyncEnabled(t, false, true, goodReplica1, goodReplica2)
}

// TestEmergencyReparentShardMasterElectNotBest tries to emergency reparent
Expand All @@ -165,13 +165,25 @@ func TestEmergencyReparentShardMasterElectNotBest(t *testing.T) {
ts := memorytopo.NewServer("cell1", "cell2")
wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient())

// Create a master, a couple good slaves
// Create a master, a couple good replicas
oldMaster := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, nil)
newMaster := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, nil)
moreAdvancedSlave := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil)
moreAdvancedReplica := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil)

// new master
newMaster.FakeMysqlDaemon.Replicating = true
// this server has executed upto 455, which is the highest among replicas
newMaster.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{
GTIDSet: mysql.MariadbGTIDSet{
mysql.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 455,
},
},
}
// It has more transactions in its relay log, but not as many as
// moreAdvancedReplica
newMaster.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{
GTIDSet: mysql.MariadbGTIDSet{
mysql.MariadbGTID{
Expand All @@ -191,9 +203,20 @@ func TestEmergencyReparentShardMasterElectNotBest(t *testing.T) {
oldMaster.StartActionLoop(t, wr)
defer oldMaster.StopActionLoop(t)

// more advanced slave
moreAdvancedSlave.FakeMysqlDaemon.Replicating = true
moreAdvancedSlave.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{
// more advanced replica
moreAdvancedReplica.FakeMysqlDaemon.Replicating = true
// position up to which this replica has executed is behind desired new master
moreAdvancedReplica.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{
GTIDSet: mysql.MariadbGTIDSet{
mysql.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 454,
},
},
}
// relay log position is more advanced than desired new master
moreAdvancedReplica.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{
GTIDSet: mysql.MariadbGTIDSet{
mysql.MariadbGTID{
Domain: 2,
Expand All @@ -202,11 +225,11 @@ func TestEmergencyReparentShardMasterElectNotBest(t *testing.T) {
},
},
}
moreAdvancedSlave.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
moreAdvancedReplica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"STOP SLAVE",
}
moreAdvancedSlave.StartActionLoop(t, wr)
defer moreAdvancedSlave.StopActionLoop(t)
moreAdvancedReplica.StartActionLoop(t, wr)
defer moreAdvancedReplica.StopActionLoop(t)

// run EmergencyReparentShard
if err := wr.EmergencyReparentShard(ctx, newMaster.Tablet.Keyspace, newMaster.Tablet.Shard, newMaster.Tablet.Alias, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is more advanced than master elect tablet") {
Expand All @@ -220,7 +243,7 @@ func TestEmergencyReparentShardMasterElectNotBest(t *testing.T) {
if err := oldMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("oldMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
if err := moreAdvancedSlave.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("moreAdvancedSlave.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
if err := moreAdvancedReplica.FakeMysqlDaemon.CheckSuperQueryList(); err != nil {
t.Fatalf("moreAdvancedReplica.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err)
}
}
Loading

0 comments on commit eddbb58

Please sign in to comment.