From 4378c835796c087c8111d92ebdbf1be9c3ac0679 Mon Sep 17 00:00:00 2001 From: deepthi Date: Wed, 8 Apr 2020 16:20:15 -0700 Subject: [PATCH 1/6] PlannedReparent: call SetReadOnly in PromoteReplica only after master tablet is written to topo Signed-off-by: deepthi --- .../vttablet/tabletmanager/rpc_replication.go | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index abb6be622b4..5e49c658b39 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -469,6 +469,7 @@ func (agent *ActionAgent) PromoteSlaveWhenCaughtUp(ctx context.Context, position pos, err := mysql.DecodePosition(position) if err != nil { + return "", err } @@ -487,19 +488,22 @@ func (agent *ActionAgent) PromoteSlaveWhenCaughtUp(ctx context.Context, position } startTime := time.Now() - if err := agent.MysqlDaemon.SetReadOnly(false); err != nil { - return "", err - } - _, err = topotools.ChangeType(ctx, agent.TopoServer, agent.TabletAlias, topodatapb.TabletType_MASTER, logutil.TimeToProto(startTime)) if err != nil { return "", err } + // We only update agent's masterTermStartTime if we were able to update the topo. // This ensures that in case of a failure, we are never in a situation where the // tablet's timestamp is ahead of the topo's timestamp. agent.setMasterTermStartTime(startTime) + // We call SetReadOnly only after the topo has been updated to avoid + // situations where two tablets are master at the DB level but not at the vitess level + if err := agent.MysqlDaemon.SetReadOnly(false); err != nil { + return "", err + } + if err := agent.refreshTablet(ctx, "PromoteSlaveWhenCaughtUp"); err != nil { return "", err } @@ -745,7 +749,7 @@ func (agent *ActionAgent) StopReplicationAndGetStatus(ctx context.Context) (*rep } // PromoteSlave makes the current tablet the master -func (agent *ActionAgent) PromoteSlave(ctx context.Context) (string, error) { +func (agent *ActionAgent) PromoteSlave(ctx context.Context) (replicationPosition string, finalErr error) { if err := agent.lock(ctx); err != nil { return "", err } @@ -763,13 +767,16 @@ func (agent *ActionAgent) PromoteSlave(ctx context.Context) (string, error) { // Set the server read-write startTime := time.Now() - if err := agent.MysqlDaemon.SetReadOnly(false); err != nil { + if _, err := topotools.ChangeType(ctx, agent.TopoServer, agent.TabletAlias, topodatapb.TabletType_MASTER, logutil.TimeToProto(startTime)); err != nil { return "", err } - if _, err := topotools.ChangeType(ctx, agent.TopoServer, agent.TabletAlias, topodatapb.TabletType_MASTER, logutil.TimeToProto(startTime)); err != nil { + // We call SetReadOnly only after the topo has been updated to avoid + // situations where two tablets are master at the DB level but not at the vitess level + if err := agent.MysqlDaemon.SetReadOnly(false); err != nil { return "", err } + // We only update agent's masterTermStartTime if we were able to update the topo. // This ensures that in case of a failure, we are never in a situation where the // tablet's timestamp is ahead of the topo's timestamp. From 0fa8c20296c14b7d1f33e94776ecc8661f67665a Mon Sep 17 00:00:00 2001 From: deepthi Date: Thu, 9 Apr 2020 14:47:11 -0700 Subject: [PATCH 2/6] PlannedReparent: Rename PromoteSlave -> PromoteReplica. Deprecate PromoteSlaveWhenCaughtUp RPC Signed-off-by: deepthi --- .../fakemysqldaemon/fakemysqldaemon.go | 10 +- go/vt/mysqlctl/mysql_daemon.go | 5 +- go/vt/mysqlctl/reparent.go | 4 +- .../tabletmanagerdata/tabletmanagerdata.pb.go | 103 ++++++++-- .../tabletmanagerservice.pb.go | 176 +++++++++++------- go/vt/vtcombo/tablet_map.go | 3 + go/vt/vttablet/agentrpctest/test_agent_rpc.go | 21 ++- go/vt/vttablet/faketmclient/fake_client.go | 5 + go/vt/vttablet/grpctmclient/client.go | 16 ++ go/vt/vttablet/grpctmserver/server.go | 13 ++ go/vt/vttablet/tabletmanager/rpc_agent.go | 4 + .../vttablet/tabletmanager/rpc_replication.go | 60 ++++-- go/vt/vttablet/tmclient/rpc_client_api.go | 5 + go/vt/wrangler/reparent.go | 46 +++-- .../testlib/emergency_reparent_shard_test.go | 2 +- .../testlib/planned_reparent_shard_test.go | 10 +- proto/tabletmanagerdata.proto | 11 ++ proto/tabletmanagerservice.proto | 5 + 18 files changed, 373 insertions(+), 126 deletions(-) diff --git a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go index da291915eec..c31b5861899 100644 --- a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go +++ b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go @@ -104,8 +104,8 @@ type FakeMysqlDaemon struct { // same it returns nil, if different it returns an error WaitMasterPosition mysql.Position - // PromoteSlaveResult is returned by PromoteSlave - PromoteSlaveResult mysql.Position + // PromoteResult is returned by Promote + PromoteResult mysql.Position // SchemaFunc provides the return value for GetSchema. // If not defined, the "Schema" field will be used instead, see below. @@ -348,9 +348,9 @@ func (fmd *FakeMysqlDaemon) WaitMasterPos(_ context.Context, pos mysql.Position) return fmt.Errorf("wrong input for WaitMasterPos: expected %v got %v", fmd.WaitMasterPosition, pos) } -// PromoteSlave is part of the MysqlDaemon interface -func (fmd *FakeMysqlDaemon) PromoteSlave(hookExtraEnv map[string]string) (mysql.Position, error) { - return fmd.PromoteSlaveResult, nil +// Promote is part of the MysqlDaemon interface +func (fmd *FakeMysqlDaemon) Promote(hookExtraEnv map[string]string) (mysql.Position, error) { + return fmd.PromoteResult, nil } // ExecuteSuperQueryList is part of the MysqlDaemon interface diff --git a/go/vt/mysqlctl/mysql_daemon.go b/go/vt/mysqlctl/mysql_daemon.go index f205527ba97..94e439b8c0e 100644 --- a/go/vt/mysqlctl/mysql_daemon.go +++ b/go/vt/mysqlctl/mysql_daemon.go @@ -65,10 +65,9 @@ type MysqlDaemon interface { WaitMasterPos(context.Context, mysql.Position) error - // PromoteSlave makes the slave the new master. It will not change + // Promote makes the current server master. It will not change // the read_only state of the server. - PromoteSlave(map[string]string) (mysql.Position, error) - + Promote(map[string]string) (mysql.Position, error) // Schema related methods GetSchema(dbName string, tables, excludeTables []string, includeViews bool) (*tabletmanagerdatapb.SchemaDefinition, error) GetColumns(dbName, table string) ([]*querypb.Field, []string, error) diff --git a/go/vt/mysqlctl/reparent.go b/go/vt/mysqlctl/reparent.go index ce865b4e12d..7d4d5b0bafc 100644 --- a/go/vt/mysqlctl/reparent.go +++ b/go/vt/mysqlctl/reparent.go @@ -97,8 +97,8 @@ func (mysqld *Mysqld) DemoteMaster() (rp mysql.Position, err error) { return mysqld.MasterPosition() } -// PromoteSlave will promote a slave to be the new master. -func (mysqld *Mysqld) PromoteSlave(hookExtraEnv map[string]string) (mysql.Position, error) { +// Promote will promote this server to be the new master. +func (mysqld *Mysqld) Promote(hookExtraEnv map[string]string) (mysql.Position, error) { ctx := context.TODO() conn, err := getPoolReconnect(ctx, mysqld.dbaPool) if err != nil { diff --git a/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go b/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go index 52c75655d2e..cd40e4c5918 100644 --- a/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go +++ b/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go @@ -3220,6 +3220,7 @@ func (m *UndoDemoteMasterResponse) XXX_DiscardUnknown() { var xxx_messageInfo_UndoDemoteMasterResponse proto.InternalMessageInfo +// Deprecated type PromoteSlaveWhenCaughtUpRequest struct { Position string `protobuf:"bytes,1,opt,name=position,proto3" json:"position,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -3259,6 +3260,7 @@ func (m *PromoteSlaveWhenCaughtUpRequest) GetPosition() string { return "" } +// Deprecated type PromoteSlaveWhenCaughtUpResponse struct { Position string `protobuf:"bytes,1,opt,name=position,proto3" json:"position,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -3595,6 +3597,7 @@ func (m *StopReplicationAndGetStatusResponse) GetStatus() *replicationdata.Statu return nil } +// Deprecated type PromoteSlaveRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -3626,6 +3629,7 @@ func (m *PromoteSlaveRequest) XXX_DiscardUnknown() { var xxx_messageInfo_PromoteSlaveRequest proto.InternalMessageInfo +// Deprecated type PromoteSlaveResponse struct { Position string `protobuf:"bytes,1,opt,name=position,proto3" json:"position,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -3665,6 +3669,76 @@ func (m *PromoteSlaveResponse) GetPosition() string { return "" } +type PromoteReplicaRequest struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PromoteReplicaRequest) Reset() { *m = PromoteReplicaRequest{} } +func (m *PromoteReplicaRequest) String() string { return proto.CompactTextString(m) } +func (*PromoteReplicaRequest) ProtoMessage() {} +func (*PromoteReplicaRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_ff9ac4f89e61ffa4, []int{92} +} + +func (m *PromoteReplicaRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PromoteReplicaRequest.Unmarshal(m, b) +} +func (m *PromoteReplicaRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PromoteReplicaRequest.Marshal(b, m, deterministic) +} +func (m *PromoteReplicaRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_PromoteReplicaRequest.Merge(m, src) +} +func (m *PromoteReplicaRequest) XXX_Size() int { + return xxx_messageInfo_PromoteReplicaRequest.Size(m) +} +func (m *PromoteReplicaRequest) XXX_DiscardUnknown() { + xxx_messageInfo_PromoteReplicaRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_PromoteReplicaRequest proto.InternalMessageInfo + +type PromoteReplicaResponse struct { + Position string `protobuf:"bytes,1,opt,name=position,proto3" json:"position,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PromoteReplicaResponse) Reset() { *m = PromoteReplicaResponse{} } +func (m *PromoteReplicaResponse) String() string { return proto.CompactTextString(m) } +func (*PromoteReplicaResponse) ProtoMessage() {} +func (*PromoteReplicaResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_ff9ac4f89e61ffa4, []int{93} +} + +func (m *PromoteReplicaResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PromoteReplicaResponse.Unmarshal(m, b) +} +func (m *PromoteReplicaResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PromoteReplicaResponse.Marshal(b, m, deterministic) +} +func (m *PromoteReplicaResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_PromoteReplicaResponse.Merge(m, src) +} +func (m *PromoteReplicaResponse) XXX_Size() int { + return xxx_messageInfo_PromoteReplicaResponse.Size(m) +} +func (m *PromoteReplicaResponse) XXX_DiscardUnknown() { + xxx_messageInfo_PromoteReplicaResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_PromoteReplicaResponse proto.InternalMessageInfo + +func (m *PromoteReplicaResponse) GetPosition() string { + if m != nil { + return m.Position + } + return "" +} + type BackupRequest struct { Concurrency int64 `protobuf:"varint,1,opt,name=concurrency,proto3" json:"concurrency,omitempty"` AllowMaster bool `protobuf:"varint,2,opt,name=allowMaster,proto3" json:"allowMaster,omitempty"` @@ -3677,7 +3751,7 @@ func (m *BackupRequest) Reset() { *m = BackupRequest{} } func (m *BackupRequest) String() string { return proto.CompactTextString(m) } func (*BackupRequest) ProtoMessage() {} func (*BackupRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ff9ac4f89e61ffa4, []int{92} + return fileDescriptor_ff9ac4f89e61ffa4, []int{94} } func (m *BackupRequest) XXX_Unmarshal(b []byte) error { @@ -3723,7 +3797,7 @@ func (m *BackupResponse) Reset() { *m = BackupResponse{} } func (m *BackupResponse) String() string { return proto.CompactTextString(m) } func (*BackupResponse) ProtoMessage() {} func (*BackupResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ff9ac4f89e61ffa4, []int{93} + return fileDescriptor_ff9ac4f89e61ffa4, []int{95} } func (m *BackupResponse) XXX_Unmarshal(b []byte) error { @@ -3761,7 +3835,7 @@ func (m *RestoreFromBackupRequest) Reset() { *m = RestoreFromBackupReque func (m *RestoreFromBackupRequest) String() string { return proto.CompactTextString(m) } func (*RestoreFromBackupRequest) ProtoMessage() {} func (*RestoreFromBackupRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ff9ac4f89e61ffa4, []int{94} + return fileDescriptor_ff9ac4f89e61ffa4, []int{96} } func (m *RestoreFromBackupRequest) XXX_Unmarshal(b []byte) error { @@ -3793,7 +3867,7 @@ func (m *RestoreFromBackupResponse) Reset() { *m = RestoreFromBackupResp func (m *RestoreFromBackupResponse) String() string { return proto.CompactTextString(m) } func (*RestoreFromBackupResponse) ProtoMessage() {} func (*RestoreFromBackupResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ff9ac4f89e61ffa4, []int{95} + return fileDescriptor_ff9ac4f89e61ffa4, []int{97} } func (m *RestoreFromBackupResponse) XXX_Unmarshal(b []byte) error { @@ -3917,6 +3991,8 @@ func init() { proto.RegisterType((*StopReplicationAndGetStatusResponse)(nil), "tabletmanagerdata.StopReplicationAndGetStatusResponse") proto.RegisterType((*PromoteSlaveRequest)(nil), "tabletmanagerdata.PromoteSlaveRequest") proto.RegisterType((*PromoteSlaveResponse)(nil), "tabletmanagerdata.PromoteSlaveResponse") + proto.RegisterType((*PromoteReplicaRequest)(nil), "tabletmanagerdata.PromoteReplicaRequest") + proto.RegisterType((*PromoteReplicaResponse)(nil), "tabletmanagerdata.PromoteReplicaResponse") proto.RegisterType((*BackupRequest)(nil), "tabletmanagerdata.BackupRequest") proto.RegisterType((*BackupResponse)(nil), "tabletmanagerdata.BackupResponse") proto.RegisterType((*RestoreFromBackupRequest)(nil), "tabletmanagerdata.RestoreFromBackupRequest") @@ -3926,7 +4002,7 @@ func init() { func init() { proto.RegisterFile("tabletmanagerdata.proto", fileDescriptor_ff9ac4f89e61ffa4) } var fileDescriptor_ff9ac4f89e61ffa4 = []byte{ - // 2108 bytes of a gzipped FileDescriptorProto + // 2121 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x59, 0x5b, 0x6f, 0x1b, 0xc7, 0x15, 0x06, 0x49, 0x49, 0xa6, 0x0e, 0x2f, 0x22, 0x97, 0x94, 0x48, 0xc9, 0x8d, 0x2e, 0x6b, 0xa7, 0x51, 0x5d, 0x94, 0x4a, 0x94, 0x34, 0x08, 0x52, 0xa4, 0xa8, 0xac, 0x8b, 0xed, 0x44, 0x89, 0x95, @@ -4051,12 +4127,13 @@ var fileDescriptor_ff9ac4f89e61ffa4 = []byte{ 0x2f, 0xc6, 0xde, 0x3a, 0x58, 0x0a, 0x19, 0x8f, 0x81, 0xbb, 0x39, 0x26, 0xbe, 0x1f, 0x67, 0x88, 0xd2, 0x7a, 0x1e, 0x83, 0x2d, 0x06, 0xab, 0xd1, 0x8d, 0xf6, 0x42, 0x4f, 0x34, 0xf1, 0xd4, 0xa6, 0xf3, 0x1a, 0x1e, 0xdd, 0xc8, 0x75, 0xdf, 0xcd, 0x67, 0x19, 0x6a, 0x66, 0xba, 0x18, 0xf9, 0x9e, - 0x86, 0x6f, 0x91, 0x39, 0xa7, 0x50, 0x7a, 0x8a, 0x3a, 0x17, 0xf1, 0x28, 0x4d, 0x37, 0xa1, 0xd0, - 0x21, 0x61, 0x27, 0xa6, 0x14, 0x87, 0x9d, 0xa1, 0x6e, 0x6a, 0x26, 0x24, 0x38, 0xe4, 0x27, 0x9f, - 0x0a, 0xbd, 0xfe, 0x4e, 0x34, 0x21, 0xfb, 0x4b, 0x28, 0x27, 0x42, 0xb5, 0x09, 0x8f, 0x61, 0x1e, - 0x0f, 0xc6, 0xa1, 0x2f, 0xb7, 0x92, 0x7f, 0x7a, 0x1c, 0x0a, 0xd4, 0x51, 0x44, 0x3d, 0xc2, 0x38, - 0xa1, 0xf8, 0x88, 0x92, 0x7e, 0xca, 0x2e, 0x7b, 0x0f, 0x56, 0x67, 0xd0, 0xee, 0x22, 0xfe, 0xe9, - 0xa7, 0x3f, 0xb5, 0x06, 0x3e, 0xc7, 0x8c, 0xb5, 0x7c, 0xb2, 0xa3, 0x7e, 0xed, 0x74, 0xc9, 0xce, - 0x80, 0xef, 0xc8, 0x7f, 0xbd, 0xec, 0x4c, 0x7d, 0xab, 0xb5, 0x17, 0x24, 0xe1, 0xf3, 0xff, 0x05, - 0x00, 0x00, 0xff, 0xff, 0x05, 0x80, 0x1f, 0x06, 0x04, 0x1a, 0x00, 0x00, + 0x86, 0x6f, 0x91, 0x39, 0x0d, 0x58, 0xd6, 0x77, 0xb4, 0x95, 0xc6, 0x12, 0x35, 0x49, 0xb8, 0x85, + 0xb8, 0x53, 0x28, 0x3d, 0x45, 0x9d, 0x8b, 0x78, 0x94, 0xf5, 0x9b, 0x50, 0xe8, 0x90, 0xb0, 0x13, + 0x53, 0x8a, 0xc3, 0xce, 0x50, 0xf7, 0x48, 0x13, 0x12, 0x1c, 0xf2, 0x0b, 0x52, 0xbd, 0xa4, 0xfe, + 0xec, 0x34, 0x21, 0xfb, 0x4b, 0x28, 0x27, 0x42, 0xb5, 0x09, 0x8f, 0x61, 0x1e, 0x0f, 0xc6, 0x2f, + 0x59, 0x6e, 0x25, 0xff, 0x43, 0x39, 0x14, 0xa8, 0xa3, 0x88, 0x7a, 0x22, 0x72, 0x42, 0xf1, 0x11, + 0x25, 0xfd, 0x94, 0x5d, 0xf6, 0x1e, 0xac, 0xce, 0xa0, 0xdd, 0x45, 0xfc, 0xd3, 0x4f, 0x7f, 0x6a, + 0x0d, 0x7c, 0x8e, 0x19, 0x6b, 0xf9, 0x64, 0x47, 0xfd, 0xda, 0xe9, 0x92, 0x9d, 0x01, 0xdf, 0x91, + 0xff, 0xc9, 0xd9, 0x99, 0xfa, 0xf4, 0x6b, 0x2f, 0x48, 0xc2, 0xe7, 0xff, 0x0b, 0x00, 0x00, 0xff, + 0xff, 0x4b, 0xd6, 0x65, 0xc3, 0x53, 0x1a, 0x00, 0x00, } diff --git a/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go b/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go index 1f494400a18..44bf80b1a70 100644 --- a/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go +++ b/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go @@ -29,73 +29,73 @@ const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package func init() { proto.RegisterFile("tabletmanagerservice.proto", fileDescriptor_9ee75fe63cfd9360) } var fileDescriptor_9ee75fe63cfd9360 = []byte{ - // 1041 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x98, 0x6d, 0x6f, 0x1b, 0x45, - 0x10, 0xc7, 0xb1, 0x04, 0x95, 0x58, 0x1e, 0xbb, 0xaa, 0x28, 0x0a, 0x12, 0x4f, 0x6d, 0x78, 0x48, - 0x51, 0xdc, 0x34, 0x94, 0xf7, 0x6e, 0x9a, 0xb4, 0x41, 0x8d, 0x30, 0x76, 0x43, 0x10, 0x48, 0x48, - 0x1b, 0x7b, 0x62, 0x1f, 0x39, 0xef, 0x1e, 0xbb, 0x7b, 0x56, 0xf3, 0x0a, 0x09, 0x89, 0x57, 0x48, - 0x7c, 0x11, 0xbe, 0x24, 0xba, 0x87, 0xdd, 0x9b, 0x3d, 0xcf, 0xad, 0xed, 0x77, 0x91, 0xff, 0xbf, - 0x99, 0xd9, 0x87, 0x99, 0xd9, 0xc9, 0xb1, 0x1d, 0x2b, 0x2e, 0x53, 0xb0, 0x0b, 0x21, 0xc5, 0x0c, - 0xb4, 0x01, 0xbd, 0x4c, 0x26, 0xb0, 0x9f, 0x69, 0x65, 0x15, 0xbf, 0x43, 0x69, 0x3b, 0x77, 0x83, - 0x5f, 0xa7, 0xc2, 0x8a, 0x0a, 0x7f, 0xf4, 0xdf, 0x2e, 0x7b, 0xe7, 0x65, 0xa9, 0x9d, 0x55, 0x1a, - 0x3f, 0x65, 0xaf, 0x0f, 0x13, 0x39, 0xe3, 0x1f, 0xef, 0xaf, 0xda, 0x14, 0xc2, 0x08, 0xfe, 0xc8, - 0xc1, 0xd8, 0x9d, 0x4f, 0x3a, 0x75, 0x93, 0x29, 0x69, 0xe0, 0xf3, 0xd7, 0xf8, 0x0b, 0xf6, 0xc6, - 0x38, 0x05, 0xc8, 0x38, 0xc5, 0x96, 0x8a, 0x73, 0xf6, 0x69, 0x37, 0xe0, 0xbd, 0xfd, 0xc6, 0xde, - 0x3a, 0x7e, 0x05, 0x93, 0xdc, 0xc2, 0x73, 0xa5, 0xae, 0xf9, 0x2e, 0x61, 0x82, 0x74, 0xe7, 0xf9, - 0x8b, 0x75, 0x98, 0xf7, 0xff, 0x33, 0x7b, 0xf3, 0x19, 0xd8, 0xf1, 0x64, 0x0e, 0x0b, 0xc1, 0xef, - 0x11, 0x66, 0x5e, 0x75, 0xbe, 0xef, 0xc7, 0x21, 0xef, 0x79, 0xc6, 0xde, 0x7d, 0x06, 0x76, 0x08, - 0x7a, 0x91, 0x18, 0x93, 0x28, 0x69, 0xf8, 0x57, 0xb4, 0x25, 0x42, 0x5c, 0x8c, 0xaf, 0x37, 0x20, - 0xf1, 0x11, 0x8d, 0xc1, 0x8e, 0x40, 0x4c, 0x7f, 0x90, 0xe9, 0x0d, 0x79, 0x44, 0x48, 0x8f, 0x1d, - 0x51, 0x80, 0x79, 0xff, 0x82, 0xbd, 0x5d, 0x0b, 0x17, 0x3a, 0xb1, 0xc0, 0x23, 0x96, 0x25, 0xe0, - 0x22, 0x7c, 0xb9, 0x96, 0xf3, 0x21, 0x7e, 0x65, 0xec, 0x68, 0x2e, 0xe4, 0x0c, 0x5e, 0xde, 0x64, - 0xc0, 0xa9, 0x13, 0x6e, 0x64, 0xe7, 0x7e, 0x77, 0x0d, 0x85, 0xd7, 0x3f, 0x82, 0x2b, 0x0d, 0x66, - 0x3e, 0xb6, 0xa2, 0x63, 0xfd, 0x18, 0x88, 0xad, 0x3f, 0xe4, 0xf0, 0x5d, 0x8f, 0x72, 0xf9, 0x1c, - 0x44, 0x6a, 0xe7, 0x47, 0x73, 0x98, 0x5c, 0x93, 0x77, 0x1d, 0x22, 0xb1, 0xbb, 0x6e, 0x93, 0x3e, - 0x50, 0xc6, 0x6e, 0x9f, 0xce, 0xa4, 0xd2, 0x50, 0xc9, 0xc7, 0x5a, 0x2b, 0xcd, 0x1f, 0x10, 0x1e, - 0x56, 0x28, 0x17, 0xee, 0x9b, 0xcd, 0xe0, 0xf0, 0xf4, 0x52, 0x25, 0xa6, 0x75, 0x8d, 0xd0, 0xa7, - 0xd7, 0x00, 0xf1, 0xd3, 0xc3, 0x9c, 0x0f, 0xf1, 0x3b, 0x7b, 0x6f, 0xa8, 0xe1, 0x2a, 0x4d, 0x66, - 0x73, 0x57, 0x89, 0xd4, 0xa1, 0xb4, 0x18, 0x17, 0x68, 0x6f, 0x13, 0x14, 0x17, 0xcb, 0x20, 0xcb, - 0xd2, 0x9b, 0x3a, 0x0e, 0x95, 0x44, 0x48, 0x8f, 0x15, 0x4b, 0x80, 0xe1, 0x4c, 0x7e, 0xa1, 0x26, - 0xd7, 0x65, 0x77, 0x35, 0x64, 0x26, 0x37, 0x72, 0x2c, 0x93, 0x31, 0x85, 0xef, 0xe2, 0x5c, 0xa6, - 0x8d, 0x7b, 0x6a, 0x59, 0x18, 0x88, 0xdd, 0x45, 0xc8, 0xe1, 0x04, 0xab, 0x1b, 0xe5, 0x09, 0xd8, - 0xc9, 0x7c, 0x60, 0x9e, 0x5e, 0x0a, 0x32, 0xc1, 0x56, 0xa8, 0x58, 0x82, 0x11, 0xb0, 0x8f, 0xf8, - 0x27, 0xfb, 0x20, 0x94, 0x07, 0x69, 0x3a, 0xd4, 0xc9, 0xd2, 0xf0, 0x87, 0x6b, 0x3d, 0x39, 0xd4, - 0xc5, 0x3e, 0xd8, 0xc2, 0xa2, 0x7b, 0xcb, 0x83, 0x2c, 0xdb, 0x60, 0xcb, 0x83, 0x2c, 0xdb, 0x7c, - 0xcb, 0x25, 0x1c, 0x74, 0xec, 0x54, 0x2c, 0xa1, 0x68, 0x23, 0xb9, 0xa1, 0x3b, 0x76, 0xa3, 0x47, - 0x3b, 0x36, 0xc6, 0x70, 0x3b, 0x3a, 0x13, 0xc6, 0x82, 0x1e, 0x2a, 0x93, 0xd8, 0x44, 0x49, 0xb2, - 0x1d, 0x85, 0x48, 0xac, 0x1d, 0xb5, 0x49, 0x5c, 0xb9, 0x17, 0x22, 0xb1, 0x27, 0xaa, 0x89, 0x44, - 0xd9, 0xb7, 0x98, 0x58, 0xe5, 0xae, 0xa0, 0xf8, 0xa5, 0x1e, 0x5b, 0x95, 0x95, 0x3b, 0x26, 0x5f, - 0x6a, 0xaf, 0xc6, 0x5e, 0x6a, 0x04, 0x79, 0xcf, 0x0b, 0xf6, 0xbe, 0xff, 0xf9, 0x2c, 0x91, 0xc9, - 0x22, 0x5f, 0xf0, 0xbd, 0x98, 0x6d, 0x0d, 0xb9, 0x38, 0x0f, 0x36, 0x62, 0x71, 0x8b, 0x18, 0x5b, - 0xa1, 0x6d, 0xb5, 0x13, 0x7a, 0x91, 0x4e, 0x8e, 0xb5, 0x08, 0x4c, 0x79, 0xe7, 0x37, 0xec, 0x4e, - 0xf3, 0xfb, 0xb9, 0xb4, 0x49, 0x3a, 0xb8, 0xb2, 0xa0, 0xf9, 0x7e, 0xd4, 0x41, 0x03, 0xba, 0x80, - 0xfd, 0x8d, 0x79, 0x1f, 0xfa, 0x9f, 0x1e, 0xdb, 0xa9, 0xa6, 0xca, 0xe3, 0x57, 0x16, 0xb4, 0x14, - 0x69, 0x31, 0x46, 0x64, 0x42, 0x83, 0xb4, 0x30, 0xe5, 0xdf, 0x12, 0x1e, 0xbb, 0x71, 0xb7, 0x8e, - 0xc7, 0x5b, 0x5a, 0xf9, 0xd5, 0xfc, 0xd5, 0x63, 0x77, 0xdb, 0xe0, 0x71, 0x0a, 0x93, 0x62, 0x29, - 0x07, 0x1b, 0x38, 0xad, 0x59, 0xb7, 0x8e, 0x47, 0xdb, 0x98, 0xb4, 0xa7, 0xcb, 0xe2, 0xc8, 0x4c, - 0xe7, 0x74, 0x59, 0xaa, 0xeb, 0xa6, 0xcb, 0x1a, 0xc2, 0x39, 0xfb, 0xd3, 0x08, 0xb2, 0x34, 0x99, - 0x88, 0xa2, 0x4e, 0x8a, 0x6e, 0x43, 0xe6, 0x6c, 0x1b, 0x8a, 0xe5, 0xec, 0x2a, 0x8b, 0x9b, 0x34, - 0x56, 0x9b, 0x2a, 0x25, 0x9b, 0x34, 0x8d, 0xc6, 0x9a, 0x74, 0x97, 0x05, 0xde, 0xef, 0x08, 0x4c, - 0x31, 0x3d, 0x7a, 0x8e, 0xdc, 0x6f, 0x1b, 0x8a, 0xed, 0x77, 0x95, 0xc5, 0x35, 0x7a, 0x2a, 0x13, - 0x5b, 0x35, 0x3e, 0xb2, 0x46, 0x1b, 0x39, 0x56, 0xa3, 0x98, 0x0a, 0x52, 0x73, 0xa8, 0xb2, 0x3c, - 0x2d, 0x87, 0xc8, 0x2a, 0x77, 0xbf, 0x57, 0x79, 0x91, 0x44, 0x64, 0x6a, 0x76, 0xb0, 0xb1, 0xd4, - 0xec, 0x34, 0xc1, 0xa9, 0x59, 0x2c, 0xae, 0xbb, 0x9d, 0x7a, 0x35, 0x96, 0x9a, 0x08, 0xc2, 0x53, - 0xca, 0x53, 0x58, 0x28, 0x0b, 0xf5, 0xe9, 0x51, 0xef, 0x16, 0x06, 0x62, 0x53, 0x4a, 0xc8, 0xe1, - 0x6c, 0x38, 0x97, 0x53, 0x15, 0x84, 0xd9, 0x23, 0x87, 0x9c, 0x10, 0x8a, 0x65, 0xc3, 0x2a, 0xeb, - 0xc3, 0xfd, 0xdd, 0x63, 0x1f, 0x0e, 0xb5, 0x2a, 0xb4, 0x72, 0xb3, 0x17, 0x73, 0x90, 0x47, 0x22, - 0x9f, 0xcd, 0xed, 0x79, 0xc6, 0xc9, 0xe3, 0xef, 0x80, 0x5d, 0xfc, 0xc3, 0xad, 0x6c, 0x82, 0x87, - 0xaa, 0x94, 0x85, 0xa9, 0xe9, 0x29, 0xfd, 0x50, 0xb5, 0xa0, 0xe8, 0x43, 0xb5, 0xc2, 0x06, 0x2f, - 0x2e, 0xb8, 0x1a, 0xb8, 0x47, 0xff, 0x37, 0x17, 0x9e, 0xeb, 0xfd, 0x38, 0x84, 0x47, 0x2e, 0x17, - 0x77, 0x04, 0xa6, 0x78, 0x56, 0x60, 0xca, 0x63, 0xab, 0xf3, 0x54, 0x6c, 0xe4, 0x22, 0x60, 0x1f, - 0xf1, 0xdf, 0x1e, 0xfb, 0xa8, 0x78, 0x93, 0x51, 0xb9, 0x0f, 0xe4, 0xb4, 0xe8, 0xac, 0xd5, 0x0c, - 0xf6, 0xb8, 0xe3, 0x0d, 0xef, 0xe0, 0xdd, 0x32, 0xbe, 0xdb, 0xd6, 0x0c, 0x57, 0x09, 0xbe, 0x71, - 0xb2, 0x4a, 0x30, 0x10, 0xab, 0x92, 0x90, 0xf3, 0x21, 0x7e, 0x64, 0xb7, 0x9e, 0x88, 0xc9, 0x75, - 0x9e, 0x71, 0xea, 0x4b, 0x4b, 0x25, 0x39, 0xb7, 0x9f, 0x45, 0x08, 0xe7, 0xf0, 0x61, 0x8f, 0x6b, - 0x76, 0xbb, 0x38, 0x5d, 0xa5, 0xe1, 0x44, 0xab, 0x45, 0xed, 0xbd, 0xa3, 0xb7, 0x86, 0x54, 0xec, - 0xe2, 0x08, 0xb8, 0x89, 0xf9, 0xe4, 0xf0, 0x97, 0x83, 0x65, 0x62, 0xc1, 0x98, 0xfd, 0x44, 0xf5, - 0xab, 0xbf, 0xfa, 0x33, 0xd5, 0x5f, 0xda, 0x7e, 0xf9, 0x35, 0xab, 0x4f, 0x7d, 0xfb, 0xba, 0xbc, - 0x55, 0x6a, 0x87, 0xff, 0x07, 0x00, 0x00, 0xff, 0xff, 0xcf, 0xa2, 0x81, 0x89, 0x36, 0x13, 0x00, - 0x00, + // 1055 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x98, 0x5b, 0x6f, 0x5b, 0x45, + 0x10, 0xc7, 0xb1, 0x04, 0x95, 0x58, 0xae, 0x5d, 0x55, 0x14, 0x05, 0x89, 0x5b, 0x2f, 0x40, 0x8a, + 0xe2, 0xa6, 0xa1, 0xbc, 0xbb, 0x69, 0xd2, 0x06, 0x35, 0xc2, 0xd8, 0x0d, 0x41, 0x20, 0x21, 0x6d, + 0xec, 0x89, 0x7d, 0xc8, 0xf1, 0xee, 0x61, 0x77, 0x8f, 0xd5, 0x3c, 0x21, 0x21, 0xf1, 0x84, 0xc4, + 0x13, 0x1f, 0xb8, 0x3a, 0x97, 0xdd, 0x33, 0x7b, 0x3c, 0x67, 0x6d, 0xbf, 0x45, 0xfe, 0xff, 0x66, + 0x66, 0x2f, 0x33, 0xb3, 0x93, 0xc3, 0x76, 0xac, 0xb8, 0x48, 0xc1, 0x2e, 0x84, 0x14, 0x33, 0xd0, + 0x06, 0xf4, 0x32, 0x99, 0xc0, 0x5e, 0xa6, 0x95, 0x55, 0xfc, 0x16, 0xa5, 0xed, 0xdc, 0x0e, 0x7e, + 0x9d, 0x0a, 0x2b, 0x2a, 0xfc, 0xd1, 0xff, 0xf7, 0xd9, 0x7b, 0x2f, 0x4b, 0xed, 0xb4, 0xd2, 0xf8, + 0x09, 0x7b, 0x73, 0x98, 0xc8, 0x19, 0xff, 0x74, 0x6f, 0xd5, 0xa6, 0x10, 0x46, 0xf0, 0x67, 0x0e, + 0xc6, 0xee, 0x7c, 0xd6, 0xa9, 0x9b, 0x4c, 0x49, 0x03, 0x5f, 0xbe, 0xc1, 0x5f, 0xb0, 0xb7, 0xc6, + 0x29, 0x40, 0xc6, 0x29, 0xb6, 0x54, 0x9c, 0xb3, 0xcf, 0xbb, 0x01, 0xef, 0xed, 0x77, 0xf6, 0xce, + 0xd1, 0x2b, 0x98, 0xe4, 0x16, 0x9e, 0x2b, 0x75, 0xc5, 0xef, 0x11, 0x26, 0x48, 0x77, 0x9e, 0xef, + 0xaf, 0xc3, 0xbc, 0xff, 0x5f, 0xd8, 0xdb, 0xcf, 0xc0, 0x8e, 0x27, 0x73, 0x58, 0x08, 0x7e, 0x87, + 0x30, 0xf3, 0xaa, 0xf3, 0x7d, 0x37, 0x0e, 0x79, 0xcf, 0x33, 0xf6, 0xfe, 0x33, 0xb0, 0x43, 0xd0, + 0x8b, 0xc4, 0x98, 0x44, 0x49, 0xc3, 0xbf, 0xa6, 0x2d, 0x11, 0xe2, 0x62, 0x7c, 0xb3, 0x01, 0x89, + 0x8f, 0x68, 0x0c, 0x76, 0x04, 0x62, 0xfa, 0xa3, 0x4c, 0xaf, 0xc9, 0x23, 0x42, 0x7a, 0xec, 0x88, + 0x02, 0xcc, 0xfb, 0x17, 0xec, 0xdd, 0x5a, 0x38, 0xd7, 0x89, 0x05, 0x1e, 0xb1, 0x2c, 0x01, 0x17, + 0xe1, 0xab, 0xb5, 0x9c, 0x0f, 0xf1, 0x1b, 0x63, 0x87, 0x73, 0x21, 0x67, 0xf0, 0xf2, 0x3a, 0x03, + 0x4e, 0x9d, 0x70, 0x23, 0x3b, 0xf7, 0xf7, 0xd6, 0x50, 0x78, 0xfd, 0x23, 0xb8, 0xd4, 0x60, 0xe6, + 0x63, 0x2b, 0x3a, 0xd6, 0x8f, 0x81, 0xd8, 0xfa, 0x43, 0x0e, 0xdf, 0xf5, 0x28, 0x97, 0xcf, 0x41, + 0xa4, 0x76, 0x7e, 0x38, 0x87, 0xc9, 0x15, 0x79, 0xd7, 0x21, 0x12, 0xbb, 0xeb, 0x36, 0xe9, 0x03, + 0x65, 0xec, 0xe6, 0xc9, 0x4c, 0x2a, 0x0d, 0x95, 0x7c, 0xa4, 0xb5, 0xd2, 0xfc, 0x01, 0xe1, 0x61, + 0x85, 0x72, 0xe1, 0xbe, 0xdd, 0x0c, 0x0e, 0x4f, 0x2f, 0x55, 0x62, 0x5a, 0xd7, 0x08, 0x7d, 0x7a, + 0x0d, 0x10, 0x3f, 0x3d, 0xcc, 0xf9, 0x10, 0x7f, 0xb0, 0x0f, 0x86, 0x1a, 0x2e, 0xd3, 0x64, 0x36, + 0x77, 0x95, 0x48, 0x1d, 0x4a, 0x8b, 0x71, 0x81, 0x76, 0x37, 0x41, 0x71, 0xb1, 0x0c, 0xb2, 0x2c, + 0xbd, 0xae, 0xe3, 0x50, 0x49, 0x84, 0xf4, 0x58, 0xb1, 0x04, 0x18, 0xce, 0xe4, 0x17, 0x6a, 0x72, + 0x55, 0x76, 0x57, 0x43, 0x66, 0x72, 0x23, 0xc7, 0x32, 0x19, 0x53, 0xf8, 0x2e, 0xce, 0x64, 0xda, + 0xb8, 0xa7, 0x96, 0x85, 0x81, 0xd8, 0x5d, 0x84, 0x1c, 0x4e, 0xb0, 0xba, 0x51, 0x1e, 0x83, 0x9d, + 0xcc, 0x07, 0xe6, 0xe9, 0x85, 0x20, 0x13, 0x6c, 0x85, 0x8a, 0x25, 0x18, 0x01, 0xfb, 0x88, 0x7f, + 0xb1, 0x8f, 0x42, 0x79, 0x90, 0xa6, 0x43, 0x9d, 0x2c, 0x0d, 0x7f, 0xb8, 0xd6, 0x93, 0x43, 0x5d, + 0xec, 0xfd, 0x2d, 0x2c, 0xba, 0xb7, 0x3c, 0xc8, 0xb2, 0x0d, 0xb6, 0x3c, 0xc8, 0xb2, 0xcd, 0xb7, + 0x5c, 0xc2, 0x41, 0xc7, 0x4e, 0xc5, 0x12, 0x8a, 0x36, 0x92, 0x1b, 0xba, 0x63, 0x37, 0x7a, 0xb4, + 0x63, 0x63, 0x0c, 0xb7, 0xa3, 0x53, 0x61, 0x2c, 0xe8, 0xa1, 0x32, 0x89, 0x4d, 0x94, 0x24, 0xdb, + 0x51, 0x88, 0xc4, 0xda, 0x51, 0x9b, 0xc4, 0x95, 0x7b, 0x2e, 0x12, 0x7b, 0xac, 0x9a, 0x48, 0x94, + 0x7d, 0x8b, 0x89, 0x55, 0xee, 0x0a, 0x8a, 0x5f, 0xea, 0xb1, 0x55, 0x59, 0xb9, 0x63, 0xf2, 0xa5, + 0xf6, 0x6a, 0xec, 0xa5, 0x46, 0x90, 0xf7, 0xbc, 0x60, 0x1f, 0xfa, 0x9f, 0x4f, 0x13, 0x99, 0x2c, + 0xf2, 0x05, 0xdf, 0x8d, 0xd9, 0xd6, 0x90, 0x8b, 0xf3, 0x60, 0x23, 0x16, 0xb7, 0x88, 0xb1, 0x15, + 0xda, 0x56, 0x3b, 0xa1, 0x17, 0xe9, 0xe4, 0x58, 0x8b, 0xc0, 0x94, 0x77, 0x7e, 0xcd, 0x6e, 0x35, + 0xbf, 0x9f, 0x49, 0x9b, 0xa4, 0x83, 0x4b, 0x0b, 0x9a, 0xef, 0x45, 0x1d, 0x34, 0xa0, 0x0b, 0xd8, + 0xdf, 0x98, 0xf7, 0xa1, 0xff, 0xed, 0xb1, 0x9d, 0x6a, 0xaa, 0x3c, 0x7a, 0x65, 0x41, 0x4b, 0x91, + 0x16, 0x63, 0x44, 0x26, 0x34, 0x48, 0x0b, 0x53, 0xfe, 0x1d, 0xe1, 0xb1, 0x1b, 0x77, 0xeb, 0x78, + 0xbc, 0xa5, 0x95, 0x5f, 0xcd, 0xdf, 0x3d, 0x76, 0xbb, 0x0d, 0x1e, 0xa5, 0x30, 0x29, 0x96, 0xb2, + 0xbf, 0x81, 0xd3, 0x9a, 0x75, 0xeb, 0x78, 0xb4, 0x8d, 0x49, 0x7b, 0xba, 0x2c, 0x8e, 0xcc, 0x74, + 0x4e, 0x97, 0xa5, 0xba, 0x6e, 0xba, 0xac, 0x21, 0x9c, 0xb3, 0x3f, 0x8f, 0x20, 0x4b, 0x93, 0x89, + 0x28, 0xea, 0xa4, 0xe8, 0x36, 0x64, 0xce, 0xb6, 0xa1, 0x58, 0xce, 0xae, 0xb2, 0xb8, 0x49, 0x63, + 0xb5, 0xa9, 0x52, 0xb2, 0x49, 0xd3, 0x68, 0xac, 0x49, 0x77, 0x59, 0xe0, 0xfd, 0x8e, 0xc0, 0x14, + 0xd3, 0xa3, 0xe7, 0xc8, 0xfd, 0xb6, 0xa1, 0xd8, 0x7e, 0x57, 0x59, 0x5c, 0xa3, 0x27, 0x32, 0xb1, + 0x55, 0xe3, 0x23, 0x6b, 0xb4, 0x91, 0x63, 0x35, 0x8a, 0xa9, 0x20, 0x35, 0x87, 0x2a, 0xcb, 0xd3, + 0x72, 0x88, 0xac, 0x72, 0xf7, 0x07, 0x95, 0x17, 0x49, 0x44, 0xa6, 0x66, 0x07, 0x1b, 0x4b, 0xcd, + 0x4e, 0x13, 0x9c, 0x9a, 0xc5, 0xe2, 0xba, 0xdb, 0xa9, 0x57, 0x63, 0xa9, 0x89, 0x20, 0x3c, 0xa5, + 0x3c, 0x85, 0x85, 0xb2, 0x50, 0x9f, 0x1e, 0xf5, 0x6e, 0x61, 0x20, 0x36, 0xa5, 0x84, 0x1c, 0xce, + 0x86, 0x33, 0x39, 0x55, 0x41, 0x98, 0x5d, 0x72, 0xc8, 0x09, 0xa1, 0x58, 0x36, 0xac, 0xb2, 0x3e, + 0xdc, 0x3f, 0x3d, 0xf6, 0xf1, 0x50, 0xab, 0x42, 0x2b, 0x37, 0x7b, 0x3e, 0x07, 0x79, 0x28, 0xf2, + 0xd9, 0xdc, 0x9e, 0x65, 0x9c, 0x3c, 0xfe, 0x0e, 0xd8, 0xc5, 0x3f, 0xd8, 0xca, 0x26, 0x78, 0xa8, + 0x4a, 0x59, 0x98, 0x9a, 0x9e, 0xd2, 0x0f, 0x55, 0x0b, 0x8a, 0x3e, 0x54, 0x2b, 0x6c, 0xf0, 0xe2, + 0x82, 0xab, 0x81, 0x3b, 0xf4, 0x7f, 0x73, 0xe1, 0xb9, 0xde, 0x8d, 0x43, 0x78, 0xe4, 0x72, 0x71, + 0x47, 0x60, 0x8a, 0x67, 0x05, 0xa6, 0x3c, 0xb6, 0x3a, 0x4f, 0xc5, 0x46, 0x2e, 0x02, 0xf6, 0x11, + 0xff, 0xeb, 0xb1, 0x4f, 0x8a, 0x37, 0x19, 0x95, 0xfb, 0x40, 0x4e, 0x8b, 0xce, 0x5a, 0xcd, 0x60, + 0x8f, 0x3b, 0xde, 0xf0, 0x0e, 0xde, 0x2d, 0xe3, 0xfb, 0x6d, 0xcd, 0x70, 0x95, 0xe0, 0x1b, 0x27, + 0xab, 0x04, 0x03, 0xb1, 0x2a, 0x09, 0x39, 0x3c, 0x06, 0xd6, 0x4a, 0xbd, 0x1c, 0x72, 0x0c, 0x0c, + 0x91, 0xd8, 0x18, 0xd8, 0x26, 0x7d, 0xa0, 0x9f, 0xd8, 0x8d, 0x27, 0x62, 0x72, 0x95, 0x67, 0x9c, + 0xfa, 0xa4, 0x53, 0x49, 0xce, 0xf1, 0x17, 0x11, 0xc2, 0x39, 0x7c, 0xd8, 0xe3, 0x9a, 0xdd, 0x2c, + 0xae, 0x51, 0x69, 0x38, 0xd6, 0x6a, 0x51, 0x7b, 0xef, 0x68, 0xe2, 0x21, 0x15, 0xcb, 0x10, 0x02, + 0x6e, 0x62, 0x3e, 0x39, 0xf8, 0x75, 0x7f, 0x99, 0x58, 0x30, 0x66, 0x2f, 0x51, 0xfd, 0xea, 0xaf, + 0xfe, 0x4c, 0xf5, 0x97, 0xb6, 0x5f, 0x7e, 0x36, 0xeb, 0x53, 0x1f, 0xd9, 0x2e, 0x6e, 0x94, 0xda, + 0xc1, 0xeb, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd0, 0x9a, 0xbc, 0x94, 0x9f, 0x13, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -197,6 +197,7 @@ type TabletManagerClient interface { UndoDemoteMaster(ctx context.Context, in *tabletmanagerdata.UndoDemoteMasterRequest, opts ...grpc.CallOption) (*tabletmanagerdata.UndoDemoteMasterResponse, error) // PromoteSlaveWhenCaughtUp tells the remote tablet to catch up, // and then be the master + // Deprecated PromoteSlaveWhenCaughtUp(ctx context.Context, in *tabletmanagerdata.PromoteSlaveWhenCaughtUpRequest, opts ...grpc.CallOption) (*tabletmanagerdata.PromoteSlaveWhenCaughtUpResponse, error) // SlaveWasPromoted tells the remote tablet it is now the master SlaveWasPromoted(ctx context.Context, in *tabletmanagerdata.SlaveWasPromotedRequest, opts ...grpc.CallOption) (*tabletmanagerdata.SlaveWasPromotedResponse, error) @@ -208,7 +209,10 @@ type TabletManagerClient interface { // replication status StopReplicationAndGetStatus(ctx context.Context, in *tabletmanagerdata.StopReplicationAndGetStatusRequest, opts ...grpc.CallOption) (*tabletmanagerdata.StopReplicationAndGetStatusResponse, error) // PromoteSlave makes the slave the new master + // Deprecated PromoteSlave(ctx context.Context, in *tabletmanagerdata.PromoteSlaveRequest, opts ...grpc.CallOption) (*tabletmanagerdata.PromoteSlaveResponse, error) + // PromoteReplica makes the replica the new master + PromoteReplica(ctx context.Context, in *tabletmanagerdata.PromoteReplicaRequest, opts ...grpc.CallOption) (*tabletmanagerdata.PromoteReplicaResponse, error) Backup(ctx context.Context, in *tabletmanagerdata.BackupRequest, opts ...grpc.CallOption) (TabletManager_BackupClient, error) // RestoreFromBackup deletes all local data and restores it from the latest backup. RestoreFromBackup(ctx context.Context, in *tabletmanagerdata.RestoreFromBackupRequest, opts ...grpc.CallOption) (TabletManager_RestoreFromBackupClient, error) @@ -609,6 +613,15 @@ func (c *tabletManagerClient) PromoteSlave(ctx context.Context, in *tabletmanage return out, nil } +func (c *tabletManagerClient) PromoteReplica(ctx context.Context, in *tabletmanagerdata.PromoteReplicaRequest, opts ...grpc.CallOption) (*tabletmanagerdata.PromoteReplicaResponse, error) { + out := new(tabletmanagerdata.PromoteReplicaResponse) + err := c.cc.Invoke(ctx, "/tabletmanagerservice.TabletManager/PromoteReplica", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *tabletManagerClient) Backup(ctx context.Context, in *tabletmanagerdata.BackupRequest, opts ...grpc.CallOption) (TabletManager_BackupClient, error) { stream, err := c.cc.NewStream(ctx, &_TabletManager_serviceDesc.Streams[0], "/tabletmanagerservice.TabletManager/Backup", opts...) if err != nil { @@ -762,6 +775,7 @@ type TabletManagerServer interface { UndoDemoteMaster(context.Context, *tabletmanagerdata.UndoDemoteMasterRequest) (*tabletmanagerdata.UndoDemoteMasterResponse, error) // PromoteSlaveWhenCaughtUp tells the remote tablet to catch up, // and then be the master + // Deprecated PromoteSlaveWhenCaughtUp(context.Context, *tabletmanagerdata.PromoteSlaveWhenCaughtUpRequest) (*tabletmanagerdata.PromoteSlaveWhenCaughtUpResponse, error) // SlaveWasPromoted tells the remote tablet it is now the master SlaveWasPromoted(context.Context, *tabletmanagerdata.SlaveWasPromotedRequest) (*tabletmanagerdata.SlaveWasPromotedResponse, error) @@ -773,7 +787,10 @@ type TabletManagerServer interface { // replication status StopReplicationAndGetStatus(context.Context, *tabletmanagerdata.StopReplicationAndGetStatusRequest) (*tabletmanagerdata.StopReplicationAndGetStatusResponse, error) // PromoteSlave makes the slave the new master + // Deprecated PromoteSlave(context.Context, *tabletmanagerdata.PromoteSlaveRequest) (*tabletmanagerdata.PromoteSlaveResponse, error) + // PromoteReplica makes the replica the new master + PromoteReplica(context.Context, *tabletmanagerdata.PromoteReplicaRequest) (*tabletmanagerdata.PromoteReplicaResponse, error) Backup(*tabletmanagerdata.BackupRequest, TabletManager_BackupServer) error // RestoreFromBackup deletes all local data and restores it from the latest backup. RestoreFromBackup(*tabletmanagerdata.RestoreFromBackupRequest, TabletManager_RestoreFromBackupServer) error @@ -912,6 +929,9 @@ func (*UnimplementedTabletManagerServer) StopReplicationAndGetStatus(ctx context func (*UnimplementedTabletManagerServer) PromoteSlave(ctx context.Context, req *tabletmanagerdata.PromoteSlaveRequest) (*tabletmanagerdata.PromoteSlaveResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method PromoteSlave not implemented") } +func (*UnimplementedTabletManagerServer) PromoteReplica(ctx context.Context, req *tabletmanagerdata.PromoteReplicaRequest) (*tabletmanagerdata.PromoteReplicaResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PromoteReplica not implemented") +} func (*UnimplementedTabletManagerServer) Backup(req *tabletmanagerdata.BackupRequest, srv TabletManager_BackupServer) error { return status.Errorf(codes.Unimplemented, "method Backup not implemented") } @@ -1697,6 +1717,24 @@ func _TabletManager_PromoteSlave_Handler(srv interface{}, ctx context.Context, d return interceptor(ctx, in, info, handler) } +func _TabletManager_PromoteReplica_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(tabletmanagerdata.PromoteReplicaRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TabletManagerServer).PromoteReplica(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/tabletmanagerservice.TabletManager/PromoteReplica", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TabletManagerServer).PromoteReplica(ctx, req.(*tabletmanagerdata.PromoteReplicaRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _TabletManager_Backup_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(tabletmanagerdata.BackupRequest) if err := stream.RecvMsg(m); err != nil { @@ -1915,6 +1953,10 @@ var _TabletManager_serviceDesc = grpc.ServiceDesc{ MethodName: "PromoteSlave", Handler: _TabletManager_PromoteSlave_Handler, }, + { + MethodName: "PromoteReplica", + Handler: _TabletManager_PromoteReplica_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/go/vt/vtcombo/tablet_map.go b/go/vt/vtcombo/tablet_map.go index 2119bd325ba..94cd5cadf83 100644 --- a/go/vt/vtcombo/tablet_map.go +++ b/go/vt/vtcombo/tablet_map.go @@ -688,6 +688,9 @@ func (itmc *internalTabletManagerClient) StopReplicationAndGetStatus(ctx context return nil, fmt.Errorf("not implemented in vtcombo") } +func (itmc *internalTabletManagerClient) PromoteReplica(ctx context.Context, tablet *topodatapb.Tablet) (string, error) { + return "", fmt.Errorf("not implemented in vtcombo") +} func (itmc *internalTabletManagerClient) PromoteSlave(ctx context.Context, tablet *topodatapb.Tablet) (string, error) { return "", fmt.Errorf("not implemented in vtcombo") } diff --git a/go/vt/vttablet/agentrpctest/test_agent_rpc.go b/go/vt/vttablet/agentrpctest/test_agent_rpc.go index de059c9ee5c..388260f6f90 100644 --- a/go/vt/vttablet/agentrpctest/test_agent_rpc.go +++ b/go/vt/vttablet/agentrpctest/test_agent_rpc.go @@ -944,7 +944,7 @@ func agentRPCTestInitMasterPanic(ctx context.Context, t *testing.T, client tmcli var testPopulateReparentJournalCalled = false var testTimeCreatedNS int64 = 4569900 -var testWaitPosition string = "test wait position" +var testWaitPosition = "test wait position" var testActionName = "TestActionName" var testMasterAlias = &topodatapb.TabletAlias{ Cell: "ce", @@ -1157,6 +1157,23 @@ func agentRPCTestPromoteSlavePanic(ctx context.Context, t *testing.T, client tmc expectHandleRPCPanic(t, "PromoteSlave", true /*verbose*/, err) } +func (fra *fakeRPCAgent) PromoteReplica(ctx context.Context) (string, error) { + if fra.panics { + panic(fmt.Errorf("test-triggered panic")) + } + return testReplicationPosition, nil +} + +func agentRPCTestPromoteReplica(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) { + rp, err := client.PromoteReplica(ctx, tablet) + compareError(t, "PromoteReplica", err, rp, testReplicationPosition) +} + +func agentRPCTestPromoteReplicaPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) { + _, err := client.PromoteReplica(ctx, tablet) + expectHandleRPCPanic(t, "PromoteReplica", true /*verbose*/, err) +} + // // Backup / restore related methods // @@ -1298,6 +1315,7 @@ func Run(t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.T agentRPCTestSlaveWasRestarted(ctx, t, client, tablet) agentRPCTestStopReplicationAndGetStatus(ctx, t, client, tablet) agentRPCTestPromoteSlave(ctx, t, client, tablet) + agentRPCTestPromoteReplica(ctx, t, client, tablet) // Backup / restore related methods agentRPCTestBackup(ctx, t, client, tablet) @@ -1352,6 +1370,7 @@ func Run(t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.T agentRPCTestSlaveWasRestartedPanic(ctx, t, client, tablet) agentRPCTestStopReplicationAndGetStatusPanic(ctx, t, client, tablet) agentRPCTestPromoteSlavePanic(ctx, t, client, tablet) + agentRPCTestPromoteReplicaPanic(ctx, t, client, tablet) // Backup / restore related methods agentRPCTestBackupPanic(ctx, t, client, tablet) diff --git a/go/vt/vttablet/faketmclient/fake_client.go b/go/vt/vttablet/faketmclient/fake_client.go index 60658787f72..cc56fc26d92 100644 --- a/go/vt/vttablet/faketmclient/fake_client.go +++ b/go/vt/vttablet/faketmclient/fake_client.go @@ -290,6 +290,11 @@ func (client *FakeTabletManagerClient) StopReplicationAndGetStatus(ctx context.C return &replicationdatapb.Status{}, nil } +// PromoteReplica is part of the tmclient.TabletManagerClient interface. +func (client *FakeTabletManagerClient) PromoteReplica(ctx context.Context, tablet *topodatapb.Tablet) (string, error) { + return "", nil +} + // PromoteSlave is part of the tmclient.TabletManagerClient interface. func (client *FakeTabletManagerClient) PromoteSlave(ctx context.Context, tablet *topodatapb.Tablet) (string, error) { return "", nil diff --git a/go/vt/vttablet/grpctmclient/client.go b/go/vt/vttablet/grpctmclient/client.go index d79e8003da3..d95996a8ece 100644 --- a/go/vt/vttablet/grpctmclient/client.go +++ b/go/vt/vttablet/grpctmclient/client.go @@ -686,6 +686,7 @@ func (client *Client) UndoDemoteMaster(ctx context.Context, tablet *topodatapb.T } // PromoteSlaveWhenCaughtUp is part of the tmclient.TabletManagerClient interface. +// Deprecated func (client *Client) PromoteSlaveWhenCaughtUp(ctx context.Context, tablet *topodatapb.Tablet, pos string) (string, error) { cc, c, err := client.dial(tablet) if err != nil { @@ -755,7 +756,22 @@ func (client *Client) StopReplicationAndGetStatus(ctx context.Context, tablet *t return response.Status, nil } +// PromoteReplica is part of the tmclient.TabletManagerClient interface. +func (client *Client) PromoteReplica(ctx context.Context, tablet *topodatapb.Tablet) (string, error) { + cc, c, err := client.dial(tablet) + if err != nil { + return "", err + } + defer cc.Close() + response, err := c.PromoteReplica(ctx, &tabletmanagerdatapb.PromoteReplicaRequest{}) + if err != nil { + return "", err + } + return response.Position, nil +} + // PromoteSlave is part of the tmclient.TabletManagerClient interface. +// Deprecated func (client *Client) PromoteSlave(ctx context.Context, tablet *topodatapb.Tablet) (string, error) { cc, c, err := client.dial(tablet) if err != nil { diff --git a/go/vt/vttablet/grpctmserver/server.go b/go/vt/vttablet/grpctmserver/server.go index 8d2d6b0c297..d3dc8cbe04d 100644 --- a/go/vt/vttablet/grpctmserver/server.go +++ b/go/vt/vttablet/grpctmserver/server.go @@ -387,6 +387,7 @@ func (s *server) UndoDemoteMaster(ctx context.Context, request *tabletmanagerdat return response, err } +// Deprecated func (s *server) PromoteSlaveWhenCaughtUp(ctx context.Context, request *tabletmanagerdatapb.PromoteSlaveWhenCaughtUpRequest) (response *tabletmanagerdatapb.PromoteSlaveWhenCaughtUpResponse, err error) { defer s.agent.HandleRPCPanic(ctx, "PromoteSlaveWhenCaughtUp", request, response, true /*verbose*/, &err) ctx = callinfo.GRPCCallInfo(ctx) @@ -430,6 +431,7 @@ func (s *server) StopReplicationAndGetStatus(ctx context.Context, request *table return response, err } +// Deprecated func (s *server) PromoteSlave(ctx context.Context, request *tabletmanagerdatapb.PromoteSlaveRequest) (response *tabletmanagerdatapb.PromoteSlaveResponse, err error) { defer s.agent.HandleRPCPanic(ctx, "PromoteSlave", request, response, true /*verbose*/, &err) ctx = callinfo.GRPCCallInfo(ctx) @@ -441,6 +443,17 @@ func (s *server) PromoteSlave(ctx context.Context, request *tabletmanagerdatapb. return response, err } +func (s *server) PromoteReplica(ctx context.Context, request *tabletmanagerdatapb.PromoteReplicaRequest) (response *tabletmanagerdatapb.PromoteReplicaResponse, err error) { + defer s.agent.HandleRPCPanic(ctx, "PromoteReplica", request, response, true /*verbose*/, &err) + ctx = callinfo.GRPCCallInfo(ctx) + response = &tabletmanagerdatapb.PromoteReplicaResponse{} + position, err := s.agent.PromoteReplica(ctx) + if err == nil { + response.Position = position + } + return response, err +} + func (s *server) Backup(request *tabletmanagerdatapb.BackupRequest, stream tabletmanagerservicepb.TabletManager_BackupServer) (err error) { ctx := stream.Context() defer s.agent.HandleRPCPanic(ctx, "Backup", request, nil, true /*verbose*/, &err) diff --git a/go/vt/vttablet/tabletmanager/rpc_agent.go b/go/vt/vttablet/tabletmanager/rpc_agent.go index eedcbf96c17..410f96823d4 100644 --- a/go/vt/vttablet/tabletmanager/rpc_agent.go +++ b/go/vt/vttablet/tabletmanager/rpc_agent.go @@ -114,6 +114,7 @@ type RPCAgent interface { UndoDemoteMaster(ctx context.Context) error + // Deprecated PromoteSlaveWhenCaughtUp(ctx context.Context, replicationPosition string) (string, error) SlaveWasPromoted(ctx context.Context) error @@ -124,6 +125,9 @@ type RPCAgent interface { StopReplicationAndGetStatus(ctx context.Context) (*replicationdatapb.Status, error) + PromoteReplica(ctx context.Context) (string, error) + + // Deprecated PromoteSlave(ctx context.Context) (string, error) // Backup / restore related methods diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index 5e49c658b39..622139fbbc1 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -461,6 +461,7 @@ func (agent *ActionAgent) UndoDemoteMaster(ctx context.Context) error { // PromoteSlaveWhenCaughtUp waits for this slave to be caught up on // replication up to the provided point, and then makes the slave the // shard master. +// Deprecated func (agent *ActionAgent) PromoteSlaveWhenCaughtUp(ctx context.Context, position string) (string, error) { if err := agent.lock(ctx); err != nil { return "", err @@ -469,7 +470,6 @@ func (agent *ActionAgent) PromoteSlaveWhenCaughtUp(ctx context.Context, position pos, err := mysql.DecodePosition(position) if err != nil { - return "", err } @@ -477,7 +477,7 @@ func (agent *ActionAgent) PromoteSlaveWhenCaughtUp(ctx context.Context, position return "", err } - pos, err = agent.MysqlDaemon.PromoteSlave(agent.hookExtraEnv()) + pos, err = agent.MysqlDaemon.Promote(agent.hookExtraEnv()) if err != nil { return "", err } @@ -488,6 +488,10 @@ func (agent *ActionAgent) PromoteSlaveWhenCaughtUp(ctx context.Context, position } startTime := time.Now() + if err := agent.MysqlDaemon.SetReadOnly(false); err != nil { + return "", err + } + _, err = topotools.ChangeType(ctx, agent.TopoServer, agent.TabletAlias, topodatapb.TabletType_MASTER, logutil.TimeToProto(startTime)) if err != nil { return "", err @@ -498,12 +502,6 @@ func (agent *ActionAgent) PromoteSlaveWhenCaughtUp(ctx context.Context, position // tablet's timestamp is ahead of the topo's timestamp. agent.setMasterTermStartTime(startTime) - // We call SetReadOnly only after the topo has been updated to avoid - // situations where two tablets are master at the DB level but not at the vitess level - if err := agent.MysqlDaemon.SetReadOnly(false); err != nil { - return "", err - } - if err := agent.refreshTablet(ctx, "PromoteSlaveWhenCaughtUp"); err != nil { return "", err } @@ -748,14 +746,14 @@ func (agent *ActionAgent) StopReplicationAndGetStatus(ctx context.Context) (*rep return mysql.SlaveStatusToProto(rs), nil } -// PromoteSlave makes the current tablet the master -func (agent *ActionAgent) PromoteSlave(ctx context.Context) (replicationPosition string, finalErr error) { +// PromoteReplica makes the current tablet the master +func (agent *ActionAgent) PromoteReplica(ctx context.Context) (string, error) { if err := agent.lock(ctx); err != nil { return "", err } defer agent.unlock() - pos, err := agent.MysqlDaemon.PromoteSlave(agent.hookExtraEnv()) + pos, err := agent.MysqlDaemon.Promote(agent.hookExtraEnv()) if err != nil { return "", err } @@ -782,6 +780,46 @@ func (agent *ActionAgent) PromoteSlave(ctx context.Context) (replicationPosition // tablet's timestamp is ahead of the topo's timestamp. agent.setMasterTermStartTime(startTime) + if err := agent.refreshTablet(ctx, "PromoteReplica"); err != nil { + return "", err + } + + return mysql.EncodePosition(pos), nil +} + +// PromoteSlave makes the current tablet the master +// Deprecated +func (agent *ActionAgent) PromoteSlave(ctx context.Context) (string, error) { + if err := agent.lock(ctx); err != nil { + return "", err + } + defer agent.unlock() + + pos, err := agent.MysqlDaemon.Promote(agent.hookExtraEnv()) + if err != nil { + return "", err + } + + // If using semi-sync, we need to enable it before going read-write. + if err := agent.fixSemiSync(topodatapb.TabletType_MASTER); err != nil { + return "", err + } + + // Set the server read-write + startTime := time.Now() + if err := agent.MysqlDaemon.SetReadOnly(false); err != nil { + return "", err + } + + if _, err := topotools.ChangeType(ctx, agent.TopoServer, agent.TabletAlias, topodatapb.TabletType_MASTER, logutil.TimeToProto(startTime)); err != nil { + return "", err + } + + // We only update agent's masterTermStartTime if we were able to update the topo. + // This ensures that in case of a failure, we are never in a situation where the + // tablet's timestamp is ahead of the topo's timestamp. + agent.setMasterTermStartTime(startTime) + if err := agent.refreshTablet(ctx, "PromoteSlave"); err != nil { return "", err } diff --git a/go/vt/vttablet/tmclient/rpc_client_api.go b/go/vt/vttablet/tmclient/rpc_client_api.go index 6e2ed4e4f20..7029cda982e 100644 --- a/go/vt/vttablet/tmclient/rpc_client_api.go +++ b/go/vt/vttablet/tmclient/rpc_client_api.go @@ -179,6 +179,7 @@ type TabletManagerClient interface { UndoDemoteMaster(ctx context.Context, tablet *topodatapb.Tablet) error // PromoteSlaveWhenCaughtUp transforms the tablet from a slave to a master. + // Deprecated PromoteSlaveWhenCaughtUp(ctx context.Context, tablet *topodatapb.Tablet, pos string) (string, error) // SlaveWasPromoted tells the remote tablet it is now the master @@ -197,8 +198,12 @@ type TabletManagerClient interface { StopReplicationAndGetStatus(ctx context.Context, tablet *topodatapb.Tablet) (*replicationdatapb.Status, error) // PromoteSlave makes the tablet the new master + // Deprecated PromoteSlave(ctx context.Context, tablet *topodatapb.Tablet) (string, error) + // PromoteReplica makes the tablet the new master + PromoteReplica(ctx context.Context, tablet *topodatapb.Tablet) (string, error) + // // Backup / restore related methods // diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index a36f7a6cf96..294637d3e7b 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -518,7 +518,7 @@ func (wr *Wrangler) plannedReparentShardLocked(ctx context.Context, ev *events.R // Promote the selected candidate to master. promoteCtx, promoteCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer promoteCancel() - rp, err := wr.tmc.PromoteSlave(promoteCtx, masterElectTabletInfo.Tablet) + rp, err := wr.tmc.PromoteReplica(promoteCtx, masterElectTabletInfo.Tablet) if err != nil { return vterrors.Wrapf(err, "failed to promote %v to master", masterElectTabletAliasStr) } @@ -594,18 +594,13 @@ func (wr *Wrangler) plannedReparentShardLocked(ctx context.Context, ev *events.R return fmt.Errorf("old master tablet %v DemoteMaster failed: %v", topoproto.TabletAliasString(shardInfo.MasterAlias), err) } - // Wait on the master-elect tablet until it reaches that position, - // then promote it. - wr.logger.Infof("promote replica %v", masterElectTabletAliasStr) - event.DispatchUpdate(ev, "promoting replica") + waitCtx, waitCancel := context.WithTimeout(ctx, waitReplicasTimeout) + defer waitCancel() - promoteCtx, promoteCancel := context.WithTimeout(ctx, waitReplicasTimeout) - defer promoteCancel() - - rp, err = wr.tmc.PromoteSlaveWhenCaughtUp(promoteCtx, masterElectTabletInfo.Tablet, rp) - if err != nil || (ctx.Err() != nil && ctx.Err() == context.DeadlineExceeded) { - // If we fail to promote the new master, try to roll back to the - // original master before aborting. + waitErr := wr.tmc.WaitForPosition(waitCtx, masterElectTabletInfo.Tablet, rp) + if waitErr != nil || ctx.Err() == context.DeadlineExceeded { + // If the new master fails to catch up within the timeout, + // we try to roll back to the original master before aborting. // It is possible that we have used up the original context, or that // not enough time is left on it before it times out. // But at this point we really need to be able to Undo so as not to @@ -613,10 +608,25 @@ func (wr *Wrangler) plannedReparentShardLocked(ctx context.Context, ev *events.R // So we create a fresh context based on context.Background(). undoCtx, undoCancel := context.WithTimeout(context.Background(), *topo.RemoteOperationTimeout) defer undoCancel() - if err1 := wr.tmc.UndoDemoteMaster(undoCtx, oldMasterTabletInfo.Tablet); err1 != nil { - log.Warningf("Encountered error %v while trying to undo DemoteMaster", err1) + if undoErr := wr.tmc.UndoDemoteMaster(undoCtx, oldMasterTabletInfo.Tablet); undoErr != nil { + log.Warningf("Encountered error while trying to undo DemoteMaster: %v", undoErr) } - return fmt.Errorf("master-elect tablet %v failed to catch up with replication or be upgraded to master: %v", masterElectTabletAliasStr, err) + if waitErr != nil { + return vterrors.Wrapf(err, "master-elect tablet %v failed to catch up with replication", masterElectTabletAliasStr) + } + return vterrors.New(vtrpcpb.Code_DEADLINE_EXCEEDED, "PlannedReparent timed out, please try again.") + } + + promoteCtx, promoteCancel := context.WithTimeout(ctx, waitReplicasTimeout) + defer promoteCancel() + rp, err = wr.tmc.PromoteReplica(promoteCtx, masterElectTabletInfo.Tablet) + if err != nil { + return vterrors.Wrapf(err, "master-elect tablet %v failed to be upgraded to master - please try again", masterElectTabletAliasStr) + } + + if ctx.Err() == context.DeadlineExceeded { + // PromoteReplica succeeded but the context has expired. PRS needs to be re-run to complete + return vterrors.New(vtrpcpb.Code_DEADLINE_EXCEEDED, "PlannedReparent timed out after promoting new master. Please re-run to fixup replicas.") } reparentJournalPos = rp } @@ -952,9 +962,9 @@ func (wr *Wrangler) emergencyReparentShardLocked(ctx context.Context, ev *events } // Promote the masterElect - wr.logger.Infof("promote slave %v", topoproto.TabletAliasString(masterElectTabletAlias)) - event.DispatchUpdate(ev, "promoting slave") - rp, err := wr.tmc.PromoteSlave(ctx, masterElectTabletInfo.Tablet) + wr.logger.Infof("promote tablet %v to master", topoproto.TabletAliasString(masterElectTabletAlias)) + event.DispatchUpdate(ev, "promoting replica") + rp, err := wr.tmc.PromoteReplica(ctx, masterElectTabletInfo.Tablet) if err != nil { return fmt.Errorf("master-elect tablet %v failed to be upgraded to master: %v", topoproto.TabletAliasString(masterElectTabletAlias), err) } diff --git a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go index 6749d8e8c03..01045d25ca4 100644 --- a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go @@ -63,7 +63,7 @@ func TestEmergencyReparentShard(t *testing.T) { "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES", } - newMaster.FakeMysqlDaemon.PromoteSlaveResult = mysql.Position{ + newMaster.FakeMysqlDaemon.PromoteResult = mysql.Position{ GTIDSet: mysql.MariadbGTIDSet{ mysql.MariadbGTID{ Domain: 2, diff --git a/go/vt/wrangler/testlib/planned_reparent_shard_test.go b/go/vt/wrangler/testlib/planned_reparent_shard_test.go index 2221a4b7350..8173f9e2401 100644 --- a/go/vt/wrangler/testlib/planned_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/planned_reparent_shard_test.go @@ -56,7 +56,7 @@ func TestPlannedReparentShardNoMasterProvided(t *testing.T) { }, }, } - newMaster.FakeMysqlDaemon.PromoteSlaveResult = mysql.Position{ + newMaster.FakeMysqlDaemon.PromoteResult = mysql.Position{ GTIDSet: mysql.MariadbGTIDSet{ mysql.MariadbGTID{ Domain: 7, @@ -172,7 +172,7 @@ func TestPlannedReparentShard(t *testing.T) { }, }, } - newMaster.FakeMysqlDaemon.PromoteSlaveResult = mysql.Position{ + newMaster.FakeMysqlDaemon.PromoteResult = mysql.Position{ GTIDSet: mysql.MariadbGTIDSet{ mysql.MariadbGTID{ Domain: 7, @@ -328,7 +328,7 @@ func TestPlannedReparentShardPromoteSlaveFail(t *testing.T) { }, }, } - newMaster.FakeMysqlDaemon.PromoteSlaveResult = mysql.Position{ + newMaster.FakeMysqlDaemon.PromoteResult = mysql.Position{ GTIDSet: mysql.MariadbGTIDSet{ mysql.MariadbGTID{ Domain: 7, @@ -349,7 +349,7 @@ func TestPlannedReparentShardPromoteSlaveFail(t *testing.T) { oldMaster.FakeMysqlDaemon.ReadOnly = false oldMaster.FakeMysqlDaemon.Replicating = false // set to incorrect value to make promote fail on WaitForMasterPos - oldMaster.FakeMysqlDaemon.CurrentMasterPosition = newMaster.FakeMysqlDaemon.PromoteSlaveResult + oldMaster.FakeMysqlDaemon.CurrentMasterPosition = newMaster.FakeMysqlDaemon.PromoteResult oldMaster.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) oldMaster.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", @@ -425,7 +425,7 @@ func TestPlannedReparentShardPromoteSlaveTimeout(t *testing.T) { }, }, } - newMaster.FakeMysqlDaemon.PromoteSlaveResult = mysql.Position{ + newMaster.FakeMysqlDaemon.PromoteResult = mysql.Position{ GTIDSet: mysql.MariadbGTIDSet{ mysql.MariadbGTID{ Domain: 7, diff --git a/proto/tabletmanagerdata.proto b/proto/tabletmanagerdata.proto index 9f05f103553..c9165e3a95e 100644 --- a/proto/tabletmanagerdata.proto +++ b/proto/tabletmanagerdata.proto @@ -395,10 +395,12 @@ message UndoDemoteMasterRequest { message UndoDemoteMasterResponse { } +// Deprecated message PromoteSlaveWhenCaughtUpRequest { string position = 1; } +// Deprecated message PromoteSlaveWhenCaughtUpResponse { string position = 1; } @@ -434,13 +436,22 @@ message StopReplicationAndGetStatusResponse { replicationdata.Status status = 1; } +// Deprecated message PromoteSlaveRequest { } +// Deprecated message PromoteSlaveResponse { string position = 1; } +message PromoteReplicaRequest { +} + +message PromoteReplicaResponse { + string position = 1; +} + // Backup / Restore related messages message BackupRequest { diff --git a/proto/tabletmanagerservice.proto b/proto/tabletmanagerservice.proto index f1351f63e35..85ed11594ee 100644 --- a/proto/tabletmanagerservice.proto +++ b/proto/tabletmanagerservice.proto @@ -165,6 +165,7 @@ service TabletManager { // PromoteSlaveWhenCaughtUp tells the remote tablet to catch up, // and then be the master + // Deprecated rpc PromoteSlaveWhenCaughtUp(tabletmanagerdata.PromoteSlaveWhenCaughtUpRequest) returns (tabletmanagerdata.PromoteSlaveWhenCaughtUpResponse) {}; // SlaveWasPromoted tells the remote tablet it is now the master @@ -181,8 +182,12 @@ service TabletManager { rpc StopReplicationAndGetStatus(tabletmanagerdata.StopReplicationAndGetStatusRequest) returns (tabletmanagerdata.StopReplicationAndGetStatusResponse) {}; // PromoteSlave makes the slave the new master + // Deprecated rpc PromoteSlave(tabletmanagerdata.PromoteSlaveRequest) returns (tabletmanagerdata.PromoteSlaveResponse) {}; + // PromoteReplica makes the replica the new master + rpc PromoteReplica(tabletmanagerdata.PromoteReplicaRequest) returns (tabletmanagerdata.PromoteReplicaResponse) {}; + // // Backup related methods // From bfe7931429c30d8b69c148fd7b40645f625aff28 Mon Sep 17 00:00:00 2001 From: deepthi Date: Fri, 10 Apr 2020 18:37:14 -0700 Subject: [PATCH 3/6] PlannedReparent: set ReadOnly on master if needed Signed-off-by: deepthi --- go/vt/wrangler/reparent.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index 294637d3e7b..fbeb7ef0b51 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -524,11 +524,19 @@ func (wr *Wrangler) plannedReparentShardLocked(ctx context.Context, ev *events.R } reparentJournalPos = rp } else if topoproto.TabletAliasEqual(currentMaster.Alias, masterElectTabletAlias) { - refreshCtx, refreshCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) - defer refreshCancel() + // It is possible that a previous attempt to reparent failed to SetReadWrite + // so call it here to make sure underlying mysql is ReadWrite + rwCtx, rwCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) + defer rwCancel() + if err := wr.tmc.SetReadWrite(rwCtx, masterElectTabletInfo.Tablet); err != nil { + return vterrors.Wrapf(err, "failed to SetReadWrite on current master %v", masterElectTabletAliasStr) + } // The master is already the one we want according to its tablet record. // Refresh it to make sure the tablet has read its record recently. + refreshCtx, refreshCancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) + defer refreshCancel() + if err := wr.tmc.RefreshState(refreshCtx, masterElectTabletInfo.Tablet); err != nil { return vterrors.Wrapf(err, "failed to RefreshState on current master %v", masterElectTabletAliasStr) } @@ -954,10 +962,10 @@ func (wr *Wrangler) emergencyReparentShardLocked(ctx context.Context, ev *events } pos, err := mysql.DecodePosition(status.Position) if err != nil { - return fmt.Errorf("cannot decode slave %v position %v: %v", alias, status.Position, err) + return fmt.Errorf("cannot decode replica %v position %v: %v", alias, status.Position, err) } if !masterElectPos.AtLeast(pos) { - return fmt.Errorf("tablet %v is more advanced than master elect tablet %v: %v > %v", alias, masterElectTabletAliasStr, status.Position, masterElectStatus) + return fmt.Errorf("tablet %v is more advanced than master elect tablet %v: %v > %v", alias, masterElectTabletAliasStr, status.Position, masterElectStatus.Position) } } From eddbb58cdd7c6932cdc458393b66d6ca9ad0a227 Mon Sep 17 00:00:00 2001 From: deepthi Date: Fri, 10 Apr 2020 18:39:55 -0700 Subject: [PATCH 4/6] PlannedReparent: tests for Promote failure, re-run failed PRS, PRS with current master Signed-off-by: deepthi --- .../fakemysqldaemon/fakemysqldaemon.go | 6 + go/vt/wrangler/reparent.go | 2 +- .../testlib/emergency_reparent_shard_test.go | 107 ++-- .../testlib/planned_reparent_shard_test.go | 460 +++++++++++++----- 4 files changed, 407 insertions(+), 168 deletions(-) diff --git a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go index c31b5861899..0d0fea3f051 100644 --- a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go +++ b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go @@ -107,6 +107,9 @@ type FakeMysqlDaemon struct { // PromoteResult is returned by Promote PromoteResult mysql.Position + // PromoteError is used by Promote + PromoteError error + // SchemaFunc provides the return value for GetSchema. // If not defined, the "Schema" field will be used instead, see below. SchemaFunc func() (*tabletmanagerdatapb.SchemaDefinition, error) @@ -350,6 +353,9 @@ func (fmd *FakeMysqlDaemon) WaitMasterPos(_ context.Context, pos mysql.Position) // Promote is part of the MysqlDaemon interface func (fmd *FakeMysqlDaemon) Promote(hookExtraEnv map[string]string) (mysql.Position, error) { + if fmd.PromoteError != nil { + return mysql.Position{}, fmd.PromoteError + } return fmd.PromoteResult, nil } diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index fbeb7ef0b51..e24aa83a898 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -641,7 +641,7 @@ func (wr *Wrangler) plannedReparentShardLocked(ctx context.Context, ev *events.R // Check we still have the topology lock. if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { - return fmt.Errorf("lost topology lock, aborting: %v", err) + return vterrors.Wrap(err, "lost topology lock, aborting") } // Create a cancelable context for the following RPCs. diff --git a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go index 01045d25ca4..5fd63be499c 100644 --- a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go @@ -39,11 +39,11 @@ func TestEmergencyReparentShard(t *testing.T) { vp := NewVtctlPipe(t, ts) defer vp.Close() - // Create a master, a couple good slaves + // Create a master, a couple good replicas oldMaster := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, nil) newMaster := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, nil) - goodSlave1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) - goodSlave2 := NewFakeTablet(t, wr, "cell2", 3, topodatapb.TabletType_REPLICA, nil) + goodReplica1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) + goodReplica2 := NewFakeTablet(t, wr, "cell2", 3, topodatapb.TabletType_REPLICA, nil) // new master newMaster.FakeMysqlDaemon.ReadOnly = true @@ -80,10 +80,10 @@ func TestEmergencyReparentShard(t *testing.T) { oldMaster.StartActionLoop(t, wr) defer oldMaster.StopActionLoop(t) - // good slave 1 is replicating - goodSlave1.FakeMysqlDaemon.ReadOnly = true - goodSlave1.FakeMysqlDaemon.Replicating = true - goodSlave1.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{ + // good replica 1 is replicating + goodReplica1.FakeMysqlDaemon.ReadOnly = true + goodReplica1.FakeMysqlDaemon.Replicating = true + goodReplica1.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{ GTIDSet: mysql.MariadbGTIDSet{ mysql.MariadbGTID{ Domain: 2, @@ -92,19 +92,19 @@ func TestEmergencyReparentShard(t *testing.T) { }, }, } - goodSlave1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) - goodSlave1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + goodReplica1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) + goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", "FAKE SET MASTER", "START SLAVE", } - goodSlave1.StartActionLoop(t, wr) - defer goodSlave1.StopActionLoop(t) + goodReplica1.StartActionLoop(t, wr) + defer goodReplica1.StopActionLoop(t) - // good slave 2 is not replicating - goodSlave2.FakeMysqlDaemon.ReadOnly = true - goodSlave2.FakeMysqlDaemon.Replicating = false - goodSlave2.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{ + // good replica 2 is not replicating + goodReplica2.FakeMysqlDaemon.ReadOnly = true + goodReplica2.FakeMysqlDaemon.Replicating = false + goodReplica2.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{ GTIDSet: mysql.MariadbGTIDSet{ mysql.MariadbGTID{ Domain: 2, @@ -113,12 +113,12 @@ func TestEmergencyReparentShard(t *testing.T) { }, }, } - goodSlave2.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) - goodSlave2.StartActionLoop(t, wr) - goodSlave2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + goodReplica2.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) + goodReplica2.StartActionLoop(t, wr) + goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", } - defer goodSlave2.StopActionLoop(t) + defer goodReplica2.StopActionLoop(t) // run EmergencyReparentShard if err := vp.Run([]string{"EmergencyReparentShard", "-wait_slave_timeout", "10s", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, topoproto.TabletAliasString(newMaster.Tablet.Alias)}); err != nil { @@ -132,30 +132,30 @@ func TestEmergencyReparentShard(t *testing.T) { if err := oldMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { t.Fatalf("oldMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } - if err := goodSlave1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Fatalf("goodSlave1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) + if err := goodReplica1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { + t.Fatalf("goodReplica1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } - if err := goodSlave2.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Fatalf("goodSlave2.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) + if err := goodReplica2.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { + t.Fatalf("goodReplica2.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } if newMaster.FakeMysqlDaemon.ReadOnly { t.Errorf("newMaster.FakeMysqlDaemon.ReadOnly set") } // old master read-only flag doesn't matter, it is scrapped - if !goodSlave1.FakeMysqlDaemon.ReadOnly { - t.Errorf("goodSlave1.FakeMysqlDaemon.ReadOnly not set") + if !goodReplica1.FakeMysqlDaemon.ReadOnly { + t.Errorf("goodReplica1.FakeMysqlDaemon.ReadOnly not set") } - if !goodSlave2.FakeMysqlDaemon.ReadOnly { - t.Errorf("goodSlave2.FakeMysqlDaemon.ReadOnly not set") + if !goodReplica2.FakeMysqlDaemon.ReadOnly { + t.Errorf("goodReplica2.FakeMysqlDaemon.ReadOnly not set") } - if !goodSlave1.FakeMysqlDaemon.Replicating { - t.Errorf("goodSlave1.FakeMysqlDaemon.Replicating not set") + if !goodReplica1.FakeMysqlDaemon.Replicating { + t.Errorf("goodReplica1.FakeMysqlDaemon.Replicating not set") } - if goodSlave2.FakeMysqlDaemon.Replicating { - t.Errorf("goodSlave2.FakeMysqlDaemon.Replicating set") + if goodReplica2.FakeMysqlDaemon.Replicating { + t.Errorf("goodReplica2.FakeMysqlDaemon.Replicating set") } checkSemiSyncEnabled(t, true, true, newMaster) - checkSemiSyncEnabled(t, false, true, goodSlave1, goodSlave2) + checkSemiSyncEnabled(t, false, true, goodReplica1, goodReplica2) } // TestEmergencyReparentShardMasterElectNotBest tries to emergency reparent @@ -165,13 +165,25 @@ func TestEmergencyReparentShardMasterElectNotBest(t *testing.T) { ts := memorytopo.NewServer("cell1", "cell2") wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) - // Create a master, a couple good slaves + // Create a master, a couple good replicas oldMaster := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, nil) newMaster := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, nil) - moreAdvancedSlave := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) + moreAdvancedReplica := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) // new master newMaster.FakeMysqlDaemon.Replicating = true + // this server has executed upto 455, which is the highest among replicas + newMaster.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{ + GTIDSet: mysql.MariadbGTIDSet{ + mysql.MariadbGTID{ + Domain: 2, + Server: 123, + Sequence: 455, + }, + }, + } + // It has more transactions in its relay log, but not as many as + // moreAdvancedReplica newMaster.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{ GTIDSet: mysql.MariadbGTIDSet{ mysql.MariadbGTID{ @@ -191,9 +203,20 @@ func TestEmergencyReparentShardMasterElectNotBest(t *testing.T) { oldMaster.StartActionLoop(t, wr) defer oldMaster.StopActionLoop(t) - // more advanced slave - moreAdvancedSlave.FakeMysqlDaemon.Replicating = true - moreAdvancedSlave.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{ + // more advanced replica + moreAdvancedReplica.FakeMysqlDaemon.Replicating = true + // position up to which this replica has executed is behind desired new master + moreAdvancedReplica.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{ + GTIDSet: mysql.MariadbGTIDSet{ + mysql.MariadbGTID{ + Domain: 2, + Server: 123, + Sequence: 454, + }, + }, + } + // relay log position is more advanced than desired new master + moreAdvancedReplica.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{ GTIDSet: mysql.MariadbGTIDSet{ mysql.MariadbGTID{ Domain: 2, @@ -202,11 +225,11 @@ func TestEmergencyReparentShardMasterElectNotBest(t *testing.T) { }, }, } - moreAdvancedSlave.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + moreAdvancedReplica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", } - moreAdvancedSlave.StartActionLoop(t, wr) - defer moreAdvancedSlave.StopActionLoop(t) + moreAdvancedReplica.StartActionLoop(t, wr) + defer moreAdvancedReplica.StopActionLoop(t) // run EmergencyReparentShard if err := wr.EmergencyReparentShard(ctx, newMaster.Tablet.Keyspace, newMaster.Tablet.Shard, newMaster.Tablet.Alias, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is more advanced than master elect tablet") { @@ -220,7 +243,7 @@ func TestEmergencyReparentShardMasterElectNotBest(t *testing.T) { if err := oldMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { t.Fatalf("oldMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } - if err := moreAdvancedSlave.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Fatalf("moreAdvancedSlave.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) + if err := moreAdvancedReplica.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { + t.Fatalf("moreAdvancedReplica.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } } diff --git a/go/vt/wrangler/testlib/planned_reparent_shard_test.go b/go/vt/wrangler/testlib/planned_reparent_shard_test.go index 8173f9e2401..dd2dc09d5c5 100644 --- a/go/vt/wrangler/testlib/planned_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/planned_reparent_shard_test.go @@ -22,6 +22,8 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/topo/memorytopo" @@ -39,10 +41,10 @@ func TestPlannedReparentShardNoMasterProvided(t *testing.T) { vp := NewVtctlPipe(t, ts) defer vp.Close() - // Create a master, a couple good slaves + // Create a master, a couple good replicas oldMaster := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, nil) newMaster := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, nil) - goodSlave1 := NewFakeTablet(t, wr, "cell2", 2, topodatapb.TabletType_REPLICA, nil) + goodReplica1 := NewFakeTablet(t, wr, "cell2", 2, topodatapb.TabletType_REPLICA, nil) // new master newMaster.FakeMysqlDaemon.ReadOnly = true @@ -96,17 +98,17 @@ func TestPlannedReparentShardNoMasterProvided(t *testing.T) { // SetMaster is called on new master to make sure it's replicating before reparenting. newMaster.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(oldMaster.Tablet) - // good slave 1 is replicating - goodSlave1.FakeMysqlDaemon.ReadOnly = true - goodSlave1.FakeMysqlDaemon.Replicating = true - goodSlave1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) - goodSlave1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + // good replica 1 is replicating + goodReplica1.FakeMysqlDaemon.ReadOnly = true + goodReplica1.FakeMysqlDaemon.Replicating = true + goodReplica1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) + goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", "FAKE SET MASTER", "START SLAVE", } - goodSlave1.StartActionLoop(t, wr) - defer goodSlave1.StopActionLoop(t) + goodReplica1.StartActionLoop(t, wr) + defer goodReplica1.StopActionLoop(t) // run PlannedReparentShard if err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard}); err != nil { @@ -120,8 +122,8 @@ func TestPlannedReparentShardNoMasterProvided(t *testing.T) { if err := oldMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { t.Errorf("oldMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } - if err := goodSlave1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("goodSlave1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) + if err := goodReplica1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { + t.Errorf("goodReplica1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } if newMaster.FakeMysqlDaemon.ReadOnly { t.Errorf("newMaster.FakeMysqlDaemon.ReadOnly set") @@ -129,36 +131,36 @@ func TestPlannedReparentShardNoMasterProvided(t *testing.T) { if !oldMaster.FakeMysqlDaemon.ReadOnly { t.Errorf("oldMaster.FakeMysqlDaemon.ReadOnly not set") } - if !goodSlave1.FakeMysqlDaemon.ReadOnly { - t.Errorf("goodSlave1.FakeMysqlDaemon.ReadOnly not set") + if !goodReplica1.FakeMysqlDaemon.ReadOnly { + t.Errorf("goodReplica1.FakeMysqlDaemon.ReadOnly not set") } if !oldMaster.Agent.QueryServiceControl.IsServing() { t.Errorf("oldMaster...QueryServiceControl not serving") } // verify the old master was told to start replicating (and not - // the slave that wasn't replicating in the first place) + // the replica that wasn't replicating in the first place) if !oldMaster.FakeMysqlDaemon.Replicating { t.Errorf("oldMaster.FakeMysqlDaemon.Replicating not set") } - if !goodSlave1.FakeMysqlDaemon.Replicating { - t.Errorf("goodSlave1.FakeMysqlDaemon.Replicating not set") + if !goodReplica1.FakeMysqlDaemon.Replicating { + t.Errorf("goodReplica1.FakeMysqlDaemon.Replicating not set") } checkSemiSyncEnabled(t, true, true, newMaster) - checkSemiSyncEnabled(t, false, true, goodSlave1, oldMaster) + checkSemiSyncEnabled(t, false, true, goodReplica1, oldMaster) } -func TestPlannedReparentShard(t *testing.T) { +func TestPlannedReparentShardNoError(t *testing.T) { ts := memorytopo.NewServer("cell1", "cell2") wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) vp := NewVtctlPipe(t, ts) defer vp.Close() - // Create a master, a couple good slaves + // Create a master, a couple good replicas oldMaster := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, nil) newMaster := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, nil) - goodSlave1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) - goodSlave2 := NewFakeTablet(t, wr, "cell2", 3, topodatapb.TabletType_REPLICA, nil) + goodReplica1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) + goodReplica2 := NewFakeTablet(t, wr, "cell2", 3, topodatapb.TabletType_REPLICA, nil) // new master newMaster.FakeMysqlDaemon.ReadOnly = true @@ -212,27 +214,27 @@ func TestPlannedReparentShard(t *testing.T) { // SetMaster is called on new master to make sure it's replicating before reparenting. newMaster.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(oldMaster.Tablet) - // good slave 1 is replicating - goodSlave1.FakeMysqlDaemon.ReadOnly = true - goodSlave1.FakeMysqlDaemon.Replicating = true - goodSlave1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) - goodSlave1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + // goodReplica1 is replicating + goodReplica1.FakeMysqlDaemon.ReadOnly = true + goodReplica1.FakeMysqlDaemon.Replicating = true + goodReplica1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) + goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", "FAKE SET MASTER", "START SLAVE", } - goodSlave1.StartActionLoop(t, wr) - defer goodSlave1.StopActionLoop(t) + goodReplica1.StartActionLoop(t, wr) + defer goodReplica1.StopActionLoop(t) - // good slave 2 is not replicating - goodSlave2.FakeMysqlDaemon.ReadOnly = true - goodSlave2.FakeMysqlDaemon.Replicating = false - goodSlave2.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) - goodSlave2.StartActionLoop(t, wr) - goodSlave2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + // goodReplica2 is not replicating + goodReplica2.FakeMysqlDaemon.ReadOnly = true + goodReplica2.FakeMysqlDaemon.Replicating = false + goodReplica2.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) + goodReplica2.StartActionLoop(t, wr) + goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", } - defer goodSlave2.StopActionLoop(t) + defer goodReplica2.StopActionLoop(t) // run PlannedReparentShard if err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(newMaster.Tablet.Alias)}); err != nil { @@ -246,11 +248,11 @@ func TestPlannedReparentShard(t *testing.T) { if err := oldMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { t.Errorf("oldMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } - if err := goodSlave1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("goodSlave1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) + if err := goodReplica1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { + t.Errorf("goodReplica1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } - if err := goodSlave2.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("goodSlave2.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) + if err := goodReplica2.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { + t.Errorf("goodReplica2.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } if newMaster.FakeMysqlDaemon.ReadOnly { t.Errorf("newMaster.FakeMysqlDaemon.ReadOnly set") @@ -258,30 +260,30 @@ func TestPlannedReparentShard(t *testing.T) { if !oldMaster.FakeMysqlDaemon.ReadOnly { t.Errorf("oldMaster.FakeMysqlDaemon.ReadOnly not set") } - if !goodSlave1.FakeMysqlDaemon.ReadOnly { - t.Errorf("goodSlave1.FakeMysqlDaemon.ReadOnly not set") + if !goodReplica1.FakeMysqlDaemon.ReadOnly { + t.Errorf("goodReplica1.FakeMysqlDaemon.ReadOnly not set") } - if !goodSlave2.FakeMysqlDaemon.ReadOnly { - t.Errorf("goodSlave2.FakeMysqlDaemon.ReadOnly not set") + if !goodReplica2.FakeMysqlDaemon.ReadOnly { + t.Errorf("goodReplica2.FakeMysqlDaemon.ReadOnly not set") } if !oldMaster.Agent.QueryServiceControl.IsServing() { t.Errorf("oldMaster...QueryServiceControl not serving") } // verify the old master was told to start replicating (and not - // the slave that wasn't replicating in the first place) + // the replica that wasn't replicating in the first place) if !oldMaster.FakeMysqlDaemon.Replicating { t.Errorf("oldMaster.FakeMysqlDaemon.Replicating not set") } - if !goodSlave1.FakeMysqlDaemon.Replicating { - t.Errorf("goodSlave1.FakeMysqlDaemon.Replicating not set") + if !goodReplica1.FakeMysqlDaemon.Replicating { + t.Errorf("goodReplica1.FakeMysqlDaemon.Replicating not set") } - if goodSlave2.FakeMysqlDaemon.Replicating { - t.Errorf("goodSlave2.FakeMysqlDaemon.Replicating set") + if goodReplica2.FakeMysqlDaemon.Replicating { + t.Errorf("goodReplica2.FakeMysqlDaemon.Replicating set") } checkSemiSyncEnabled(t, true, true, newMaster) - checkSemiSyncEnabled(t, false, true, goodSlave1, goodSlave2, oldMaster) + checkSemiSyncEnabled(t, false, true, goodReplica1, goodReplica2, oldMaster) } func TestPlannedReparentNoMaster(t *testing.T) { @@ -304,17 +306,19 @@ func TestPlannedReparentNoMaster(t *testing.T) { } } -func TestPlannedReparentShardPromoteSlaveFail(t *testing.T) { +// TestPlannedReparentShardWaitForPositionFail simulates a failure of the WaitForPosition call +// on the desired new master tablet +func TestPlannedReparentShardWaitForPositionFail(t *testing.T) { ts := memorytopo.NewServer("cell1", "cell2") wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) vp := NewVtctlPipe(t, ts) defer vp.Close() - // Create a master, a couple good slaves + // Create a master, a couple good replicas oldMaster := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, nil) newMaster := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, nil) - goodSlave1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) - goodSlave2 := NewFakeTablet(t, wr, "cell2", 3, topodatapb.TabletType_REPLICA, nil) + goodReplica1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) + goodReplica2 := NewFakeTablet(t, wr, "cell2", 3, topodatapb.TabletType_REPLICA, nil) // new master newMaster.FakeMysqlDaemon.ReadOnly = true @@ -338,6 +342,9 @@ func TestPlannedReparentShardPromoteSlaveFail(t *testing.T) { }, } newMaster.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "STOP SLAVE", + "FAKE SET MASTER", + "START SLAVE", "CREATE DATABASE IF NOT EXISTS _vt", "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES", @@ -358,34 +365,36 @@ func TestPlannedReparentShardPromoteSlaveFail(t *testing.T) { oldMaster.StartActionLoop(t, wr) defer oldMaster.StopActionLoop(t) oldMaster.Agent.QueryServiceControl.(*tabletservermock.Controller).SetQueryServiceEnabledForTests(true) + // SetMaster is called on new master to make sure it's replicating before reparenting. + newMaster.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(oldMaster.Tablet) - // good slave 1 is replicating - goodSlave1.FakeMysqlDaemon.ReadOnly = true - goodSlave1.FakeMysqlDaemon.Replicating = true - goodSlave1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) - goodSlave1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + // good replica 1 is replicating + goodReplica1.FakeMysqlDaemon.ReadOnly = true + goodReplica1.FakeMysqlDaemon.Replicating = true + goodReplica1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) + goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", "FAKE SET MASTER", "START SLAVE", } - goodSlave1.StartActionLoop(t, wr) - defer goodSlave1.StopActionLoop(t) + goodReplica1.StartActionLoop(t, wr) + defer goodReplica1.StopActionLoop(t) - // good slave 2 is not replicating - goodSlave2.FakeMysqlDaemon.ReadOnly = true - goodSlave2.FakeMysqlDaemon.Replicating = false - goodSlave2.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) - goodSlave2.StartActionLoop(t, wr) - goodSlave2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + // good replica 2 is not replicating + goodReplica2.FakeMysqlDaemon.ReadOnly = true + goodReplica2.FakeMysqlDaemon.Replicating = false + goodReplica2.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) + goodReplica2.StartActionLoop(t, wr) + goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", } - defer goodSlave2.StopActionLoop(t) + defer goodReplica2.StopActionLoop(t) // run PlannedReparentShard err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(newMaster.Tablet.Alias)}) if err == nil { - t.Fatalf("PlannedReparentShard succeeded: %v", err) + t.Fatal("PlannedReparentShard succeeded") } if !strings.Contains(err.Error(), "replication on master-elect cell1-0000000001 did not catch up in time") { t.Fatalf("PlannedReparentShard failed with the wrong error: %v", err) @@ -400,17 +409,19 @@ func TestPlannedReparentShardPromoteSlaveFail(t *testing.T) { } } -func TestPlannedReparentShardPromoteSlaveTimeout(t *testing.T) { +// TestPlannedReparentShardWaitForPositionTimeout simulates a context timeout +// during the WaitForPosition call to the desired new master +func TestPlannedReparentShardWaitForPositionTimeout(t *testing.T) { ts := memorytopo.NewServer("cell1", "cell2") wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) vp := NewVtctlPipe(t, ts) defer vp.Close() - // Create a master, a couple good slaves + // Create a master, a couple good replicas oldMaster := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, nil) newMaster := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, nil) - goodSlave1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) - goodSlave2 := NewFakeTablet(t, wr, "cell2", 3, topodatapb.TabletType_REPLICA, nil) + goodReplica1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) + goodReplica2 := NewFakeTablet(t, wr, "cell2", 3, topodatapb.TabletType_REPLICA, nil) // new master newMaster.FakeMysqlDaemon.TimeoutHook = func() error { return context.DeadlineExceeded } @@ -435,6 +446,9 @@ func TestPlannedReparentShardPromoteSlaveTimeout(t *testing.T) { }, } newMaster.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "STOP SLAVE", + "FAKE SET MASTER", + "START SLAVE", "CREATE DATABASE IF NOT EXISTS _vt", "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES", @@ -455,33 +469,35 @@ func TestPlannedReparentShardPromoteSlaveTimeout(t *testing.T) { defer oldMaster.StopActionLoop(t) oldMaster.Agent.QueryServiceControl.(*tabletservermock.Controller).SetQueryServiceEnabledForTests(true) - // good slave 1 is replicating - goodSlave1.FakeMysqlDaemon.ReadOnly = true - goodSlave1.FakeMysqlDaemon.Replicating = true - goodSlave1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) - goodSlave1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + // SetMaster is called on new master to make sure it's replicating before reparenting. + newMaster.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(oldMaster.Tablet) + // good replica 1 is replicating + goodReplica1.FakeMysqlDaemon.ReadOnly = true + goodReplica1.FakeMysqlDaemon.Replicating = true + goodReplica1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) + goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", "FAKE SET MASTER", - "START SLAVE", + "START replica", } - goodSlave1.StartActionLoop(t, wr) - defer goodSlave1.StopActionLoop(t) + goodReplica1.StartActionLoop(t, wr) + defer goodReplica1.StopActionLoop(t) - // good slave 2 is not replicating - goodSlave2.FakeMysqlDaemon.ReadOnly = true - goodSlave2.FakeMysqlDaemon.Replicating = false - goodSlave2.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) - goodSlave2.StartActionLoop(t, wr) - goodSlave2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + // good replica 2 is not replicating + goodReplica2.FakeMysqlDaemon.ReadOnly = true + goodReplica2.FakeMysqlDaemon.Replicating = false + goodReplica2.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) + goodReplica2.StartActionLoop(t, wr) + goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", } - defer goodSlave2.StopActionLoop(t) + defer goodReplica2.StopActionLoop(t) // run PlannedReparentShard err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(newMaster.Tablet.Alias)}) if err == nil { - t.Fatalf("PlannedReparentShard succeeded: %v", err) + t.Fatal("PlannedReparentShard succeeded") } if !strings.Contains(err.Error(), "replication on master-elect cell1-0000000001 did not catch up in time") { t.Fatalf("PlannedReparentShard failed with the wrong error: %v", err) @@ -502,9 +518,9 @@ func TestPlannedReparentShardRelayLogError(t *testing.T) { vp := NewVtctlPipe(t, ts) defer vp.Close() - // Create a master, a couple good slaves + // Create a master, a couple good replicas master := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, nil) - goodSlave1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) + goodReplica1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) // old master master.FakeMysqlDaemon.ReadOnly = false @@ -528,19 +544,19 @@ func TestPlannedReparentShardRelayLogError(t *testing.T) { defer master.StopActionLoop(t) master.Agent.QueryServiceControl.(*tabletservermock.Controller).SetQueryServiceEnabledForTests(true) - // good slave 1 is replicating - goodSlave1.FakeMysqlDaemon.ReadOnly = true - goodSlave1.FakeMysqlDaemon.Replicating = true - goodSlave1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(master.Tablet) + // goodReplica1 is replicating + goodReplica1.FakeMysqlDaemon.ReadOnly = true + goodReplica1.FakeMysqlDaemon.Replicating = true + goodReplica1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(master.Tablet) // simulate error that will trigger a call to RestartSlave - goodSlave1.FakeMysqlDaemon.SetMasterError = errors.New("Slave failed to initialize relay log info structure from the repository") - goodSlave1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + goodReplica1.FakeMysqlDaemon.SetMasterError = errors.New("Slave failed to initialize relay log info structure from the repository") + goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", "RESET SLAVE", "START SLAVE", } - goodSlave1.StartActionLoop(t, wr) - defer goodSlave1.StopActionLoop(t) + goodReplica1.StartActionLoop(t, wr) + defer goodReplica1.StopActionLoop(t) // run PlannedReparentShard if err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", master.Tablet.Keyspace + "/" + master.Tablet.Shard, "-new_master", topoproto.TabletAliasString(master.Tablet.Alias)}); err != nil { @@ -551,23 +567,23 @@ func TestPlannedReparentShardRelayLogError(t *testing.T) { if err := master.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { t.Errorf("master.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } - if err := goodSlave1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("goodSlave1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) + if err := goodReplica1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { + t.Errorf("goodReplica1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } if master.FakeMysqlDaemon.ReadOnly { t.Errorf("master.FakeMysqlDaemon.ReadOnly set") } - if !goodSlave1.FakeMysqlDaemon.ReadOnly { - t.Errorf("goodSlave1.FakeMysqlDaemon.ReadOnly not set") + if !goodReplica1.FakeMysqlDaemon.ReadOnly { + t.Errorf("goodReplica1.FakeMysqlDaemon.ReadOnly not set") } if !master.Agent.QueryServiceControl.IsServing() { t.Errorf("master...QueryServiceControl not serving") } // verify the old master was told to start replicating (and not - // the slave that wasn't replicating in the first place) - if !goodSlave1.FakeMysqlDaemon.Replicating { - t.Errorf("goodSlave1.FakeMysqlDaemon.Replicating not set") + // the replica that wasn't replicating in the first place) + if !goodReplica1.FakeMysqlDaemon.Replicating { + t.Errorf("goodReplica1.FakeMysqlDaemon.Replicating not set") } } @@ -578,9 +594,9 @@ func TestPlannedReparentShardRelayLogErrorStartSlave(t *testing.T) { vp := NewVtctlPipe(t, ts) defer vp.Close() - // Create a master, a couple good slaves + // Create a master, a couple good replicas master := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, nil) - goodSlave1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) + goodReplica1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) // old master master.FakeMysqlDaemon.ReadOnly = false @@ -604,22 +620,22 @@ func TestPlannedReparentShardRelayLogErrorStartSlave(t *testing.T) { defer master.StopActionLoop(t) master.Agent.QueryServiceControl.(*tabletservermock.Controller).SetQueryServiceEnabledForTests(true) - // good slave 1 is not replicating - goodSlave1.FakeMysqlDaemon.ReadOnly = true - goodSlave1.FakeMysqlDaemon.Replicating = true - goodSlave1.FakeMysqlDaemon.SlaveIORunning = false - goodSlave1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(master.Tablet) - goodSlave1.FakeMysqlDaemon.CurrentMasterHost = topoproto.MysqlHostname(master.Tablet) - goodSlave1.FakeMysqlDaemon.CurrentMasterPort = int(topoproto.MysqlPort(master.Tablet)) + // good replica 1 is not replicating + goodReplica1.FakeMysqlDaemon.ReadOnly = true + goodReplica1.FakeMysqlDaemon.Replicating = true + goodReplica1.FakeMysqlDaemon.SlaveIORunning = false + goodReplica1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(master.Tablet) + goodReplica1.FakeMysqlDaemon.CurrentMasterHost = topoproto.MysqlHostname(master.Tablet) + goodReplica1.FakeMysqlDaemon.CurrentMasterPort = int(topoproto.MysqlPort(master.Tablet)) // simulate error that will trigger a call to RestartSlave - goodSlave1.FakeMysqlDaemon.StartSlaveError = errors.New("Slave failed to initialize relay log info structure from the repository") - goodSlave1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + goodReplica1.FakeMysqlDaemon.StartSlaveError = errors.New("Slave failed to initialize relay log info structure from the repository") + goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", "RESET SLAVE", "START SLAVE", } - goodSlave1.StartActionLoop(t, wr) - defer goodSlave1.StopActionLoop(t) + goodReplica1.StartActionLoop(t, wr) + defer goodReplica1.StopActionLoop(t) // run PlannedReparentShard if err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", master.Tablet.Keyspace + "/" + master.Tablet.Shard, "-new_master", topoproto.TabletAliasString(master.Tablet.Alias)}); err != nil { @@ -630,23 +646,217 @@ func TestPlannedReparentShardRelayLogErrorStartSlave(t *testing.T) { if err := master.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { t.Errorf("master.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } - if err := goodSlave1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("goodSlave1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) + if err := goodReplica1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { + t.Errorf("goodReplica1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) } if master.FakeMysqlDaemon.ReadOnly { t.Errorf("master.FakeMysqlDaemon.ReadOnly set") } - if !goodSlave1.FakeMysqlDaemon.ReadOnly { - t.Errorf("goodSlave1.FakeMysqlDaemon.ReadOnly not set") + if !goodReplica1.FakeMysqlDaemon.ReadOnly { + t.Errorf("goodReplica1.FakeMysqlDaemon.ReadOnly not set") } if !master.Agent.QueryServiceControl.IsServing() { t.Errorf("master...QueryServiceControl not serving") } // verify the old master was told to start replicating (and not - // the slave that wasn't replicating in the first place) - if !goodSlave1.FakeMysqlDaemon.Replicating { - t.Errorf("goodSlave1.FakeMysqlDaemon.Replicating not set") + // the replica that wasn't replicating in the first place) + if !goodReplica1.FakeMysqlDaemon.Replicating { + t.Errorf("goodReplica1.FakeMysqlDaemon.Replicating not set") + } +} + +// TestPlannedReparentShardPromoteReplicaFail simulates a failure of the PromoteReplica call +// on the desired new master tablet +func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { + ts := memorytopo.NewServer("cell1", "cell2") + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + vp := NewVtctlPipe(t, ts) + defer vp.Close() + + // Create a master, a couple good replicas + oldMaster := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, nil) + newMaster := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, nil) + goodReplica1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) + goodReplica2 := NewFakeTablet(t, wr, "cell2", 3, topodatapb.TabletType_REPLICA, nil) + + // new master + newMaster.FakeMysqlDaemon.ReadOnly = true + newMaster.FakeMysqlDaemon.Replicating = true + // make promote fail + newMaster.FakeMysqlDaemon.PromoteError = errors.New("some error") + newMaster.FakeMysqlDaemon.WaitMasterPosition = mysql.Position{ + GTIDSet: mysql.MariadbGTIDSet{ + mysql.MariadbGTID{ + Domain: 7, + Server: 123, + Sequence: 990, + }, + }, + } + newMaster.FakeMysqlDaemon.PromoteResult = mysql.Position{ + GTIDSet: mysql.MariadbGTIDSet{ + mysql.MariadbGTID{ + Domain: 7, + Server: 456, + Sequence: 991, + }, + }, } + newMaster.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "STOP SLAVE", + "FAKE SET MASTER", + "START SLAVE", + "CREATE DATABASE IF NOT EXISTS _vt", + "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", + "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES", + } + newMaster.StartActionLoop(t, wr) + defer newMaster.StopActionLoop(t) + // old master + oldMaster.FakeMysqlDaemon.ReadOnly = false + oldMaster.FakeMysqlDaemon.Replicating = false + oldMaster.FakeMysqlDaemon.SlaveStatusError = mysql.ErrNotSlave + oldMaster.FakeMysqlDaemon.CurrentMasterPosition = newMaster.FakeMysqlDaemon.WaitMasterPosition + oldMaster.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) + oldMaster.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "FAKE SET MASTER", + "START SLAVE", + } + oldMaster.StartActionLoop(t, wr) + defer oldMaster.StopActionLoop(t) + oldMaster.Agent.QueryServiceControl.(*tabletservermock.Controller).SetQueryServiceEnabledForTests(true) + + // SetMaster is called on new master to make sure it's replicating before reparenting. + newMaster.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(oldMaster.Tablet) + // good replica 1 is replicating + goodReplica1.FakeMysqlDaemon.ReadOnly = true + goodReplica1.FakeMysqlDaemon.Replicating = true + goodReplica1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) + goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "STOP SLAVE", + "FAKE SET MASTER", + "START SLAVE", + } + goodReplica1.StartActionLoop(t, wr) + defer goodReplica1.StopActionLoop(t) + + // good replica 2 is not replicating + goodReplica2.FakeMysqlDaemon.ReadOnly = true + goodReplica2.FakeMysqlDaemon.Replicating = false + goodReplica2.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(newMaster.Tablet) + goodReplica2.StartActionLoop(t, wr) + goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "FAKE SET MASTER", + } + defer goodReplica2.StopActionLoop(t) + + // run PlannedReparentShard + err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(newMaster.Tablet.Alias)}) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "some error") + + // when promote fails, we don't call UndoDemoteMaster, so the old master should be read-only + assert.True(t, newMaster.FakeMysqlDaemon.ReadOnly, "newMaster.FakeMysqlDaemon.ReadOnly") + assert.True(t, oldMaster.FakeMysqlDaemon.ReadOnly, "oldMaster.FakeMysqlDaemon.ReadOnly") + + // retrying should work + newMaster.FakeMysqlDaemon.PromoteError = nil + newMaster.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "STOP SLAVE", + "FAKE SET MASTER", + "START SLAVE", + // extra commands because of retry + "STOP SLAVE", + "FAKE SET MASTER", + "START SLAVE", + "CREATE DATABASE IF NOT EXISTS _vt", + "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", + "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES", + } + oldMaster.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "FAKE SET MASTER", + "START SLAVE", + // extra commands because of retry + "FAKE SET MASTER", + "START SLAVE", + } + + // run PlannedReparentShard + err = vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(newMaster.Tablet.Alias)}) + assert.NoError(t, err) + + // check that mastership changed correctly + assert.False(t, newMaster.FakeMysqlDaemon.ReadOnly, "newMaster.FakeMysqlDaemon.ReadOnly") + assert.True(t, oldMaster.FakeMysqlDaemon.ReadOnly, "oldMaster.FakeMysqlDaemon.ReadOnly") +} + +// TestPlannedReparentShardSameMaster tests PRS with oldMaster works correctly +// Simulate failure of previous PRS and oldMaster is ReadOnly +// Verify that master correctly gets set to ReadWrite +func TestPlannedReparentShardSameMaster(t *testing.T) { + ts := memorytopo.NewServer("cell1", "cell2") + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + vp := NewVtctlPipe(t, ts) + defer vp.Close() + + // Create a master, a couple good replicas + oldMaster := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_MASTER, nil) + goodReplica1 := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) + goodReplica2 := NewFakeTablet(t, wr, "cell2", 3, topodatapb.TabletType_REPLICA, nil) + + // old master + oldMaster.FakeMysqlDaemon.ReadOnly = true + oldMaster.FakeMysqlDaemon.Replicating = false + oldMaster.FakeMysqlDaemon.SlaveStatusError = mysql.ErrNotSlave + oldMaster.FakeMysqlDaemon.CurrentMasterPosition = mysql.Position{ + GTIDSet: mysql.MariadbGTIDSet{ + mysql.MariadbGTID{ + Domain: 7, + Server: 123, + Sequence: 990, + }, + }, + } + oldMaster.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "CREATE DATABASE IF NOT EXISTS _vt", + "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", + "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES", + } + oldMaster.StartActionLoop(t, wr) + defer oldMaster.StopActionLoop(t) + oldMaster.Agent.QueryServiceControl.(*tabletservermock.Controller).SetQueryServiceEnabledForTests(true) + + // good replica 1 is replicating + goodReplica1.FakeMysqlDaemon.ReadOnly = true + goodReplica1.FakeMysqlDaemon.Replicating = true + goodReplica1.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(oldMaster.Tablet) + goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "STOP SLAVE", + "FAKE SET MASTER", + "START SLAVE", + } + goodReplica1.StartActionLoop(t, wr) + defer goodReplica1.StopActionLoop(t) + + // goodReplica2 is not replicating + goodReplica2.FakeMysqlDaemon.ReadOnly = true + goodReplica2.FakeMysqlDaemon.Replicating = false + goodReplica2.FakeMysqlDaemon.SetMasterInput = topoproto.MysqlAddr(oldMaster.Tablet) + goodReplica2.StartActionLoop(t, wr) + goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "FAKE SET MASTER", + } + defer goodReplica2.StopActionLoop(t) + + // run PlannedReparentShard + if err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", oldMaster.Tablet.Keyspace + "/" + oldMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(oldMaster.Tablet.Alias)}); err != nil { + t.Fatalf("PlannedReparent failed: %v", err) + } + + if oldMaster.FakeMysqlDaemon.ReadOnly { + t.Errorf("oldMaster.FakeMysqlDaemon.ReadOnly set") + } } From bcebd61a6a6acde830749f119c1ded2d19e58aa8 Mon Sep 17 00:00:00 2001 From: deepthi Date: Tue, 14 Apr 2020 17:42:41 -0700 Subject: [PATCH 5/6] use assertions in tests instead of if/else Signed-off-by: deepthi --- .../fakemysqldaemon/fakemysqldaemon.go | 4 +- .../testlib/emergency_reparent_shard_test.go | 73 +++--- .../testlib/planned_reparent_shard_test.go | 226 ++++++------------ go/vt/wrangler/testlib/semi_sync_test.go | 12 +- 4 files changed, 107 insertions(+), 208 deletions(-) diff --git a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go index 0d0fea3f051..e95f217f437 100644 --- a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go +++ b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go @@ -487,9 +487,9 @@ func (fmd *FakeMysqlDaemon) GetAllPrivsConnection() (*dbconnpool.DBConnection, e } // SetSemiSyncEnabled is part of the MysqlDaemon interface. -func (fmd *FakeMysqlDaemon) SetSemiSyncEnabled(master, slave bool) error { +func (fmd *FakeMysqlDaemon) SetSemiSyncEnabled(master, replica bool) error { fmd.SemiSyncMasterEnabled = master - fmd.SemiSyncSlaveEnabled = slave + fmd.SemiSyncSlaveEnabled = replica return nil } diff --git a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go index 5fd63be499c..55acd1aa567 100644 --- a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go @@ -17,10 +17,11 @@ limitations under the License. package testlib import ( - "strings" "testing" "time" + "github.com/stretchr/testify/assert" + "golang.org/x/net/context" "vitess.io/vitess/go/mysql" @@ -121,39 +122,25 @@ func TestEmergencyReparentShard(t *testing.T) { defer goodReplica2.StopActionLoop(t) // run EmergencyReparentShard - if err := vp.Run([]string{"EmergencyReparentShard", "-wait_slave_timeout", "10s", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, topoproto.TabletAliasString(newMaster.Tablet.Alias)}); err != nil { - t.Fatalf("EmergencyReparentShard failed: %v", err) - } - + err := vp.Run([]string{"EmergencyReparentShard", "-wait_slave_timeout", "10s", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, + topoproto.TabletAliasString(newMaster.Tablet.Alias)}) + assert.NoError(t, err) // check what was run - if err := newMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Fatalf("newMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if err := oldMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Fatalf("oldMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if err := goodReplica1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Fatalf("goodReplica1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if err := goodReplica2.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Fatalf("goodReplica2.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if newMaster.FakeMysqlDaemon.ReadOnly { - t.Errorf("newMaster.FakeMysqlDaemon.ReadOnly set") - } + err = newMaster.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + err = oldMaster.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + err = goodReplica1.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + err = goodReplica2.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + + assert.False(t, newMaster.FakeMysqlDaemon.ReadOnly, "newMaster.FakeMysqlDaemon.ReadOnly set") // old master read-only flag doesn't matter, it is scrapped - if !goodReplica1.FakeMysqlDaemon.ReadOnly { - t.Errorf("goodReplica1.FakeMysqlDaemon.ReadOnly not set") - } - if !goodReplica2.FakeMysqlDaemon.ReadOnly { - t.Errorf("goodReplica2.FakeMysqlDaemon.ReadOnly not set") - } - if !goodReplica1.FakeMysqlDaemon.Replicating { - t.Errorf("goodReplica1.FakeMysqlDaemon.Replicating not set") - } - if goodReplica2.FakeMysqlDaemon.Replicating { - t.Errorf("goodReplica2.FakeMysqlDaemon.Replicating set") - } + assert.True(t, goodReplica1.FakeMysqlDaemon.ReadOnly, "goodReplica1.FakeMysqlDaemon.ReadOnly not set") + assert.True(t, goodReplica2.FakeMysqlDaemon.ReadOnly, "goodReplica2.FakeMysqlDaemon.ReadOnly not set") + assert.True(t, goodReplica1.FakeMysqlDaemon.Replicating, "goodReplica1.FakeMysqlDaemon.Replicating not set") + assert.False(t, goodReplica2.FakeMysqlDaemon.Replicating, "goodReplica2.FakeMysqlDaemon.Replicating set") checkSemiSyncEnabled(t, true, true, newMaster) checkSemiSyncEnabled(t, false, true, goodReplica1, goodReplica2) } @@ -232,18 +219,14 @@ func TestEmergencyReparentShardMasterElectNotBest(t *testing.T) { defer moreAdvancedReplica.StopActionLoop(t) // run EmergencyReparentShard - if err := wr.EmergencyReparentShard(ctx, newMaster.Tablet.Keyspace, newMaster.Tablet.Shard, newMaster.Tablet.Alias, 10*time.Second); err == nil || !strings.Contains(err.Error(), "is more advanced than master elect tablet") { - t.Fatalf("EmergencyReparentShard returned the wrong error: %v", err) - } - + err := wr.EmergencyReparentShard(ctx, newMaster.Tablet.Keyspace, newMaster.Tablet.Shard, newMaster.Tablet.Alias, 10*time.Second) + assert.Error(t, err) + assert.Contains(t, err.Error(), "is more advanced than master elect tablet") // check what was run - if err := newMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Fatalf("newMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if err := oldMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Fatalf("oldMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if err := moreAdvancedReplica.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Fatalf("moreAdvancedReplica.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } + err = newMaster.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + err = oldMaster.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + err = moreAdvancedReplica.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) } diff --git a/go/vt/wrangler/testlib/planned_reparent_shard_test.go b/go/vt/wrangler/testlib/planned_reparent_shard_test.go index dd2dc09d5c5..0587550b023 100644 --- a/go/vt/wrangler/testlib/planned_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/planned_reparent_shard_test.go @@ -19,7 +19,6 @@ package testlib import ( "context" "errors" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -111,41 +110,27 @@ func TestPlannedReparentShardNoMasterProvided(t *testing.T) { defer goodReplica1.StopActionLoop(t) // run PlannedReparentShard - if err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard}); err != nil { - t.Fatalf("PlannedReparentShard failed: %v", err) - } + err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard}) + assert.NoError(t, err) // check what was run - if err := newMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("newMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if err := oldMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("oldMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if err := goodReplica1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("goodReplica1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if newMaster.FakeMysqlDaemon.ReadOnly { - t.Errorf("newMaster.FakeMysqlDaemon.ReadOnly set") - } - if !oldMaster.FakeMysqlDaemon.ReadOnly { - t.Errorf("oldMaster.FakeMysqlDaemon.ReadOnly not set") - } - if !goodReplica1.FakeMysqlDaemon.ReadOnly { - t.Errorf("goodReplica1.FakeMysqlDaemon.ReadOnly not set") - } - if !oldMaster.Agent.QueryServiceControl.IsServing() { - t.Errorf("oldMaster...QueryServiceControl not serving") - } + err = newMaster.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + + err = oldMaster.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + + err = goodReplica1.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + assert.False(t, newMaster.FakeMysqlDaemon.ReadOnly, "newMaster.FakeMysqlDaemon.ReadOnly is set") + assert.True(t, oldMaster.FakeMysqlDaemon.ReadOnly, "oldMaster.FakeMysqlDaemon.ReadOnly not set") + assert.True(t, goodReplica1.FakeMysqlDaemon.ReadOnly, "goodReplica1.FakeMysqlDaemon.ReadOnly not set") + assert.True(t, oldMaster.Agent.QueryServiceControl.IsServing(), "oldMaster...QueryServiceControl not serving") // verify the old master was told to start replicating (and not // the replica that wasn't replicating in the first place) - if !oldMaster.FakeMysqlDaemon.Replicating { - t.Errorf("oldMaster.FakeMysqlDaemon.Replicating not set") - } - if !goodReplica1.FakeMysqlDaemon.Replicating { - t.Errorf("goodReplica1.FakeMysqlDaemon.Replicating not set") - } + assert.True(t, oldMaster.FakeMysqlDaemon.Replicating, "oldMaster.FakeMysqlDaemon.Replicating not set") + assert.True(t, goodReplica1.FakeMysqlDaemon.Replicating, "goodReplica1.FakeMysqlDaemon.Replicating not set") checkSemiSyncEnabled(t, true, true, newMaster) checkSemiSyncEnabled(t, false, true, goodReplica1, oldMaster) } @@ -237,50 +222,32 @@ func TestPlannedReparentShardNoError(t *testing.T) { defer goodReplica2.StopActionLoop(t) // run PlannedReparentShard - if err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(newMaster.Tablet.Alias)}); err != nil { - t.Fatalf("PlannedReparentShard failed: %v", err) - } + err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, "-new_master", + topoproto.TabletAliasString(newMaster.Tablet.Alias)}) + assert.NoError(t, err) // check what was run - if err := newMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("newMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if err := oldMaster.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("oldMaster.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if err := goodReplica1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("goodReplica1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if err := goodReplica2.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("goodReplica2.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if newMaster.FakeMysqlDaemon.ReadOnly { - t.Errorf("newMaster.FakeMysqlDaemon.ReadOnly set") - } - if !oldMaster.FakeMysqlDaemon.ReadOnly { - t.Errorf("oldMaster.FakeMysqlDaemon.ReadOnly not set") - } - if !goodReplica1.FakeMysqlDaemon.ReadOnly { - t.Errorf("goodReplica1.FakeMysqlDaemon.ReadOnly not set") - } - if !goodReplica2.FakeMysqlDaemon.ReadOnly { - t.Errorf("goodReplica2.FakeMysqlDaemon.ReadOnly not set") - } - if !oldMaster.Agent.QueryServiceControl.IsServing() { - t.Errorf("oldMaster...QueryServiceControl not serving") - } + err = newMaster.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + err = oldMaster.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + err = goodReplica1.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + err = goodReplica2.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + + assert.False(t, newMaster.FakeMysqlDaemon.ReadOnly, "newMaster.FakeMysqlDaemon.ReadOnly set") + assert.True(t, oldMaster.FakeMysqlDaemon.ReadOnly, "oldMaster.FakeMysqlDaemon.ReadOnly not set") + assert.True(t, goodReplica1.FakeMysqlDaemon.ReadOnly, "goodReplica1.FakeMysqlDaemon.ReadOnly not set") + + assert.True(t, goodReplica2.FakeMysqlDaemon.ReadOnly, "goodReplica2.FakeMysqlDaemon.ReadOnly not set") + assert.True(t, oldMaster.Agent.QueryServiceControl.IsServing(), "oldMaster...QueryServiceControl not serving") // verify the old master was told to start replicating (and not // the replica that wasn't replicating in the first place) - if !oldMaster.FakeMysqlDaemon.Replicating { - t.Errorf("oldMaster.FakeMysqlDaemon.Replicating not set") - } - if !goodReplica1.FakeMysqlDaemon.Replicating { - t.Errorf("goodReplica1.FakeMysqlDaemon.Replicating not set") - } - if goodReplica2.FakeMysqlDaemon.Replicating { - t.Errorf("goodReplica2.FakeMysqlDaemon.Replicating set") - } + assert.True(t, oldMaster.FakeMysqlDaemon.Replicating, "oldMaster.FakeMysqlDaemon.Replicating not set") + assert.True(t, goodReplica1.FakeMysqlDaemon.Replicating, "goodReplica1.FakeMysqlDaemon.Replicating not set") + assert.False(t, goodReplica2.FakeMysqlDaemon.Replicating, "goodReplica2.FakeMysqlDaemon.Replicating set") checkSemiSyncEnabled(t, true, true, newMaster) checkSemiSyncEnabled(t, false, true, goodReplica1, goodReplica2, oldMaster) @@ -298,12 +265,8 @@ func TestPlannedReparentNoMaster(t *testing.T) { NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, nil) err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", replica1.Tablet.Keyspace + "/" + replica1.Tablet.Shard, "-new_master", topoproto.TabletAliasString(replica1.Tablet.Alias)}) - if err == nil { - t.Fatalf("PlannedReparentShard succeeded: %v", err) - } - if !strings.Contains(err.Error(), "the shard has no master") { - t.Fatalf("PlannedReparentShard failed with the wrong error: %v", err) - } + assert.Error(t, err) + assert.Contains(t, err.Error(), "the shard has no master") } // TestPlannedReparentShardWaitForPositionFail simulates a failure of the WaitForPosition call @@ -392,21 +355,12 @@ func TestPlannedReparentShardWaitForPositionFail(t *testing.T) { // run PlannedReparentShard err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(newMaster.Tablet.Alias)}) - - if err == nil { - t.Fatal("PlannedReparentShard succeeded") - } - if !strings.Contains(err.Error(), "replication on master-elect cell1-0000000001 did not catch up in time") { - t.Fatalf("PlannedReparentShard failed with the wrong error: %v", err) - } + assert.Error(t, err) + assert.Contains(t, err.Error(), "replication on master-elect cell1-0000000001 did not catch up in time") // now check that DemoteMaster was undone and old master is still master - if !newMaster.FakeMysqlDaemon.ReadOnly { - t.Errorf("newMaster.FakeMysqlDaemon.ReadOnly not set") - } - if oldMaster.FakeMysqlDaemon.ReadOnly { - t.Errorf("oldMaster.FakeMysqlDaemon.ReadOnly set") - } + assert.True(t, newMaster.FakeMysqlDaemon.ReadOnly, "newMaster.FakeMysqlDaemon.ReadOnly not set") + assert.False(t, oldMaster.FakeMysqlDaemon.ReadOnly, "oldMaster.FakeMysqlDaemon.ReadOnly set") } // TestPlannedReparentShardWaitForPositionTimeout simulates a context timeout @@ -495,21 +449,12 @@ func TestPlannedReparentShardWaitForPositionTimeout(t *testing.T) { // run PlannedReparentShard err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(newMaster.Tablet.Alias)}) - - if err == nil { - t.Fatal("PlannedReparentShard succeeded") - } - if !strings.Contains(err.Error(), "replication on master-elect cell1-0000000001 did not catch up in time") { - t.Fatalf("PlannedReparentShard failed with the wrong error: %v", err) - } + assert.Error(t, err) + assert.Contains(t, err.Error(), "replication on master-elect cell1-0000000001 did not catch up in time") // now check that DemoteMaster was undone and old master is still master - if !newMaster.FakeMysqlDaemon.ReadOnly { - t.Errorf("newMaster.FakeMysqlDaemon.ReadOnly not set") - } - if oldMaster.FakeMysqlDaemon.ReadOnly { - t.Errorf("oldMaster.FakeMysqlDaemon.ReadOnly set") - } + assert.True(t, newMaster.FakeMysqlDaemon.ReadOnly, "newMaster.FakeMysqlDaemon.ReadOnly not set") + assert.False(t, oldMaster.FakeMysqlDaemon.ReadOnly, "oldMaster.FakeMysqlDaemon.ReadOnly set") } func TestPlannedReparentShardRelayLogError(t *testing.T) { @@ -559,33 +504,21 @@ func TestPlannedReparentShardRelayLogError(t *testing.T) { defer goodReplica1.StopActionLoop(t) // run PlannedReparentShard - if err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", master.Tablet.Keyspace + "/" + master.Tablet.Shard, "-new_master", topoproto.TabletAliasString(master.Tablet.Alias)}); err != nil { - t.Fatalf("PlannedReparentShard failed: %v", err) - } - + err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", master.Tablet.Keyspace + "/" + master.Tablet.Shard, "-new_master", + topoproto.TabletAliasString(master.Tablet.Alias)}) + assert.NoError(t, err) // check what was run - if err := master.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("master.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if err := goodReplica1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("goodReplica1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if master.FakeMysqlDaemon.ReadOnly { - t.Errorf("master.FakeMysqlDaemon.ReadOnly set") - } - if !goodReplica1.FakeMysqlDaemon.ReadOnly { - t.Errorf("goodReplica1.FakeMysqlDaemon.ReadOnly not set") - } - if !master.Agent.QueryServiceControl.IsServing() { - t.Errorf("master...QueryServiceControl not serving") - } + err = master.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + err = goodReplica1.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + assert.False(t, master.FakeMysqlDaemon.ReadOnly, "master.FakeMysqlDaemon.ReadOnly set") + assert.True(t, goodReplica1.FakeMysqlDaemon.ReadOnly, "goodReplica1.FakeMysqlDaemon.ReadOnly not set") + assert.True(t, master.Agent.QueryServiceControl.IsServing(), "master...QueryServiceControl not serving") // verify the old master was told to start replicating (and not // the replica that wasn't replicating in the first place) - if !goodReplica1.FakeMysqlDaemon.Replicating { - t.Errorf("goodReplica1.FakeMysqlDaemon.Replicating not set") - } - + assert.True(t, goodReplica1.FakeMysqlDaemon.Replicating, "goodReplica1.FakeMysqlDaemon.Replicating not set") } func TestPlannedReparentShardRelayLogErrorStartSlave(t *testing.T) { @@ -638,32 +571,21 @@ func TestPlannedReparentShardRelayLogErrorStartSlave(t *testing.T) { defer goodReplica1.StopActionLoop(t) // run PlannedReparentShard - if err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", master.Tablet.Keyspace + "/" + master.Tablet.Shard, "-new_master", topoproto.TabletAliasString(master.Tablet.Alias)}); err != nil { - t.Fatalf("PlannedReparentShard failed: %v", err) - } - + err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", master.Tablet.Keyspace + "/" + master.Tablet.Shard, "-new_master", + topoproto.TabletAliasString(master.Tablet.Alias)}) + assert.NoError(t, err) // check what was run - if err := master.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("master.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if err := goodReplica1.FakeMysqlDaemon.CheckSuperQueryList(); err != nil { - t.Errorf("goodReplica1.FakeMysqlDaemon.CheckSuperQueryList failed: %v", err) - } - if master.FakeMysqlDaemon.ReadOnly { - t.Errorf("master.FakeMysqlDaemon.ReadOnly set") - } - if !goodReplica1.FakeMysqlDaemon.ReadOnly { - t.Errorf("goodReplica1.FakeMysqlDaemon.ReadOnly not set") - } - if !master.Agent.QueryServiceControl.IsServing() { - t.Errorf("master...QueryServiceControl not serving") - } + err = master.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + err = goodReplica1.FakeMysqlDaemon.CheckSuperQueryList() + assert.NoError(t, err) + assert.False(t, master.FakeMysqlDaemon.ReadOnly, "master.FakeMysqlDaemon.ReadOnly set") + assert.True(t, goodReplica1.FakeMysqlDaemon.ReadOnly, "goodReplica1.FakeMysqlDaemon.ReadOnly not set") + assert.True(t, master.Agent.QueryServiceControl.IsServing(), "master...QueryServiceControl not serving") // verify the old master was told to start replicating (and not // the replica that wasn't replicating in the first place) - if !goodReplica1.FakeMysqlDaemon.Replicating { - t.Errorf("goodReplica1.FakeMysqlDaemon.Replicating not set") - } + assert.True(t, goodReplica1.FakeMysqlDaemon.Replicating, "goodReplica1.FakeMysqlDaemon.Replicating not set") } // TestPlannedReparentShardPromoteReplicaFail simulates a failure of the PromoteReplica call @@ -852,11 +774,7 @@ func TestPlannedReparentShardSameMaster(t *testing.T) { defer goodReplica2.StopActionLoop(t) // run PlannedReparentShard - if err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", oldMaster.Tablet.Keyspace + "/" + oldMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(oldMaster.Tablet.Alias)}); err != nil { - t.Fatalf("PlannedReparent failed: %v", err) - } - - if oldMaster.FakeMysqlDaemon.ReadOnly { - t.Errorf("oldMaster.FakeMysqlDaemon.ReadOnly set") - } + err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", oldMaster.Tablet.Keyspace + "/" + oldMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(oldMaster.Tablet.Alias)}) + assert.NoError(t, err) + assert.False(t, oldMaster.FakeMysqlDaemon.ReadOnly, "oldMaster.FakeMysqlDaemon.ReadOnly") } diff --git a/go/vt/wrangler/testlib/semi_sync_test.go b/go/vt/wrangler/testlib/semi_sync_test.go index d668bac160c..cf8587e3f3f 100644 --- a/go/vt/wrangler/testlib/semi_sync_test.go +++ b/go/vt/wrangler/testlib/semi_sync_test.go @@ -20,6 +20,8 @@ import ( "flag" "testing" + "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/vt/topo/topoproto" ) @@ -28,13 +30,9 @@ func init() { flag.Set("enable_semi_sync", "true") } -func checkSemiSyncEnabled(t *testing.T, master, slave bool, tablets ...*FakeTablet) { +func checkSemiSyncEnabled(t *testing.T, master, replica bool, tablets ...*FakeTablet) { for _, tablet := range tablets { - if got, want := tablet.FakeMysqlDaemon.SemiSyncMasterEnabled, master; got != want { - t.Errorf("%v: SemiSyncMasterEnabled = %v, want %v", topoproto.TabletAliasString(tablet.Tablet.Alias), got, want) - } - if got, want := tablet.FakeMysqlDaemon.SemiSyncSlaveEnabled, slave; got != want { - t.Errorf("%v: SemiSyncSlaveEnabled = %v, want %v", topoproto.TabletAliasString(tablet.Tablet.Alias), got, want) - } + assert.Equal(t, master, tablet.FakeMysqlDaemon.SemiSyncMasterEnabled, "%v: SemiSyncMasterEnabled", topoproto.TabletAliasString(tablet.Tablet.Alias)) + assert.Equal(t, replica, tablet.FakeMysqlDaemon.SemiSyncSlaveEnabled, "%v: SemiSyncSlaveEnabled", topoproto.TabletAliasString(tablet.Tablet.Alias)) } } From 54480bdb6fa20c91b1317ebddc7ae82371d55702 Mon Sep 17 00:00:00 2001 From: deepthi Date: Tue, 14 Apr 2020 20:20:43 -0700 Subject: [PATCH 6/6] Use require to stop execution of tests on fatal errors Signed-off-by: deepthi --- .../testlib/emergency_reparent_shard_test.go | 20 +++++----- .../testlib/planned_reparent_shard_test.go | 38 ++++++++++--------- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go index 55acd1aa567..25399022873 100644 --- a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go @@ -17,12 +17,12 @@ limitations under the License. package testlib import ( + "context" "testing" "time" "github.com/stretchr/testify/assert" - - "golang.org/x/net/context" + "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/vt/logutil" @@ -124,16 +124,16 @@ func TestEmergencyReparentShard(t *testing.T) { // run EmergencyReparentShard err := vp.Run([]string{"EmergencyReparentShard", "-wait_slave_timeout", "10s", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, topoproto.TabletAliasString(newMaster.Tablet.Alias)}) - assert.NoError(t, err) + require.NoError(t, err) // check what was run err = newMaster.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) err = oldMaster.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) err = goodReplica1.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) err = goodReplica2.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) assert.False(t, newMaster.FakeMysqlDaemon.ReadOnly, "newMaster.FakeMysqlDaemon.ReadOnly set") // old master read-only flag doesn't matter, it is scrapped @@ -224,9 +224,9 @@ func TestEmergencyReparentShardMasterElectNotBest(t *testing.T) { assert.Contains(t, err.Error(), "is more advanced than master elect tablet") // check what was run err = newMaster.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) err = oldMaster.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) err = moreAdvancedReplica.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) } diff --git a/go/vt/wrangler/testlib/planned_reparent_shard_test.go b/go/vt/wrangler/testlib/planned_reparent_shard_test.go index 0587550b023..2ac323ff116 100644 --- a/go/vt/wrangler/testlib/planned_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/planned_reparent_shard_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/vt/logutil" @@ -111,17 +112,18 @@ func TestPlannedReparentShardNoMasterProvided(t *testing.T) { // run PlannedReparentShard err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard}) - assert.NoError(t, err) + require.NoError(t, err) // check what was run err = newMaster.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) err = oldMaster.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) err = goodReplica1.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) + assert.False(t, newMaster.FakeMysqlDaemon.ReadOnly, "newMaster.FakeMysqlDaemon.ReadOnly is set") assert.True(t, oldMaster.FakeMysqlDaemon.ReadOnly, "oldMaster.FakeMysqlDaemon.ReadOnly not set") assert.True(t, goodReplica1.FakeMysqlDaemon.ReadOnly, "goodReplica1.FakeMysqlDaemon.ReadOnly not set") @@ -224,17 +226,17 @@ func TestPlannedReparentShardNoError(t *testing.T) { // run PlannedReparentShard err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(newMaster.Tablet.Alias)}) - assert.NoError(t, err) + require.NoError(t, err) // check what was run err = newMaster.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) err = oldMaster.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) err = goodReplica1.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) err = goodReplica2.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) assert.False(t, newMaster.FakeMysqlDaemon.ReadOnly, "newMaster.FakeMysqlDaemon.ReadOnly set") assert.True(t, oldMaster.FakeMysqlDaemon.ReadOnly, "oldMaster.FakeMysqlDaemon.ReadOnly not set") @@ -506,12 +508,13 @@ func TestPlannedReparentShardRelayLogError(t *testing.T) { // run PlannedReparentShard err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", master.Tablet.Keyspace + "/" + master.Tablet.Shard, "-new_master", topoproto.TabletAliasString(master.Tablet.Alias)}) - assert.NoError(t, err) + require.NoError(t, err) // check what was run err = master.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) err = goodReplica1.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) + assert.False(t, master.FakeMysqlDaemon.ReadOnly, "master.FakeMysqlDaemon.ReadOnly set") assert.True(t, goodReplica1.FakeMysqlDaemon.ReadOnly, "goodReplica1.FakeMysqlDaemon.ReadOnly not set") assert.True(t, master.Agent.QueryServiceControl.IsServing(), "master...QueryServiceControl not serving") @@ -573,12 +576,13 @@ func TestPlannedReparentShardRelayLogErrorStartSlave(t *testing.T) { // run PlannedReparentShard err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", master.Tablet.Keyspace + "/" + master.Tablet.Shard, "-new_master", topoproto.TabletAliasString(master.Tablet.Alias)}) - assert.NoError(t, err) + require.NoError(t, err) // check what was run err = master.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) err = goodReplica1.FakeMysqlDaemon.CheckSuperQueryList() - assert.NoError(t, err) + require.NoError(t, err) + assert.False(t, master.FakeMysqlDaemon.ReadOnly, "master.FakeMysqlDaemon.ReadOnly set") assert.True(t, goodReplica1.FakeMysqlDaemon.ReadOnly, "goodReplica1.FakeMysqlDaemon.ReadOnly not set") assert.True(t, master.Agent.QueryServiceControl.IsServing(), "master...QueryServiceControl not serving") @@ -708,7 +712,7 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { // run PlannedReparentShard err = vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(newMaster.Tablet.Alias)}) - assert.NoError(t, err) + require.NoError(t, err) // check that mastership changed correctly assert.False(t, newMaster.FakeMysqlDaemon.ReadOnly, "newMaster.FakeMysqlDaemon.ReadOnly") @@ -775,6 +779,6 @@ func TestPlannedReparentShardSameMaster(t *testing.T) { // run PlannedReparentShard err := vp.Run([]string{"PlannedReparentShard", "-wait_slave_timeout", "10s", "-keyspace_shard", oldMaster.Tablet.Keyspace + "/" + oldMaster.Tablet.Shard, "-new_master", topoproto.TabletAliasString(oldMaster.Tablet.Alias)}) - assert.NoError(t, err) + require.NoError(t, err) assert.False(t, oldMaster.FakeMysqlDaemon.ReadOnly, "oldMaster.FakeMysqlDaemon.ReadOnly") }