From 986a19817a80efdff098486dd52f5a58e403cc3b Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Fri, 16 Jul 2021 11:37:15 +0530 Subject: [PATCH 001/176] remove unused constants Signed-off-by: GuptaManan100 --- go/vt/wrangler/reparent.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index 7b8357ebe11..515306ad804 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -43,12 +43,6 @@ import ( vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" ) -const ( - plannedReparentShardOperation = "PlannedReparentShard" //nolint - emergencyReparentShardOperation = "EmergencyReparentShard" //nolint - tabletExternallyReparentedOperation = "TabletExternallyReparented" //nolint -) - // ShardReplicationStatuses returns the ReplicationStatus for each tablet in a shard. func (wr *Wrangler) ShardReplicationStatuses(ctx context.Context, keyspace, shard string) ([]*topo.TabletInfo, []*replicationdatapb.Status, error) { tabletMap, err := wr.ts.GetTabletMapForShard(ctx, keyspace, shard) From 0beabaae4d3c749615985f5d18078e0717f4bac7 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 19 Jul 2021 10:12:23 +0530 Subject: [PATCH 002/176] moved reparent operations to vtorc Signed-off-by: GuptaManan100 --- .../reparentutil/emergency_reparenter.go | 0 .../reparentutil/emergency_reparenter_test.go | 0 .../{vtctl => orchestrator}/reparentutil/planned_reparenter.go | 0 .../reparentutil/planned_reparenter_test.go | 0 go/vt/{vtctl => orchestrator}/reparentutil/replication.go | 0 go/vt/{vtctl => orchestrator}/reparentutil/replication_test.go | 0 go/vt/{vtctl => orchestrator}/reparentutil/util.go | 0 go/vt/{vtctl => orchestrator}/reparentutil/util_test.go | 0 go/vt/vtctl/grpcvtctldserver/server.go | 2 +- go/vt/wrangler/reparent.go | 2 +- 10 files changed, 2 insertions(+), 2 deletions(-) rename go/vt/{vtctl => orchestrator}/reparentutil/emergency_reparenter.go (100%) rename go/vt/{vtctl => orchestrator}/reparentutil/emergency_reparenter_test.go (100%) rename go/vt/{vtctl => orchestrator}/reparentutil/planned_reparenter.go (100%) rename go/vt/{vtctl => orchestrator}/reparentutil/planned_reparenter_test.go (100%) rename go/vt/{vtctl => orchestrator}/reparentutil/replication.go (100%) rename go/vt/{vtctl => orchestrator}/reparentutil/replication_test.go (100%) rename go/vt/{vtctl => orchestrator}/reparentutil/util.go (100%) rename go/vt/{vtctl => orchestrator}/reparentutil/util_test.go (100%) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/orchestrator/reparentutil/emergency_reparenter.go similarity index 100% rename from go/vt/vtctl/reparentutil/emergency_reparenter.go rename to go/vt/orchestrator/reparentutil/emergency_reparenter.go diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/orchestrator/reparentutil/emergency_reparenter_test.go similarity index 100% rename from go/vt/vtctl/reparentutil/emergency_reparenter_test.go rename to go/vt/orchestrator/reparentutil/emergency_reparenter_test.go diff --git a/go/vt/vtctl/reparentutil/planned_reparenter.go b/go/vt/orchestrator/reparentutil/planned_reparenter.go similarity index 100% rename from go/vt/vtctl/reparentutil/planned_reparenter.go rename to go/vt/orchestrator/reparentutil/planned_reparenter.go diff --git a/go/vt/vtctl/reparentutil/planned_reparenter_test.go b/go/vt/orchestrator/reparentutil/planned_reparenter_test.go similarity index 100% rename from go/vt/vtctl/reparentutil/planned_reparenter_test.go rename to go/vt/orchestrator/reparentutil/planned_reparenter_test.go diff --git a/go/vt/vtctl/reparentutil/replication.go b/go/vt/orchestrator/reparentutil/replication.go similarity index 100% rename from go/vt/vtctl/reparentutil/replication.go rename to go/vt/orchestrator/reparentutil/replication.go diff --git a/go/vt/vtctl/reparentutil/replication_test.go b/go/vt/orchestrator/reparentutil/replication_test.go similarity index 100% rename from go/vt/vtctl/reparentutil/replication_test.go rename to go/vt/orchestrator/reparentutil/replication_test.go diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/orchestrator/reparentutil/util.go similarity index 100% rename from go/vt/vtctl/reparentutil/util.go rename to go/vt/orchestrator/reparentutil/util.go diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/orchestrator/reparentutil/util_test.go similarity index 100% rename from go/vt/vtctl/reparentutil/util_test.go rename to go/vt/orchestrator/reparentutil/util_test.go diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 83f9f094dcf..edac1106660 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -38,12 +38,12 @@ import ( "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" "vitess.io/vitess/go/vt/mysqlctl/mysqlctlproto" + "vitess.io/vitess/go/vt/orchestrator/reparentutil" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/vt/topotools/events" - "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/vtctl/workflow" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vttablet/tmclient" diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index 515306ad804..071b60ae2c8 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -31,12 +31,12 @@ import ( "vitess.io/vitess/go/event" "vitess.io/vitess/go/vt/concurrency" "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/orchestrator/reparentutil" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/vt/topotools/events" "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver" - "vitess.io/vitess/go/vt/vtctl/reparentutil" replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" From 53d1829297d246ccb2d3a9d624abef1fc7fb6254 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 19 Jul 2021 14:53:40 +0530 Subject: [PATCH 003/176] start of adding common emergencyReparenter Signed-off-by: GuptaManan100 --- go/vt/orchestrator/logic/tablet_discovery.go | 16 ++-- go/vt/orchestrator/logic/topology_recovery.go | 9 ++- .../reparentutil/emergency_reparenter_2.go | 76 +++++++++++++++++++ go/vt/orchestrator/reparentutil/util.go | 64 ++++++++++++++++ 4 files changed, 153 insertions(+), 12 deletions(-) create mode 100644 go/vt/orchestrator/reparentutil/emergency_reparenter_2.go diff --git a/go/vt/orchestrator/logic/tablet_discovery.go b/go/vt/orchestrator/logic/tablet_discovery.go index d62c51ef566..44c271628ee 100644 --- a/go/vt/orchestrator/logic/tablet_discovery.go +++ b/go/vt/orchestrator/logic/tablet_discovery.go @@ -239,28 +239,28 @@ func refreshTablets(tablets map[string]*topo.TabletInfo, query string, args []in } // LockShard locks the keyspace-shard preventing others from performing conflicting actions. -func LockShard(instanceKey inst.InstanceKey) (func(*error), error) { +func LockShard(ctx context.Context, instanceKey inst.InstanceKey) (context.Context, func(*error), error) { if instanceKey.Hostname == "" { - return nil, errors.New("Can't lock shard: instance is unspecified") + return nil, nil, errors.New("Can't lock shard: instance is unspecified") } val := atomic.LoadInt32(&hasReceivedSIGTERM) if val > 0 { - return nil, errors.New("Can't lock shard: SIGTERM received") + return nil, nil, errors.New("Can't lock shard: SIGTERM received") } tablet, err := inst.ReadTablet(instanceKey) if err != nil { - return nil, err + return nil, nil, err } - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(config.Config.LockShardTimeoutSeconds)*time.Second) + ctx, cancel := context.WithTimeout(ctx, time.Duration(config.Config.LockShardTimeoutSeconds)*time.Second) defer cancel() atomic.AddInt32(&shardsLockCounter, 1) - _, unlock, err := ts.LockShard(ctx, tablet.Keyspace, tablet.Shard, "Orc Recovery") + ctx, unlock, err := ts.LockShard(ctx, tablet.Keyspace, tablet.Shard, "Orc Recovery") if err != nil { atomic.AddInt32(&shardsLockCounter, -1) - return nil, err + return nil, nil, err } - return func(e *error) { + return ctx, func(e *error) { defer atomic.AddInt32(&shardsLockCounter, -1) unlock(e) }, nil diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 8fad8ec44d7..1ffc048ffc6 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -17,6 +17,7 @@ package logic import ( + "context" "encoding/json" "fmt" "math/rand" @@ -828,7 +829,7 @@ func checkAndRecoverDeadMaster(analysisEntry inst.ReplicationAnalysis, candidate // That's it! We must do recovery! // TODO(sougou): This function gets called by GracefulMasterTakeover which may // need to obtain shard lock before getting here. - unlock, err := LockShard(analysisEntry.AnalyzedInstanceKey) + _, unlock, err := LockShard(context.Background(), analysisEntry.AnalyzedInstanceKey) if err != nil { log.Infof("CheckAndRecover: Analysis: %+v, InstanceKey: %+v, candidateInstanceKey: %+v, "+ "skipProcesses: %v: NOT detecting/recovering host, could not obtain shard lock (%v)", @@ -2014,7 +2015,7 @@ func electNewMaster(analysisEntry inst.ReplicationAnalysis, candidateInstanceKey } log.Infof("Analysis: %v, will elect a new master: %v", analysisEntry.Analysis, analysisEntry.SuggestedClusterAlias) - unlock, err := LockShard(analysisEntry.AnalyzedInstanceKey) + _, unlock, err := LockShard(context.Background(), analysisEntry.AnalyzedInstanceKey) if err != nil { log.Infof("CheckAndRecover: Analysis: %+v, InstanceKey: %+v, candidateInstanceKey: %+v, "+ "skipProcesses: %v: NOT detecting/recovering host, could not obtain shard lock (%v)", @@ -2133,7 +2134,7 @@ func fixMaster(analysisEntry inst.ReplicationAnalysis, candidateInstanceKey *ins } log.Infof("Analysis: %v, will fix master to read-write %+v", analysisEntry.Analysis, analysisEntry.AnalyzedInstanceKey) - unlock, err := LockShard(analysisEntry.AnalyzedInstanceKey) + _, unlock, err := LockShard(context.Background(), analysisEntry.AnalyzedInstanceKey) if err != nil { log.Infof("CheckAndRecover: Analysis: %+v, InstanceKey: %+v, candidateInstanceKey: %+v, "+ "skipProcesses: %v: NOT detecting/recovering host, could not obtain shard lock (%v)", @@ -2165,7 +2166,7 @@ func fixReplica(analysisEntry inst.ReplicationAnalysis, candidateInstanceKey *in } log.Infof("Analysis: %v, will fix replica %+v", analysisEntry.Analysis, analysisEntry.AnalyzedInstanceKey) - unlock, err := LockShard(analysisEntry.AnalyzedInstanceKey) + _, unlock, err := LockShard(context.Background(), analysisEntry.AnalyzedInstanceKey) if err != nil { log.Infof("CheckAndRecover: Analysis: %+v, InstanceKey: %+v, candidateInstanceKey: %+v, "+ "skipProcesses: %v: NOT detecting/recovering host, could not obtain shard lock (%v)", diff --git a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go new file mode 100644 index 00000000000..48b5bb40f09 --- /dev/null +++ b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go @@ -0,0 +1,76 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reparentutil + +import ( + "context" + + "vitess.io/vitess/go/event" + + "vitess.io/vitess/go/vt/logutil" + logutilpb "vitess.io/vitess/go/vt/proto/logutil" + "vitess.io/vitess/go/vt/topotools/events" + "vitess.io/vitess/go/vt/vttablet/tmclient" +) + +// EmergencyReparenter2 performs EmergencyReparentShard operations. +type EmergencyReparenter2 struct { + tmc tmclient.TabletManagerClient + logger logutil.Logger +} + +// NewEmergencyReparenter returns a new EmergencyReparenter object, ready to +// perform EmergencyReparentShard operations using the given topo.Server, +// TabletManagerClient, and logger. +// +// Providing a nil logger instance is allowed. +func NewEmergencyReparenter2(tmc tmclient.TabletManagerClient, logger logutil.Logger) *EmergencyReparenter2 { + erp := EmergencyReparenter2{ + tmc: tmc, + logger: logger, + } + + if erp.logger == nil { + // Create a no-op logger so we can call functions on er.logger without + // needed to constantly check for non-nil. + erp.logger = logutil.NewCallbackLogger(func(*logutilpb.Event) {}) + } + + return &erp +} + +// ReparentShard performs the EmergencyReparentShard operation on the given +// keyspace and shard. +func (erp *EmergencyReparenter2) ReparentShard(ctx context.Context, reparentFunctions ReparentFunctions) (*events.Reparent, error) { + ctx, unlock, err := reparentFunctions.LockShard(ctx) + if err != nil { + return nil, err + } + defer unlock(&err) + + ev := &events.Reparent{} + defer func() { + switch err { + case nil: + event.DispatchUpdate(ev, "finished EmergencyReparentShard") + default: + event.DispatchUpdate(ev, "failed EmergencyReparentShard: "+err.Error()) + } + }() + + return ev, nil +} diff --git a/go/vt/orchestrator/reparentutil/util.go b/go/vt/orchestrator/reparentutil/util.go index 9e31c6985da..a2b7c3eadab 100644 --- a/go/vt/orchestrator/reparentutil/util.go +++ b/go/vt/orchestrator/reparentutil/util.go @@ -18,9 +18,16 @@ package reparentutil import ( "context" + "fmt" "sync" "time" + "k8s.io/apimachinery/pkg/util/sets" + + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/orchestrator/inst" + "vitess.io/vitess/go/vt/orchestrator/logic" + "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" @@ -32,6 +39,63 @@ import ( "vitess.io/vitess/go/vt/proto/vtrpc" ) +type ( + // ReparentFunctions is an interface which has all the functions implementation required for re-parenting + ReparentFunctions interface { + LockShard(context.Context) (context.Context, func(*error), error) + } + + // VtOrcReparentFunctions is the VtOrc implementation for ReparentFunctions + VtOrcReparentFunctions struct { + analysisEntry inst.ReplicationAnalysis + candidateInstanceKey *inst.InstanceKey + skipProcesses bool + } + + // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions + VtctlReparentFunctions struct { + NewPrimaryAlias *topodatapb.TabletAlias + IgnoreReplicas sets.String + WaitReplicasTimeout time.Duration + keyspace string + shard string + ts *topo.Server + lockAction string + } +) + +var ( + _ ReparentFunctions = (*VtOrcReparentFunctions)(nil) + _ ReparentFunctions = (*VtctlReparentFunctions)(nil) +) + +func (vtctlReparent *VtctlReparentFunctions) LockShard(ctx context.Context) (context.Context, func(*error), error) { + vtctlReparent.lockAction = vtctlReparent.getLockAction(vtctlReparent.NewPrimaryAlias) + + return vtctlReparent.ts.LockShard(ctx, vtctlReparent.keyspace, vtctlReparent.shard, vtctlReparent.lockAction) +} + +func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { + action := "EmergencyReparentShard" + + if newPrimaryAlias != nil { + action += fmt.Sprintf("(%v)", topoproto.TabletAliasString(newPrimaryAlias)) + } + + return action +} + +func (vtorcReparent *VtOrcReparentFunctions) LockShard(ctx context.Context) (context.Context, func(*error), error) { + _, unlock, err := logic.LockShard(ctx, vtorcReparent.analysisEntry.AnalyzedInstanceKey) + if err != nil { + log.Infof("CheckAndRecover: Analysis: %+v, InstanceKey: %+v, candidateInstanceKey: %+v, "+ + "skipProcesses: %v: NOT detecting/recovering host, could not obtain shard lock (%v)", + vtorcReparent.analysisEntry.Analysis, vtorcReparent.analysisEntry.AnalyzedInstanceKey, vtorcReparent.candidateInstanceKey, vtorcReparent.skipProcesses, err) + return nil, nil, err + } + return ctx, unlock, nil +} + // ChooseNewPrimary finds a tablet that should become a primary after reparent. // The criteria for the new primary-elect are (preferably) to be in the same // cell as the current primary, and to be different from avoidPrimaryAlias. The From 73a8698db963298bb139f4b2f95993158cf668eb Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Wed, 21 Jul 2021 20:29:37 +0530 Subject: [PATCH 004/176] check if fixed and event dispatch for the shard info Signed-off-by: GuptaManan100 --- .../orchestrator/logic/reparent_functions.go | 65 +++++++++++++++++++ .../reparentutil/emergency_reparenter_2.go | 21 +++++- go/vt/orchestrator/reparentutil/util.go | 47 ++++++++------ 3 files changed, 111 insertions(+), 22 deletions(-) create mode 100644 go/vt/orchestrator/logic/reparent_functions.go diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go new file mode 100644 index 00000000000..17de496c3f9 --- /dev/null +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -0,0 +1,65 @@ +package logic + +import ( + "context" + "fmt" + + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + + "vitess.io/vitess/go/vt/orchestrator/external/golib/log" + "vitess.io/vitess/go/vt/orchestrator/inst" + "vitess.io/vitess/go/vt/topo" +) + +// VtOrcReparentFunctions is the VtOrc implementation for ReparentFunctions +type VtOrcReparentFunctions struct { + analysisEntry inst.ReplicationAnalysis + candidateInstanceKey *inst.InstanceKey + skipProcesses bool + topologyRecovery *TopologyRecovery +} + +// LockShard implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) LockShard(ctx context.Context) (context.Context, func(*error), error) { + _, unlock, err := LockShard(ctx, vtorcReparent.analysisEntry.AnalyzedInstanceKey) + if err != nil { + log.Infof("CheckAndRecover: Analysis: %+v, InstanceKey: %+v, candidateInstanceKey: %+v, "+ + "skipProcesses: %v: NOT detecting/recovering host, could not obtain shard lock (%v)", + vtorcReparent.analysisEntry.Analysis, vtorcReparent.analysisEntry.AnalyzedInstanceKey, vtorcReparent.candidateInstanceKey, vtorcReparent.skipProcesses, err) + return nil, nil, err + } + return ctx, unlock, nil +} + +// GetTopoServer implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) GetTopoServer() *topo.Server { + return ts +} + +// GetKeyspace implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) GetKeyspace() string { + tablet, _ := inst.ReadTablet(vtorcReparent.analysisEntry.AnalyzedInstanceKey) + return tablet.Keyspace +} + +// GetShard implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) GetShard() string { + tablet, _ := inst.ReadTablet(vtorcReparent.analysisEntry.AnalyzedInstanceKey) + return tablet.Shard +} + +// CheckIfFixed implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) CheckIfFixed() bool { + // Check if someone else fixed the problem. + tablet, err := TabletRefresh(vtorcReparent.analysisEntry.AnalyzedInstanceKey) + if err == nil && tablet.Type != topodatapb.TabletType_MASTER { + // TODO(sougou); use a version that only refreshes the current shard. + RefreshTablets() + AuditTopologyRecovery(vtorcReparent.topologyRecovery, "another agent seems to have fixed the problem") + // TODO(sougou): see if we have to reset the cluster as healthy. + return true + } + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("will handle DeadMaster event on %+v", vtorcReparent.analysisEntry.ClusterDetails.ClusterName)) + recoverDeadMasterCounter.Inc(1) + return false +} diff --git a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go index 48b5bb40f09..e0cbcf18883 100644 --- a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go +++ b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go @@ -33,7 +33,7 @@ type EmergencyReparenter2 struct { logger logutil.Logger } -// NewEmergencyReparenter returns a new EmergencyReparenter object, ready to +// NewEmergencyReparenter2 returns a new EmergencyReparenter object, ready to // perform EmergencyReparentShard operations using the given topo.Server, // TabletManagerClient, and logger. // @@ -72,5 +72,24 @@ func (erp *EmergencyReparenter2) ReparentShard(ctx context.Context, reparentFunc } }() + err = erp.reparentShardLocked(ctx, ev, reparentFunctions) + return ev, nil } + +func (erp *EmergencyReparenter2) reparentShardLocked(ctx context.Context, ev *events.Reparent, reparentFunctions ReparentFunctions) error { + + if reparentFunctions.CheckIfFixed() { + return nil + } + + ts := reparentFunctions.GetTopoServer() + shardInfo, err := ts.GetShard(ctx, reparentFunctions.GetKeyspace(), reparentFunctions.GetShard()) + if err != nil { + return err + } + ev.ShardInfo = *shardInfo + event.DispatchUpdate(ev, "reading all tablets") + + return nil +} diff --git a/go/vt/orchestrator/reparentutil/util.go b/go/vt/orchestrator/reparentutil/util.go index a2b7c3eadab..318bc6f8592 100644 --- a/go/vt/orchestrator/reparentutil/util.go +++ b/go/vt/orchestrator/reparentutil/util.go @@ -24,8 +24,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" - "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/orchestrator/inst" "vitess.io/vitess/go/vt/orchestrator/logic" "vitess.io/vitess/go/vt/logutil" @@ -43,13 +41,10 @@ type ( // ReparentFunctions is an interface which has all the functions implementation required for re-parenting ReparentFunctions interface { LockShard(context.Context) (context.Context, func(*error), error) - } - - // VtOrcReparentFunctions is the VtOrc implementation for ReparentFunctions - VtOrcReparentFunctions struct { - analysisEntry inst.ReplicationAnalysis - candidateInstanceKey *inst.InstanceKey - skipProcesses bool + GetTopoServer() *topo.Server + GetKeyspace() string + GetShard() string + CheckIfFixed() bool } // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions @@ -65,16 +60,37 @@ type ( ) var ( - _ ReparentFunctions = (*VtOrcReparentFunctions)(nil) + _ ReparentFunctions = (*logic.VtOrcReparentFunctions)(nil) _ ReparentFunctions = (*VtctlReparentFunctions)(nil) ) +// LockShard implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) LockShard(ctx context.Context) (context.Context, func(*error), error) { vtctlReparent.lockAction = vtctlReparent.getLockAction(vtctlReparent.NewPrimaryAlias) return vtctlReparent.ts.LockShard(ctx, vtctlReparent.keyspace, vtctlReparent.shard, vtctlReparent.lockAction) } +// GetTopoServer implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) GetTopoServer() *topo.Server { + return vtctlReparent.ts +} + +// GetKeyspace implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) GetKeyspace() string { + return vtctlReparent.keyspace +} + +// GetShard implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) GetShard() string { + return vtctlReparent.shard +} + +// CheckIfFixed implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) CheckIfFixed() bool { + return false +} + func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { action := "EmergencyReparentShard" @@ -85,17 +101,6 @@ func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topo return action } -func (vtorcReparent *VtOrcReparentFunctions) LockShard(ctx context.Context) (context.Context, func(*error), error) { - _, unlock, err := logic.LockShard(ctx, vtorcReparent.analysisEntry.AnalyzedInstanceKey) - if err != nil { - log.Infof("CheckAndRecover: Analysis: %+v, InstanceKey: %+v, candidateInstanceKey: %+v, "+ - "skipProcesses: %v: NOT detecting/recovering host, could not obtain shard lock (%v)", - vtorcReparent.analysisEntry.Analysis, vtorcReparent.analysisEntry.AnalyzedInstanceKey, vtorcReparent.candidateInstanceKey, vtorcReparent.skipProcesses, err) - return nil, nil, err - } - return ctx, unlock, nil -} - // ChooseNewPrimary finds a tablet that should become a primary after reparent. // The criteria for the new primary-elect are (preferably) to be in the same // cell as the current primary, and to be different from avoidPrimaryAlias. The From ee3ffb09c2504934033a8f5f10a35aa67d86c1fb Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Thu, 22 Jul 2021 08:23:40 +0530 Subject: [PATCH 005/176] PreRecoveryProcesses Signed-off-by: GuptaManan100 --- go/vt/orchestrator/logic/reparent_functions.go | 15 +++++++++++++++ .../reparentutil/emergency_reparenter_2.go | 4 ++++ go/vt/orchestrator/reparentutil/util.go | 12 ++++++++++++ 3 files changed, 31 insertions(+) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 17de496c3f9..6af610050cb 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -4,6 +4,8 @@ import ( "context" "fmt" + "vitess.io/vitess/go/vt/orchestrator/config" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/orchestrator/external/golib/log" @@ -63,3 +65,16 @@ func (vtorcReparent *VtOrcReparentFunctions) CheckIfFixed() bool { recoverDeadMasterCounter.Inc(1) return false } + +// PreRecoveryProcesses implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) PreRecoveryProcesses(ctx context.Context) error { + inst.AuditOperation("recover-dead-master", &vtorcReparent.analysisEntry.AnalyzedInstanceKey, "problem found; will recover") + if !vtorcReparent.skipProcesses { + if err := executeProcesses(config.Config.PreFailoverProcesses, "PreFailoverProcesses", vtorcReparent.topologyRecovery, true); err != nil { + return vtorcReparent.topologyRecovery.AddError(err) + } + } + + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: will recover %+v", vtorcReparent.analysisEntry.AnalyzedInstanceKey)) + return nil +} diff --git a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go index e0cbcf18883..b38c4d274c3 100644 --- a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go +++ b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go @@ -91,5 +91,9 @@ func (erp *EmergencyReparenter2) reparentShardLocked(ctx context.Context, ev *ev ev.ShardInfo = *shardInfo event.DispatchUpdate(ev, "reading all tablets") + if err := reparentFunctions.PreRecoveryProcesses(ctx); err != nil { + return err + } + return nil } diff --git a/go/vt/orchestrator/reparentutil/util.go b/go/vt/orchestrator/reparentutil/util.go index 318bc6f8592..4aef838a953 100644 --- a/go/vt/orchestrator/reparentutil/util.go +++ b/go/vt/orchestrator/reparentutil/util.go @@ -45,6 +45,7 @@ type ( GetKeyspace() string GetShard() string CheckIfFixed() bool + PreRecoveryProcesses(ctx context.Context) error } // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions @@ -56,6 +57,7 @@ type ( shard string ts *topo.Server lockAction string + tabletMap map[string]*topo.TabletInfo } ) @@ -91,6 +93,16 @@ func (vtctlReparent *VtctlReparentFunctions) CheckIfFixed() bool { return false } +// PreRecoveryProcesses implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) PreRecoveryProcesses(ctx context.Context) error { + var err error + vtctlReparent.tabletMap, err = vtctlReparent.ts.GetTabletMapForShard(ctx, vtctlReparent.keyspace, vtctlReparent.shard) + if err != nil { + return vterrors.Wrapf(err, "failed to get tablet map for %v/%v: %v", vtctlReparent.keyspace, vtctlReparent.shard, err) + } + return nil +} + func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { action := "EmergencyReparentShard" From 9f1d06b95b3f6a41d1aa47df5f7edffc3f9bb5aa Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Thu, 22 Jul 2021 10:20:19 +0530 Subject: [PATCH 006/176] stop replication and check shard locked Signed-off-by: GuptaManan100 --- .../orchestrator/logic/reparent_functions.go | 12 ++++++++++++ .../reparentutil/emergency_reparenter_2.go | 15 ++++++++++++++- go/vt/orchestrator/reparentutil/util.go | 19 ++++++++++++++++++- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 6af610050cb..97bf22c2dc4 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -4,6 +4,11 @@ import ( "context" "fmt" + "vitess.io/vitess/go/vt/vttablet/tmclient" + + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/topotools/events" + "vitess.io/vitess/go/vt/orchestrator/config" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -78,3 +83,10 @@ func (vtorcReparent *VtOrcReparentFunctions) PreRecoveryProcesses(ctx context.Co AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: will recover %+v", vtorcReparent.analysisEntry.AnalyzedInstanceKey)) return nil } + +// StopReplicationAndBuildStatusMaps implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) StopReplicationAndBuildStatusMaps(context.Context, tmclient.TabletManagerClient, *events.Reparent, logutil.Logger) error { + err := TabletDemoteMaster(vtorcReparent.analysisEntry.AnalyzedInstanceKey) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: TabletDemoteMaster: %v", err)) + return err +} diff --git a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go index b38c4d274c3..c22b139951b 100644 --- a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go +++ b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go @@ -19,6 +19,9 @@ package reparentutil import ( "context" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/event" "vitess.io/vitess/go/vt/logutil" @@ -84,7 +87,9 @@ func (erp *EmergencyReparenter2) reparentShardLocked(ctx context.Context, ev *ev } ts := reparentFunctions.GetTopoServer() - shardInfo, err := ts.GetShard(ctx, reparentFunctions.GetKeyspace(), reparentFunctions.GetShard()) + keyspace := reparentFunctions.GetKeyspace() + shard := reparentFunctions.GetShard() + shardInfo, err := ts.GetShard(ctx, keyspace, shard) if err != nil { return err } @@ -95,5 +100,13 @@ func (erp *EmergencyReparenter2) reparentShardLocked(ctx context.Context, ev *ev return err } + if err := reparentFunctions.StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, erp.logger); err != nil { + return err + } + + if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { + return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) + } + return nil } diff --git a/go/vt/orchestrator/reparentutil/util.go b/go/vt/orchestrator/reparentutil/util.go index 4aef838a953..a7050e5950b 100644 --- a/go/vt/orchestrator/reparentutil/util.go +++ b/go/vt/orchestrator/reparentutil/util.go @@ -22,6 +22,10 @@ import ( "sync" "time" + replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" + + "vitess.io/vitess/go/vt/topotools/events" + "k8s.io/apimachinery/pkg/util/sets" "vitess.io/vitess/go/vt/orchestrator/logic" @@ -45,7 +49,8 @@ type ( GetKeyspace() string GetShard() string CheckIfFixed() bool - PreRecoveryProcesses(ctx context.Context) error + PreRecoveryProcesses(context.Context) error + StopReplicationAndBuildStatusMaps(context.Context, tmclient.TabletManagerClient, *events.Reparent, logutil.Logger) error } // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions @@ -58,6 +63,8 @@ type ( ts *topo.Server lockAction string tabletMap map[string]*topo.TabletInfo + statusMap map[string]*replicationdatapb.StopReplicationStatus + primaryStatusMap map[string]*replicationdatapb.MasterStatus } ) @@ -103,6 +110,16 @@ func (vtctlReparent *VtctlReparentFunctions) PreRecoveryProcesses(ctx context.Co return nil } +// StopReplicationAndBuildStatusMaps implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) StopReplicationAndBuildStatusMaps(ctx context.Context, tmc tmclient.TabletManagerClient, ev *events.Reparent, logger logutil.Logger) error { + var err error + vtctlReparent.statusMap, vtctlReparent.primaryStatusMap, err = StopReplicationAndBuildStatusMaps(ctx, tmc, ev, vtctlReparent.tabletMap, vtctlReparent.WaitReplicasTimeout, vtctlReparent.IgnoreReplicas, logger) + if err != nil { + return vterrors.Wrapf(err, "failed to stop replication and build status maps: %v", err) + } + return nil +} + func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { action := "EmergencyReparentShard" From ba99549ef58a95b11d5a7092edaedb57d0c624a0 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Thu, 22 Jul 2021 12:32:53 +0530 Subject: [PATCH 007/176] get primary recovery type Signed-off-by: GuptaManan100 --- .../orchestrator/logic/reparent_functions.go | 28 +++++++++++++++++++ .../reparentutil/emergency_reparenter_2.go | 7 +++++ go/vt/orchestrator/reparentutil/util.go | 12 ++++++++ 3 files changed, 47 insertions(+) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 97bf22c2dc4..a6007153678 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -1,3 +1,19 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package logic import ( @@ -90,3 +106,15 @@ func (vtorcReparent *VtOrcReparentFunctions) StopReplicationAndBuildStatusMaps(c AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: TabletDemoteMaster: %v", err)) return err } + +// GetPrimaryRecoveryType implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) GetPrimaryRecoveryType() MasterRecoveryType { + vtorcReparent.topologyRecovery.RecoveryType = GetMasterRecoveryType(&vtorcReparent.topologyRecovery.AnalysisEntry) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: masterRecoveryType=%+v", vtorcReparent.topologyRecovery.RecoveryType)) + return vtorcReparent.topologyRecovery.RecoveryType +} + +// AddError implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) AddError(errorMsg string) error { + return vtorcReparent.topologyRecovery.AddError(log.Errorf(errorMsg)) +} diff --git a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go index c22b139951b..d52eed1e1ed 100644 --- a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go +++ b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go @@ -19,6 +19,8 @@ package reparentutil import ( "context" + "vitess.io/vitess/go/vt/orchestrator/logic" + "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/vterrors" @@ -108,5 +110,10 @@ func (erp *EmergencyReparenter2) reparentShardLocked(ctx context.Context, ev *ev return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } + recoveryType := reparentFunctions.GetPrimaryRecoveryType() + if recoveryType != logic.MasterRecoveryGTID { + return reparentFunctions.AddError("RecoveryType unknown/unsupported") + } + return nil } diff --git a/go/vt/orchestrator/reparentutil/util.go b/go/vt/orchestrator/reparentutil/util.go index 29fc0ecae7c..f6fd96f22bf 100644 --- a/go/vt/orchestrator/reparentutil/util.go +++ b/go/vt/orchestrator/reparentutil/util.go @@ -51,6 +51,8 @@ type ( CheckIfFixed() bool PreRecoveryProcesses(context.Context) error StopReplicationAndBuildStatusMaps(context.Context, tmclient.TabletManagerClient, *events.Reparent, logutil.Logger) error + GetPrimaryRecoveryType() logic.MasterRecoveryType + AddError(string) error } // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions @@ -120,6 +122,16 @@ func (vtctlReparent *VtctlReparentFunctions) StopReplicationAndBuildStatusMaps(c return nil } +// GetPrimaryRecoveryType implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) GetPrimaryRecoveryType() logic.MasterRecoveryType { + return logic.MasterRecoveryGTID +} + +// AddError implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) AddError(errorMsg string) error { + return vterrors.New(vtrpc.Code_INTERNAL, errorMsg) +} + func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { action := "EmergencyReparentShard" From aa4a2610497d9422b869abadef4507ecb4b64739 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Thu, 22 Jul 2021 15:56:46 +0530 Subject: [PATCH 008/176] find primary candidates Signed-off-by: GuptaManan100 --- .../orchestrator/logic/reparent_functions.go | 69 ++++++++++ .../reparentutil/emergency_reparenter_2.go | 9 ++ go/vt/orchestrator/reparentutil/util.go | 122 ++++++++++++++++-- 3 files changed, 190 insertions(+), 10 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index a6007153678..72908d3b1c8 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -19,6 +19,7 @@ package logic import ( "context" "fmt" + "time" "vitess.io/vitess/go/vt/vttablet/tmclient" @@ -40,6 +41,8 @@ type VtOrcReparentFunctions struct { candidateInstanceKey *inst.InstanceKey skipProcesses bool topologyRecovery *TopologyRecovery + promotedReplica *inst.Instance + lostReplicas [](*inst.Instance) } // LockShard implements the ReparentFunctions interface @@ -118,3 +121,69 @@ func (vtorcReparent *VtOrcReparentFunctions) GetPrimaryRecoveryType() MasterReco func (vtorcReparent *VtOrcReparentFunctions) AddError(errorMsg string) error { return vtorcReparent.topologyRecovery.AddError(log.Errorf(errorMsg)) } + +// FindPrimaryCandidates implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient) error { + postponedAll := false + promotedReplicaIsIdeal := func(promoted *inst.Instance, hasBestPromotionRule bool) bool { + if promoted == nil { + return false + } + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: promotedReplicaIsIdeal(%+v)", promoted.Key)) + if vtorcReparent.candidateInstanceKey != nil { //explicit request to promote a specific server + return promoted.Key.Equals(vtorcReparent.candidateInstanceKey) + } + if promoted.DataCenter == vtorcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstanceDataCenter && + promoted.PhysicalEnvironment == vtorcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstancePhysicalEnvironment { + if promoted.PromotionRule == inst.MustPromoteRule || promoted.PromotionRule == inst.PreferPromoteRule || + (hasBestPromotionRule && promoted.PromotionRule != inst.MustNotPromoteRule) { + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: found %+v to be ideal candidate; will optimize recovery", promoted.Key)) + postponedAll = true + return true + } + } + return false + } + + AuditTopologyRecovery(vtorcReparent.topologyRecovery, "RecoverDeadMaster: regrouping replicas via GTID") + lostReplicas, _, cannotReplicateReplicas, promotedReplica, err := inst.RegroupReplicasGTID(&vtorcReparent.analysisEntry.AnalyzedInstanceKey, true, nil, &vtorcReparent.topologyRecovery.PostponedFunctionsContainer, promotedReplicaIsIdeal) + vtorcReparent.topologyRecovery.AddError(err) + lostReplicas = append(lostReplicas, cannotReplicateReplicas...) + for _, replica := range lostReplicas { + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: - lost replica: %+v", replica.Key)) + } + + if promotedReplica != nil && len(lostReplicas) > 0 && config.Config.DetachLostReplicasAfterMasterFailover { + postponedFunction := func() error { + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: lost %+v replicas during recovery process; detaching them", len(lostReplicas))) + for _, replica := range lostReplicas { + replica := replica + inst.DetachReplicaMasterHost(&replica.Key) + } + return nil + } + vtorcReparent.topologyRecovery.AddPostponedFunction(postponedFunction, fmt.Sprintf("RecoverDeadMaster, detach %+v lost replicas", len(lostReplicas))) + } + + func() error { + // TODO(sougou): Commented out: this downtime feels a little aggressive. + //inst.BeginDowntime(inst.NewDowntime(failedInstanceKey, inst.GetMaintenanceOwner(), inst.DowntimeLostInRecoveryMessage, time.Duration(config.LostInRecoveryDowntimeSeconds)*time.Second)) + acknowledgeInstanceFailureDetection(&vtorcReparent.analysisEntry.AnalyzedInstanceKey) + for _, replica := range lostReplicas { + replica := replica + inst.BeginDowntime(inst.NewDowntime(&replica.Key, inst.GetMaintenanceOwner(), inst.DowntimeLostInRecoveryMessage, time.Duration(config.LostInRecoveryDowntimeSeconds)*time.Second)) + } + return nil + }() + + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: %d postponed functions", vtorcReparent.topologyRecovery.PostponedFunctionsContainer.Len())) + + if promotedReplica != nil && !postponedAll { + promotedReplica, err = replacePromotedReplicaWithCandidate(vtorcReparent.topologyRecovery, &vtorcReparent.analysisEntry.AnalyzedInstanceKey, promotedReplica, vtorcReparent.candidateInstanceKey) + vtorcReparent.topologyRecovery.AddError(err) + } + + vtorcReparent.promotedReplica = promotedReplica + vtorcReparent.lostReplicas = lostReplicas + return nil +} diff --git a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go index d52eed1e1ed..9deaa2ca04c 100644 --- a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go +++ b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go @@ -115,5 +115,14 @@ func (erp *EmergencyReparenter2) reparentShardLocked(ctx context.Context, ev *ev return reparentFunctions.AddError("RecoveryType unknown/unsupported") } + if err := reparentFunctions.FindPrimaryCandidates(ctx, erp.logger, erp.tmc); err != nil { + return err + } + + // Check (again) we still have the topology lock. + if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { + return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) + } + return nil } diff --git a/go/vt/orchestrator/reparentutil/util.go b/go/vt/orchestrator/reparentutil/util.go index f6fd96f22bf..6fa642fe2b9 100644 --- a/go/vt/orchestrator/reparentutil/util.go +++ b/go/vt/orchestrator/reparentutil/util.go @@ -22,6 +22,10 @@ import ( "sync" "time" + "vitess.io/vitess/go/vt/concurrency" + + "vitess.io/vitess/go/mysql" + replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" "vitess.io/vitess/go/vt/topotools/events" @@ -53,20 +57,23 @@ type ( StopReplicationAndBuildStatusMaps(context.Context, tmclient.TabletManagerClient, *events.Reparent, logutil.Logger) error GetPrimaryRecoveryType() logic.MasterRecoveryType AddError(string) error + FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient) error } // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions VtctlReparentFunctions struct { - NewPrimaryAlias *topodatapb.TabletAlias - IgnoreReplicas sets.String - WaitReplicasTimeout time.Duration - keyspace string - shard string - ts *topo.Server - lockAction string - tabletMap map[string]*topo.TabletInfo - statusMap map[string]*replicationdatapb.StopReplicationStatus - primaryStatusMap map[string]*replicationdatapb.PrimaryStatus + NewPrimaryAlias *topodatapb.TabletAlias + IgnoreReplicas sets.String + WaitReplicasTimeout time.Duration + keyspace string + shard string + ts *topo.Server + lockAction string + tabletMap map[string]*topo.TabletInfo + statusMap map[string]*replicationdatapb.StopReplicationStatus + primaryStatusMap map[string]*replicationdatapb.PrimaryStatus + winningPosition mysql.Position + winningPrimaryTabletAliasStr string } ) @@ -132,6 +139,45 @@ func (vtctlReparent *VtctlReparentFunctions) AddError(errorMsg string) error { return vterrors.New(vtrpc.Code_INTERNAL, errorMsg) } +// FindPrimaryCandidates implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient) error { + validCandidates, err := FindValidEmergencyReparentCandidates(vtctlReparent.statusMap, vtctlReparent.primaryStatusMap) + if err != nil { + return err + } else if len(validCandidates) == 0 { + return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "no valid candidates for emergency reparent") + } + + // Wait for all candidates to apply relay logs + if err := vtctlReparent.waitForAllRelayLogsToApply(ctx, logger, tmc, validCandidates); err != nil { + return err + } + + // Elect the candidate with the most up-to-date position. + + for alias, position := range validCandidates { + if vtctlReparent.winningPosition.IsZero() || position.AtLeast(vtctlReparent.winningPosition) { + vtctlReparent.winningPosition = position + vtctlReparent.winningPrimaryTabletAliasStr = alias + } + } + + // If we were requested to elect a particular primary, verify it's a valid + // candidate (non-zero position, no errant GTIDs) and is at least as + // advanced as the winning position. + if vtctlReparent.NewPrimaryAlias != nil { + vtctlReparent.winningPrimaryTabletAliasStr = topoproto.TabletAliasString(vtctlReparent.NewPrimaryAlias) + pos, ok := validCandidates[vtctlReparent.winningPrimaryTabletAliasStr] + switch { + case !ok: + return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v has errant GTIDs", vtctlReparent.winningPrimaryTabletAliasStr) + case !pos.AtLeast(vtctlReparent.winningPosition): + return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v at position %v is not fully caught up. Winning position: %v", vtctlReparent.winningPrimaryTabletAliasStr, pos, vtctlReparent.winningPosition) + } + } + return nil +} + func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { action := "EmergencyReparentShard" @@ -142,6 +188,62 @@ func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topo return action } +func (vtctlReparent *VtctlReparentFunctions) waitForAllRelayLogsToApply(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position) error { + errCh := make(chan error) + defer close(errCh) + + groupCtx, groupCancel := context.WithTimeout(ctx, vtctlReparent.WaitReplicasTimeout) + defer groupCancel() + + waiterCount := 0 + + for candidate := range validCandidates { + // When we called StopReplicationAndBuildStatusMaps, we got back two + // maps: (1) the StopReplicationStatus of any replicas that actually + // stopped replication; and (2) the MasterStatus of anything that + // returned ErrNotReplica, which is a tablet that is either the current + // primary or is stuck thinking it is a MASTER but is not in actuality. + // + // If we have a tablet in the validCandidates map that does not appear + // in the statusMap, then we have either (a) the current primary, which + // is not replicating, so it is not applying relay logs; or (b) a tablet + // that is stuck thinking it is MASTER but is not in actuality. In that + // second case - (b) - we will most likely find that the stuck MASTER + // does not have a winning position, and fail the ERS. If, on the other + // hand, it does have a winning position, we are trusting the operator + // to know what they are doing by emergency-reparenting onto that + // tablet. In either case, it does not make sense to wait for relay logs + // to apply on a tablet that was never applying relay logs in the first + // place, so we skip it, and log that we did. + status, ok := vtctlReparent.statusMap[candidate] + if !ok { + logger.Infof("EmergencyReparent candidate %v not in replica status map; this means it was not running replication (because it was formerly MASTER), so skipping WaitForRelayLogsToApply step for this candidate", candidate) + continue + } + + go func(alias string, status *replicationdatapb.StopReplicationStatus) { + var err error + defer func() { errCh <- err }() + err = WaitForRelayLogsToApply(groupCtx, tmc, vtctlReparent.tabletMap[alias], status) + }(candidate, status) + + waiterCount++ + } + + errgroup := concurrency.ErrorGroup{ + NumGoroutines: waiterCount, + NumRequiredSuccesses: waiterCount, + NumAllowedErrors: 0, + } + rec := errgroup.Wait(groupCancel, errCh) + + if len(rec.Errors) != 0 { + return vterrors.Wrapf(rec.Error(), "could not apply all relay logs within the provided WaitReplicasTimeout (%s): %v", vtctlReparent.WaitReplicasTimeout, rec.Error()) + } + + return nil +} + // ChooseNewPrimary finds a tablet that should become a primary after reparent. // The criteria for the new primary-elect are (preferably) to be in the same // cell as the current primary, and to be different from avoidPrimaryAlias. The From 74cbabc5e7d55e92bd5b823a490a366dd3496d9b Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Thu, 22 Jul 2021 17:11:35 +0530 Subject: [PATCH 009/176] check if need to override primary candidate Signed-off-by: GuptaManan100 --- .../orchestrator/logic/reparent_functions.go | 50 +++++++++++++++++++ .../reparentutil/emergency_reparenter_2.go | 4 ++ go/vt/orchestrator/reparentutil/util.go | 11 +++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 72908d3b1c8..5e06d41283c 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -187,3 +187,53 @@ func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.C vtorcReparent.lostReplicas = lostReplicas return nil } + +// CheckIfNeedToOverridePrimary implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) CheckIfNeedToOverridePrimary() error { + if vtorcReparent.promotedReplica == nil { + err := TabletUndoDemoteMaster(vtorcReparent.analysisEntry.AnalyzedInstanceKey) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: TabletUndoDemoteMaster: %v", err)) + message := "Failure: no replica promoted." + AuditTopologyRecovery(vtorcReparent.topologyRecovery, message) + inst.AuditOperation("recover-dead-master", &vtorcReparent.analysisEntry.AnalyzedInstanceKey, message) + return err + } + + message := fmt.Sprintf("promoted replica: %+v", vtorcReparent.promotedReplica.Key) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, message) + inst.AuditOperation("recover-dead-master", &vtorcReparent.analysisEntry.AnalyzedInstanceKey, message) + vtorcReparent.topologyRecovery.LostReplicas.AddInstances(vtorcReparent.lostReplicas) + + var err error + overrideMasterPromotion := func() (*inst.Instance, error) { + if vtorcReparent.promotedReplica == nil { + // No promotion; nothing to override. + return vtorcReparent.promotedReplica, err + } + // Scenarios where we might cancel the promotion. + if satisfied, reason := MasterFailoverGeographicConstraintSatisfied(&vtorcReparent.analysisEntry, vtorcReparent.promotedReplica); !satisfied { + return nil, fmt.Errorf("RecoverDeadMaster: failed %+v promotion; %s", vtorcReparent.promotedReplica.Key, reason) + } + if config.Config.FailMasterPromotionOnLagMinutes > 0 && + time.Duration(vtorcReparent.promotedReplica.ReplicationLagSeconds.Int64)*time.Second >= time.Duration(config.Config.FailMasterPromotionOnLagMinutes)*time.Minute { + // candidate replica lags too much + return nil, fmt.Errorf("RecoverDeadMaster: failed promotion. FailMasterPromotionOnLagMinutes is set to %d (minutes) and promoted replica %+v 's lag is %d (seconds)", config.Config.FailMasterPromotionOnLagMinutes, vtorcReparent.promotedReplica.Key, vtorcReparent.promotedReplica.ReplicationLagSeconds.Int64) + } + if config.Config.FailMasterPromotionIfSQLThreadNotUpToDate && !vtorcReparent.promotedReplica.SQLThreadUpToDate() { + return nil, fmt.Errorf("RecoverDeadMaster: failed promotion. FailMasterPromotionIfSQLThreadNotUpToDate is set and promoted replica %+v 's sql thread is not up to date (relay logs still unapplied). Aborting promotion", vtorcReparent.promotedReplica.Key) + } + if config.Config.DelayMasterPromotionIfSQLThreadNotUpToDate && !vtorcReparent.promotedReplica.SQLThreadUpToDate() { + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("DelayMasterPromotionIfSQLThreadNotUpToDate: waiting for SQL thread on %+v", vtorcReparent.promotedReplica.Key)) + if _, err := inst.WaitForSQLThreadUpToDate(&vtorcReparent.promotedReplica.Key, 0, 0); err != nil { + return nil, fmt.Errorf("DelayMasterPromotionIfSQLThreadNotUpToDate error: %+v", err) + } + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("DelayMasterPromotionIfSQLThreadNotUpToDate: SQL thread caught up on %+v", vtorcReparent.promotedReplica.Key)) + } + // All seems well. No override done. + return vtorcReparent.promotedReplica, err + } + if vtorcReparent.promotedReplica, err = overrideMasterPromotion(); err != nil { + AuditTopologyRecovery(vtorcReparent.topologyRecovery, err.Error()) + } + return nil +} diff --git a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go index 9deaa2ca04c..6ebab8e367b 100644 --- a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go +++ b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go @@ -119,6 +119,10 @@ func (erp *EmergencyReparenter2) reparentShardLocked(ctx context.Context, ev *ev return err } + if err := reparentFunctions.CheckIfNeedToOverridePrimary(); err != nil { + return err + } + // Check (again) we still have the topology lock. if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) diff --git a/go/vt/orchestrator/reparentutil/util.go b/go/vt/orchestrator/reparentutil/util.go index 6fa642fe2b9..b3423da6ea4 100644 --- a/go/vt/orchestrator/reparentutil/util.go +++ b/go/vt/orchestrator/reparentutil/util.go @@ -58,6 +58,7 @@ type ( GetPrimaryRecoveryType() logic.MasterRecoveryType AddError(string) error FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient) error + CheckIfNeedToOverridePrimary() error } // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions @@ -72,6 +73,7 @@ type ( tabletMap map[string]*topo.TabletInfo statusMap map[string]*replicationdatapb.StopReplicationStatus primaryStatusMap map[string]*replicationdatapb.PrimaryStatus + validCandidates map[string]mysql.Position winningPosition mysql.Position winningPrimaryTabletAliasStr string } @@ -162,12 +164,19 @@ func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.C } } + vtctlReparent.validCandidates = validCandidates + + return nil +} + +// CheckIfNeedToOverridePrimary implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePrimary() error { // If we were requested to elect a particular primary, verify it's a valid // candidate (non-zero position, no errant GTIDs) and is at least as // advanced as the winning position. if vtctlReparent.NewPrimaryAlias != nil { vtctlReparent.winningPrimaryTabletAliasStr = topoproto.TabletAliasString(vtctlReparent.NewPrimaryAlias) - pos, ok := validCandidates[vtctlReparent.winningPrimaryTabletAliasStr] + pos, ok := vtctlReparent.validCandidates[vtctlReparent.winningPrimaryTabletAliasStr] switch { case !ok: return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v has errant GTIDs", vtctlReparent.winningPrimaryTabletAliasStr) From 5f7c08830369dfd319235ef39ad103764855b751 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Thu, 22 Jul 2021 19:02:05 +0530 Subject: [PATCH 010/176] start replication Signed-off-by: GuptaManan100 --- .../orchestrator/logic/reparent_functions.go | 93 +++++++++++ .../reparentutil/emergency_reparenter_2.go | 12 +- go/vt/orchestrator/reparentutil/util.go | 149 ++++++++++++++++++ 3 files changed, 253 insertions(+), 1 deletion(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 5e06d41283c..a2204e5dce3 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -21,6 +21,9 @@ import ( "fmt" "time" + "vitess.io/vitess/go/vt/orchestrator/attributes" + "vitess.io/vitess/go/vt/orchestrator/kv" + "vitess.io/vitess/go/vt/vttablet/tmclient" "vitess.io/vitess/go/vt/logutil" @@ -237,3 +240,93 @@ func (vtorcReparent *VtOrcReparentFunctions) CheckIfNeedToOverridePrimary() erro } return nil } + +// StartReplication implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) StartReplication(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) error { + // And this is the end; whether successful or not, we're done. + resolveRecovery(vtorcReparent.topologyRecovery, vtorcReparent.promotedReplica) + // Now, see whether we are successful or not. From this point there's no going back. + if vtorcReparent.promotedReplica != nil { + // Success! + recoverDeadMasterSuccessCounter.Inc(1) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: successfully promoted %+v", vtorcReparent.promotedReplica.Key)) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: promoted server coordinates: %+v", vtorcReparent.promotedReplica.SelfBinlogCoordinates)) + + AuditTopologyRecovery(vtorcReparent.topologyRecovery, "- RecoverDeadMaster: will apply MySQL changes to promoted master") + { + _, err := inst.ResetReplicationOperation(&vtorcReparent.promotedReplica.Key) + if err != nil { + // Ugly, but this is important. Let's give it another try + _, err = inst.ResetReplicationOperation(&vtorcReparent.promotedReplica.Key) + } + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying RESET SLAVE ALL on promoted master: success=%t", (err == nil))) + if err != nil { + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: NOTE that %+v is promoted even though SHOW SLAVE STATUS may still show it has a master", vtorcReparent.promotedReplica.Key)) + } + } + { + count := inst.MasterSemiSync(vtorcReparent.promotedReplica.Key) + err := inst.SetSemiSyncMaster(&vtorcReparent.promotedReplica.Key, count > 0) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying semi-sync %v: success=%t", count > 0, (err == nil))) + + // Dont' allow writes if semi-sync settings fail. + if err == nil { + _, err := inst.SetReadOnly(&vtorcReparent.promotedReplica.Key, false) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying read-only=0 on promoted master: success=%t", (err == nil))) + } + } + // Let's attempt, though we won't necessarily succeed, to set old master as read-only + go func() { + _, err := inst.SetReadOnly(&vtorcReparent.analysisEntry.AnalyzedInstanceKey, true) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying read-only=1 on demoted master: success=%t", (err == nil))) + }() + + kvPairs := inst.GetClusterMasterKVPairs(vtorcReparent.analysisEntry.ClusterDetails.ClusterAlias, &vtorcReparent.promotedReplica.Key) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("Writing KV %+v", kvPairs)) + for _, kvPair := range kvPairs { + err := kv.PutKVPair(kvPair) + log.Errore(err) + } + { + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("Distributing KV %+v", kvPairs)) + err := kv.DistributePairs(kvPairs) + log.Errore(err) + } + if config.Config.MasterFailoverDetachReplicaMasterHost { + postponedFunction := func() error { + AuditTopologyRecovery(vtorcReparent.topologyRecovery, "- RecoverDeadMaster: detaching master host on promoted master") + inst.DetachReplicaMasterHost(&vtorcReparent.promotedReplica.Key) + return nil + } + vtorcReparent.topologyRecovery.AddPostponedFunction(postponedFunction, fmt.Sprintf("RecoverDeadMaster, detaching promoted master host %+v", vtorcReparent.promotedReplica.Key)) + } + func() error { + before := vtorcReparent.analysisEntry.AnalyzedInstanceKey.StringCode() + after := vtorcReparent.promotedReplica.Key.StringCode() + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: updating cluster_alias: %v -> %v", before, after)) + //~~~inst.ReplaceClusterName(before, after) + if alias := vtorcReparent.analysisEntry.ClusterDetails.ClusterAlias; alias != "" { + inst.SetClusterAlias(vtorcReparent.promotedReplica.Key.StringCode(), alias) + } else { + inst.ReplaceAliasClusterName(before, after) + } + return nil + }() + + attributes.SetGeneralAttribute(vtorcReparent.analysisEntry.ClusterDetails.ClusterDomain, vtorcReparent.promotedReplica.Key.StringCode()) + + if !vtorcReparent.skipProcesses { + // Execute post master-failover processes + executeProcesses(config.Config.PostMasterFailoverProcesses, "PostMasterFailoverProcesses", vtorcReparent.topologyRecovery, false) + } + } else { + recoverDeadMasterFailureCounter.Inc(1) + } + return nil +} + +// GetNewPrimary implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) GetNewPrimary() *topodatapb.Tablet { + tablet, _ := inst.ReadTablet(vtorcReparent.promotedReplica.Key) + return tablet +} diff --git a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go index 6ebab8e367b..15dc107811f 100644 --- a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go +++ b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go @@ -19,6 +19,10 @@ package reparentutil import ( "context" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + + "google.golang.org/protobuf/proto" + "vitess.io/vitess/go/vt/orchestrator/logic" "vitess.io/vitess/go/vt/topo" @@ -79,7 +83,7 @@ func (erp *EmergencyReparenter2) ReparentShard(ctx context.Context, reparentFunc err = erp.reparentShardLocked(ctx, ev, reparentFunctions) - return ev, nil + return ev, err } func (erp *EmergencyReparenter2) reparentShardLocked(ctx context.Context, ev *events.Reparent, reparentFunctions ReparentFunctions) error { @@ -128,5 +132,11 @@ func (erp *EmergencyReparenter2) reparentShardLocked(ctx context.Context, ev *ev return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } + if err := reparentFunctions.StartReplication(ctx, ev, erp.logger, erp.tmc); err != nil { + return err + } + + ev.NewMaster = proto.Clone(reparentFunctions.GetNewPrimary()).(*topodatapb.Tablet) + return nil } diff --git a/go/vt/orchestrator/reparentutil/util.go b/go/vt/orchestrator/reparentutil/util.go index b3423da6ea4..ce23af53797 100644 --- a/go/vt/orchestrator/reparentutil/util.go +++ b/go/vt/orchestrator/reparentutil/util.go @@ -22,6 +22,8 @@ import ( "sync" "time" + "vitess.io/vitess/go/event" + "vitess.io/vitess/go/vt/concurrency" "vitess.io/vitess/go/mysql" @@ -59,6 +61,8 @@ type ( AddError(string) error FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient) error CheckIfNeedToOverridePrimary() error + StartReplication(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) error + GetNewPrimary() *topodatapb.Tablet } // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions @@ -187,6 +191,17 @@ func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePrimary() erro return nil } +// StartReplication implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) StartReplication(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) error { + // Do the promotion. + return vtctlReparent.promoteNewPrimary(ctx, ev, logger, tmc) +} + +// GetNewPrimary implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) GetNewPrimary() *topodatapb.Tablet { + return vtctlReparent.tabletMap[vtctlReparent.winningPrimaryTabletAliasStr].Tablet +} + func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { action := "EmergencyReparentShard" @@ -253,6 +268,140 @@ func (vtctlReparent *VtctlReparentFunctions) waitForAllRelayLogsToApply(ctx cont return nil } +func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) error { + logger.Infof("promoting tablet %v to master", vtctlReparent.winningPrimaryTabletAliasStr) + event.DispatchUpdate(ev, "promoting replica") + + newPrimaryTabletInfo, ok := vtctlReparent.tabletMap[vtctlReparent.winningPrimaryTabletAliasStr] + if !ok { + return vterrors.Errorf(vtrpc.Code_INTERNAL, "attempted to promote master-elect %v that was not in the tablet map; this an impossible situation", vtctlReparent.winningPrimaryTabletAliasStr) + } + + rp, err := tmc.PromoteReplica(ctx, newPrimaryTabletInfo.Tablet) + if err != nil { + return vterrors.Wrapf(err, "master-elect tablet %v failed to be upgraded to master: %v", vtctlReparent.winningPrimaryTabletAliasStr, err) + } + + if err := topo.CheckShardLocked(ctx, vtctlReparent.keyspace, vtctlReparent.shard); err != nil { + return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) + } + + replCtx, replCancel := context.WithTimeout(ctx, vtctlReparent.WaitReplicasTimeout) + defer replCancel() + + event.DispatchUpdate(ev, "reparenting all tablets") + + // Create a context and cancel function to watch for the first successful + // SetMaster call on a replica. We use a background context so that this + // context is only ever Done when its cancel is called by the background + // goroutine we're about to spin up. + // + // Similarly, create a context and cancel for the replica waiter goroutine + // to signal when all replica goroutines have finished. In the case where at + // least one replica succeeds, replSuccessCtx will be canceled first, while + // allReplicasDoneCtx is guaranteed to be canceled within + // opts.WaitReplicasTimeout plus some jitter. + replSuccessCtx, replSuccessCancel := context.WithCancel(context.Background()) + allReplicasDoneCtx, allReplicasDoneCancel := context.WithCancel(context.Background()) + + now := time.Now().UnixNano() + replWg := sync.WaitGroup{} + rec := concurrency.AllErrorRecorder{} + + handlePrimary := func(alias string, ti *topo.TabletInfo) error { + logger.Infof("populating reparent journal on new master %v", alias) + return tmc.PopulateReparentJournal(replCtx, ti.Tablet, now, vtctlReparent.lockAction, newPrimaryTabletInfo.Alias, rp) + } + + handleReplica := func(alias string, ti *topo.TabletInfo) { + defer replWg.Done() + logger.Infof("setting new master on replica %v", alias) + + forceStart := false + if status, ok := vtctlReparent.statusMap[alias]; ok { + fs, err := ReplicaWasRunning(status) + if err != nil { + err = vterrors.Wrapf(err, "tablet %v could not determine StopReplicationStatus: %v", alias, err) + rec.RecordError(err) + + return + } + + forceStart = fs + } + + err := tmc.SetMaster(replCtx, ti.Tablet, newPrimaryTabletInfo.Alias, now, "", forceStart) + if err != nil { + err = vterrors.Wrapf(err, "tablet %v SetMaster failed: %v", alias, err) + rec.RecordError(err) + + return + } + + // Signal that at least one goroutine succeeded to SetMaster. + replSuccessCancel() + } + + numReplicas := 0 + + for alias, ti := range vtctlReparent.tabletMap { + switch { + case alias == vtctlReparent.winningPrimaryTabletAliasStr: + continue + case !vtctlReparent.IgnoreReplicas.Has(alias): + replWg.Add(1) + numReplicas++ + go handleReplica(alias, ti) + } + } + + // Spin up a background goroutine to wait until all replica goroutines + // finished. Polling this way allows us to have promoteNewPrimary return + // success as soon as (a) the primary successfully populates its reparent + // journal and (b) at least one replica successfully begins replicating. + // + // If we were to follow the more common pattern of blocking on replWg.Wait() + // in the main body of promoteNewPrimary, we would be bound to the + // time of slowest replica, instead of the time of the fastest successful + // replica, and we want ERS to be fast. + go func() { + replWg.Wait() + allReplicasDoneCancel() + }() + + primaryErr := handlePrimary(vtctlReparent.winningPrimaryTabletAliasStr, newPrimaryTabletInfo) + if primaryErr != nil { + logger.Warningf("master failed to PopulateReparentJournal") + replCancel() + + return vterrors.Wrapf(primaryErr, "failed to PopulateReparentJournal on master: %v", primaryErr) + } + + select { + case <-replSuccessCtx.Done(): + // At least one replica was able to SetMaster successfully + return nil + case <-allReplicasDoneCtx.Done(): + // There are certain timing issues between replSuccessCtx.Done firing + // and allReplicasDoneCtx.Done firing, so we check again if truly all + // replicas failed (where `numReplicas` goroutines recorded an error) or + // one or more actually managed to succeed. + errCount := len(rec.Errors) + + switch { + case errCount > numReplicas: + // Technically, rec.Errors should never be greater than numReplicas, + // but it's better to err on the side of caution here, but also + // we're going to be explicit that this is doubly unexpected. + return vterrors.Wrapf(rec.Error(), "received more errors (= %d) than replicas (= %d), which should be impossible: %v", errCount, numReplicas, rec.Error()) + case errCount == numReplicas: + return vterrors.Wrapf(rec.Error(), "%d replica(s) failed: %v", numReplicas, rec.Error()) + default: + return nil + } + } +} + // ChooseNewPrimary finds a tablet that should become a primary after reparent. // The criteria for the new primary-elect are (preferably) to be in the same // cell as the current primary, and to be different from avoidPrimaryAlias. The From 7bfc937012e786ccef14609f3ee8ecf4ee2e7d13 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Thu, 22 Jul 2021 19:22:08 +0530 Subject: [PATCH 011/176] make vtorc use the new reparenter Signed-off-by: GuptaManan100 --- .../orchestrator/logic/reparent_functions.go | 20 ++- go/vt/orchestrator/logic/topology_recovery.go | 154 ++---------------- .../reparentutil/emergency_reparenter_2.go | 7 +- go/vt/orchestrator/reparentutil/util.go | 17 +- 4 files changed, 28 insertions(+), 170 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index a2204e5dce3..4ab13bd6ac4 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -21,6 +21,8 @@ import ( "fmt" "time" + "vitess.io/vitess/go/vt/orchestrator/reparentutil" + "vitess.io/vitess/go/vt/orchestrator/attributes" "vitess.io/vitess/go/vt/orchestrator/kv" @@ -38,6 +40,8 @@ import ( "vitess.io/vitess/go/vt/topo" ) +var _ reparentutil.ReparentFunctions = (*VtOrcReparentFunctions)(nil) + // VtOrcReparentFunctions is the VtOrc implementation for ReparentFunctions type VtOrcReparentFunctions struct { analysisEntry inst.ReplicationAnalysis @@ -46,6 +50,7 @@ type VtOrcReparentFunctions struct { topologyRecovery *TopologyRecovery promotedReplica *inst.Instance lostReplicas [](*inst.Instance) + recoveryAttempted bool } // LockShard implements the ReparentFunctions interface @@ -113,16 +118,14 @@ func (vtorcReparent *VtOrcReparentFunctions) StopReplicationAndBuildStatusMaps(c return err } -// GetPrimaryRecoveryType implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) GetPrimaryRecoveryType() MasterRecoveryType { +// CheckPrimaryRecoveryType implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) CheckPrimaryRecoveryType() error { vtorcReparent.topologyRecovery.RecoveryType = GetMasterRecoveryType(&vtorcReparent.topologyRecovery.AnalysisEntry) AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: masterRecoveryType=%+v", vtorcReparent.topologyRecovery.RecoveryType)) - return vtorcReparent.topologyRecovery.RecoveryType -} - -// AddError implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) AddError(errorMsg string) error { - return vtorcReparent.topologyRecovery.AddError(log.Errorf(errorMsg)) + if vtorcReparent.topologyRecovery.RecoveryType != MasterRecoveryGTID { + return vtorcReparent.topologyRecovery.AddError(log.Errorf("RecoveryType unknown/unsupported")) + } + return nil } // FindPrimaryCandidates implements the ReparentFunctions interface @@ -188,6 +191,7 @@ func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.C vtorcReparent.promotedReplica = promotedReplica vtorcReparent.lostReplicas = lostReplicas + vtorcReparent.recoveryAttempted = true return nil } diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 1ffc048ffc6..53bbb06e1df 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -27,14 +27,15 @@ import ( "sync/atomic" "time" + "vitess.io/vitess/go/vt/orchestrator/reparentutil" + "vitess.io/vitess/go/vt/vttablet/tmclient" + "github.com/patrickmn/go-cache" "github.com/rcrowley/go-metrics" - "vitess.io/vitess/go/vt/orchestrator/attributes" "vitess.io/vitess/go/vt/orchestrator/config" "vitess.io/vitess/go/vt/orchestrator/external/golib/log" "vitess.io/vitess/go/vt/orchestrator/inst" - "vitess.io/vitess/go/vt/orchestrator/kv" ometrics "vitess.io/vitess/go/vt/orchestrator/metrics" "vitess.io/vitess/go/vt/orchestrator/os" "vitess.io/vitess/go/vt/orchestrator/process" @@ -826,150 +827,15 @@ func checkAndRecoverDeadMaster(analysisEntry inst.ReplicationAnalysis, candidate } log.Infof("Analysis: %v, deadmaster %+v", analysisEntry.Analysis, analysisEntry.AnalyzedInstanceKey) - // That's it! We must do recovery! - // TODO(sougou): This function gets called by GracefulMasterTakeover which may - // need to obtain shard lock before getting here. - _, unlock, err := LockShard(context.Background(), analysisEntry.AnalyzedInstanceKey) - if err != nil { - log.Infof("CheckAndRecover: Analysis: %+v, InstanceKey: %+v, candidateInstanceKey: %+v, "+ - "skipProcesses: %v: NOT detecting/recovering host, could not obtain shard lock (%v)", - analysisEntry.Analysis, analysisEntry.AnalyzedInstanceKey, candidateInstanceKey, skipProcesses, err) - return false, nil, err - } - defer unlock(&err) - - // Check if someone else fixed the problem. - tablet, err := TabletRefresh(analysisEntry.AnalyzedInstanceKey) - if err == nil && tablet.Type != topodatapb.TabletType_MASTER { - // TODO(sougou); use a version that only refreshes the current shard. - RefreshTablets() - AuditTopologyRecovery(topologyRecovery, "another agent seems to have fixed the problem") - // TODO(sougou): see if we have to reset the cluster as healthy. - return false, topologyRecovery, nil - } - - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("will handle DeadMaster event on %+v", analysisEntry.ClusterDetails.ClusterName)) - recoverDeadMasterCounter.Inc(1) - recoveryAttempted, promotedReplica, lostReplicas, err := recoverDeadMaster(topologyRecovery, candidateInstanceKey, skipProcesses) - if err != nil { - AuditTopologyRecovery(topologyRecovery, err.Error()) - } - topologyRecovery.LostReplicas.AddInstances(lostReplicas) - if !recoveryAttempted { - return false, topologyRecovery, err - } - - overrideMasterPromotion := func() (*inst.Instance, error) { - if promotedReplica == nil { - // No promotion; nothing to override. - return promotedReplica, err - } - // Scenarios where we might cancel the promotion. - if satisfied, reason := MasterFailoverGeographicConstraintSatisfied(&analysisEntry, promotedReplica); !satisfied { - return nil, fmt.Errorf("RecoverDeadMaster: failed %+v promotion; %s", promotedReplica.Key, reason) - } - if config.Config.FailMasterPromotionOnLagMinutes > 0 && - time.Duration(promotedReplica.ReplicationLagSeconds.Int64)*time.Second >= time.Duration(config.Config.FailMasterPromotionOnLagMinutes)*time.Minute { - // candidate replica lags too much - return nil, fmt.Errorf("RecoverDeadMaster: failed promotion. FailMasterPromotionOnLagMinutes is set to %d (minutes) and promoted replica %+v 's lag is %d (seconds)", config.Config.FailMasterPromotionOnLagMinutes, promotedReplica.Key, promotedReplica.ReplicationLagSeconds.Int64) - } - if config.Config.FailMasterPromotionIfSQLThreadNotUpToDate && !promotedReplica.SQLThreadUpToDate() { - return nil, fmt.Errorf("RecoverDeadMaster: failed promotion. FailMasterPromotionIfSQLThreadNotUpToDate is set and promoted replica %+v 's sql thread is not up to date (relay logs still unapplied). Aborting promotion", promotedReplica.Key) - } - if config.Config.DelayMasterPromotionIfSQLThreadNotUpToDate && !promotedReplica.SQLThreadUpToDate() { - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("DelayMasterPromotionIfSQLThreadNotUpToDate: waiting for SQL thread on %+v", promotedReplica.Key)) - if _, err := inst.WaitForSQLThreadUpToDate(&promotedReplica.Key, 0, 0); err != nil { - return nil, fmt.Errorf("DelayMasterPromotionIfSQLThreadNotUpToDate error: %+v", err) - } - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("DelayMasterPromotionIfSQLThreadNotUpToDate: SQL thread caught up on %+v", promotedReplica.Key)) - } - // All seems well. No override done. - return promotedReplica, err - } - if promotedReplica, err = overrideMasterPromotion(); err != nil { - AuditTopologyRecovery(topologyRecovery, err.Error()) - } - // And this is the end; whether successful or not, we're done. - resolveRecovery(topologyRecovery, promotedReplica) - // Now, see whether we are successful or not. From this point there's no going back. - if promotedReplica != nil { - // Success! - recoverDeadMasterSuccessCounter.Inc(1) - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("RecoverDeadMaster: successfully promoted %+v", promotedReplica.Key)) - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: promoted server coordinates: %+v", promotedReplica.SelfBinlogCoordinates)) - - AuditTopologyRecovery(topologyRecovery, "- RecoverDeadMaster: will apply MySQL changes to promoted master") - { - _, err := inst.ResetReplicationOperation(&promotedReplica.Key) - if err != nil { - // Ugly, but this is important. Let's give it another try - _, err = inst.ResetReplicationOperation(&promotedReplica.Key) - } - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying RESET SLAVE ALL on promoted master: success=%t", (err == nil))) - if err != nil { - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: NOTE that %+v is promoted even though SHOW SLAVE STATUS may still show it has a master", promotedReplica.Key)) - } - } - { - count := inst.MasterSemiSync(promotedReplica.Key) - err := inst.SetSemiSyncMaster(&promotedReplica.Key, count > 0) - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying semi-sync %v: success=%t", count > 0, (err == nil))) - - // Dont' allow writes if semi-sync settings fail. - if err == nil { - _, err := inst.SetReadOnly(&promotedReplica.Key, false) - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying read-only=0 on promoted master: success=%t", (err == nil))) - } - } - // Let's attempt, though we won't necessarily succeed, to set old master as read-only - go func() { - _, err := inst.SetReadOnly(&analysisEntry.AnalyzedInstanceKey, true) - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying read-only=1 on demoted master: success=%t", (err == nil))) - }() - - kvPairs := inst.GetClusterMasterKVPairs(analysisEntry.ClusterDetails.ClusterAlias, &promotedReplica.Key) - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("Writing KV %+v", kvPairs)) - for _, kvPair := range kvPairs { - err := kv.PutKVPair(kvPair) - log.Errore(err) - } - { - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("Distributing KV %+v", kvPairs)) - err := kv.DistributePairs(kvPairs) - log.Errore(err) - } - if config.Config.MasterFailoverDetachReplicaMasterHost { - postponedFunction := func() error { - AuditTopologyRecovery(topologyRecovery, "- RecoverDeadMaster: detaching master host on promoted master") - inst.DetachReplicaMasterHost(&promotedReplica.Key) - return nil - } - topologyRecovery.AddPostponedFunction(postponedFunction, fmt.Sprintf("RecoverDeadMaster, detaching promoted master host %+v", promotedReplica.Key)) - } - func() error { - before := analysisEntry.AnalyzedInstanceKey.StringCode() - after := promotedReplica.Key.StringCode() - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: updating cluster_alias: %v -> %v", before, after)) - //~~~inst.ReplaceClusterName(before, after) - if alias := analysisEntry.ClusterDetails.ClusterAlias; alias != "" { - inst.SetClusterAlias(promotedReplica.Key.StringCode(), alias) - } else { - inst.ReplaceAliasClusterName(before, after) - } - return nil - }() - - attributes.SetGeneralAttribute(analysisEntry.ClusterDetails.ClusterDomain, promotedReplica.Key.StringCode()) - - if !skipProcesses { - // Execute post master-failover processes - executeProcesses(config.Config.PostMasterFailoverProcesses, "PostMasterFailoverProcesses", topologyRecovery, false) - } - } else { - recoverDeadMasterFailureCounter.Inc(1) + reparentFunctions := &VtOrcReparentFunctions{ + analysisEntry: analysisEntry, + candidateInstanceKey: candidateInstanceKey, + skipProcesses: skipProcesses, + topologyRecovery: topologyRecovery, } + _, err = reparentutil.NewEmergencyReparenter2(tmclient.NewTabletManagerClient(), nil).ReparentShard(context.Background(), reparentFunctions) - return true, topologyRecovery, err + return reparentFunctions.recoveryAttempted, topologyRecovery, err } // isGenerallyValidAsCandidateSiblingOfIntermediateMaster sees that basic server configuration and state are valid diff --git a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go index 15dc107811f..9882f300b9d 100644 --- a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go +++ b/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go @@ -23,8 +23,6 @@ import ( "google.golang.org/protobuf/proto" - "vitess.io/vitess/go/vt/orchestrator/logic" - "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/vterrors" @@ -114,9 +112,8 @@ func (erp *EmergencyReparenter2) reparentShardLocked(ctx context.Context, ev *ev return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } - recoveryType := reparentFunctions.GetPrimaryRecoveryType() - if recoveryType != logic.MasterRecoveryGTID { - return reparentFunctions.AddError("RecoveryType unknown/unsupported") + if err := reparentFunctions.CheckPrimaryRecoveryType(); err != nil { + return err } if err := reparentFunctions.FindPrimaryCandidates(ctx, erp.logger, erp.tmc); err != nil { diff --git a/go/vt/orchestrator/reparentutil/util.go b/go/vt/orchestrator/reparentutil/util.go index ce23af53797..848f89df480 100644 --- a/go/vt/orchestrator/reparentutil/util.go +++ b/go/vt/orchestrator/reparentutil/util.go @@ -34,8 +34,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" - "vitess.io/vitess/go/vt/orchestrator/logic" - "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" @@ -57,8 +55,7 @@ type ( CheckIfFixed() bool PreRecoveryProcesses(context.Context) error StopReplicationAndBuildStatusMaps(context.Context, tmclient.TabletManagerClient, *events.Reparent, logutil.Logger) error - GetPrimaryRecoveryType() logic.MasterRecoveryType - AddError(string) error + CheckPrimaryRecoveryType() error FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient) error CheckIfNeedToOverridePrimary() error StartReplication(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) error @@ -84,7 +81,6 @@ type ( ) var ( - _ ReparentFunctions = (*logic.VtOrcReparentFunctions)(nil) _ ReparentFunctions = (*VtctlReparentFunctions)(nil) ) @@ -135,14 +131,9 @@ func (vtctlReparent *VtctlReparentFunctions) StopReplicationAndBuildStatusMaps(c return nil } -// GetPrimaryRecoveryType implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) GetPrimaryRecoveryType() logic.MasterRecoveryType { - return logic.MasterRecoveryGTID -} - -// AddError implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) AddError(errorMsg string) error { - return vterrors.New(vtrpc.Code_INTERNAL, errorMsg) +// CheckPrimaryRecoveryType implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) CheckPrimaryRecoveryType() error { + return nil } // FindPrimaryCandidates implements the ReparentFunctions interface From 73d609ca74294a0268de03f072248c3026838f7b Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Thu, 22 Jul 2021 19:24:35 +0530 Subject: [PATCH 012/176] moved the package back Signed-off-by: GuptaManan100 --- go/vt/orchestrator/logic/reparent_functions.go | 3 +-- go/vt/orchestrator/logic/topology_recovery.go | 2 +- go/vt/vtctl/grpcvtctldserver/server.go | 2 +- .../reparentutil/emergency_reparenter.go | 0 .../reparentutil/emergency_reparenter_2.go | 0 .../reparentutil/emergency_reparenter_test.go | 0 .../{orchestrator => vtctl}/reparentutil/planned_reparenter.go | 0 .../reparentutil/planned_reparenter_test.go | 0 go/vt/{orchestrator => vtctl}/reparentutil/replication.go | 0 go/vt/{orchestrator => vtctl}/reparentutil/replication_test.go | 0 go/vt/{orchestrator => vtctl}/reparentutil/util.go | 0 go/vt/{orchestrator => vtctl}/reparentutil/util_test.go | 0 go/vt/wrangler/reparent.go | 2 +- 13 files changed, 4 insertions(+), 5 deletions(-) rename go/vt/{orchestrator => vtctl}/reparentutil/emergency_reparenter.go (100%) rename go/vt/{orchestrator => vtctl}/reparentutil/emergency_reparenter_2.go (100%) rename go/vt/{orchestrator => vtctl}/reparentutil/emergency_reparenter_test.go (100%) rename go/vt/{orchestrator => vtctl}/reparentutil/planned_reparenter.go (100%) rename go/vt/{orchestrator => vtctl}/reparentutil/planned_reparenter_test.go (100%) rename go/vt/{orchestrator => vtctl}/reparentutil/replication.go (100%) rename go/vt/{orchestrator => vtctl}/reparentutil/replication_test.go (100%) rename go/vt/{orchestrator => vtctl}/reparentutil/util.go (100%) rename go/vt/{orchestrator => vtctl}/reparentutil/util_test.go (100%) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 4ab13bd6ac4..de19f2e7453 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -21,10 +21,9 @@ import ( "fmt" "time" - "vitess.io/vitess/go/vt/orchestrator/reparentutil" - "vitess.io/vitess/go/vt/orchestrator/attributes" "vitess.io/vitess/go/vt/orchestrator/kv" + "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/vttablet/tmclient" diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 53bbb06e1df..cb242090d5f 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -27,7 +27,7 @@ import ( "sync/atomic" "time" - "vitess.io/vitess/go/vt/orchestrator/reparentutil" + "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/vttablet/tmclient" "github.com/patrickmn/go-cache" diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index c77936a2258..8a5835857f7 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -38,12 +38,12 @@ import ( "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" "vitess.io/vitess/go/vt/mysqlctl/mysqlctlproto" - "vitess.io/vitess/go/vt/orchestrator/reparentutil" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/vt/topotools/events" + "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/vtctl/workflow" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vttablet/tmclient" diff --git a/go/vt/orchestrator/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go similarity index 100% rename from go/vt/orchestrator/reparentutil/emergency_reparenter.go rename to go/vt/vtctl/reparentutil/emergency_reparenter.go diff --git a/go/vt/orchestrator/reparentutil/emergency_reparenter_2.go b/go/vt/vtctl/reparentutil/emergency_reparenter_2.go similarity index 100% rename from go/vt/orchestrator/reparentutil/emergency_reparenter_2.go rename to go/vt/vtctl/reparentutil/emergency_reparenter_2.go diff --git a/go/vt/orchestrator/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go similarity index 100% rename from go/vt/orchestrator/reparentutil/emergency_reparenter_test.go rename to go/vt/vtctl/reparentutil/emergency_reparenter_test.go diff --git a/go/vt/orchestrator/reparentutil/planned_reparenter.go b/go/vt/vtctl/reparentutil/planned_reparenter.go similarity index 100% rename from go/vt/orchestrator/reparentutil/planned_reparenter.go rename to go/vt/vtctl/reparentutil/planned_reparenter.go diff --git a/go/vt/orchestrator/reparentutil/planned_reparenter_test.go b/go/vt/vtctl/reparentutil/planned_reparenter_test.go similarity index 100% rename from go/vt/orchestrator/reparentutil/planned_reparenter_test.go rename to go/vt/vtctl/reparentutil/planned_reparenter_test.go diff --git a/go/vt/orchestrator/reparentutil/replication.go b/go/vt/vtctl/reparentutil/replication.go similarity index 100% rename from go/vt/orchestrator/reparentutil/replication.go rename to go/vt/vtctl/reparentutil/replication.go diff --git a/go/vt/orchestrator/reparentutil/replication_test.go b/go/vt/vtctl/reparentutil/replication_test.go similarity index 100% rename from go/vt/orchestrator/reparentutil/replication_test.go rename to go/vt/vtctl/reparentutil/replication_test.go diff --git a/go/vt/orchestrator/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go similarity index 100% rename from go/vt/orchestrator/reparentutil/util.go rename to go/vt/vtctl/reparentutil/util.go diff --git a/go/vt/orchestrator/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go similarity index 100% rename from go/vt/orchestrator/reparentutil/util_test.go rename to go/vt/vtctl/reparentutil/util_test.go diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index 3aaddb48fee..482df8378f7 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -31,12 +31,12 @@ import ( "vitess.io/vitess/go/event" "vitess.io/vitess/go/vt/concurrency" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/orchestrator/reparentutil" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/topotools" "vitess.io/vitess/go/vt/topotools/events" "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver" + "vitess.io/vitess/go/vt/vtctl/reparentutil" replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" From 08103c3c0bef2288f23b71352bd949168257a59c Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Thu, 22 Jul 2021 19:47:42 +0530 Subject: [PATCH 013/176] bug fix Signed-off-by: GuptaManan100 --- go/vt/orchestrator/logic/reparent_functions.go | 4 ++-- go/vt/orchestrator/logic/tablet_discovery.go | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index de19f2e7453..878643ccf34 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -54,7 +54,7 @@ type VtOrcReparentFunctions struct { // LockShard implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) LockShard(ctx context.Context) (context.Context, func(*error), error) { - _, unlock, err := LockShard(ctx, vtorcReparent.analysisEntry.AnalyzedInstanceKey) + ctx, unlock, err := LockShard(ctx, vtorcReparent.analysisEntry.AnalyzedInstanceKey) if err != nil { log.Infof("CheckAndRecover: Analysis: %+v, InstanceKey: %+v, candidateInstanceKey: %+v, "+ "skipProcesses: %v: NOT detecting/recovering host, could not obtain shard lock (%v)", @@ -114,7 +114,7 @@ func (vtorcReparent *VtOrcReparentFunctions) PreRecoveryProcesses(ctx context.Co func (vtorcReparent *VtOrcReparentFunctions) StopReplicationAndBuildStatusMaps(context.Context, tmclient.TabletManagerClient, *events.Reparent, logutil.Logger) error { err := TabletDemoteMaster(vtorcReparent.analysisEntry.AnalyzedInstanceKey) AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: TabletDemoteMaster: %v", err)) - return err + return nil } // CheckPrimaryRecoveryType implements the ReparentFunctions interface diff --git a/go/vt/orchestrator/logic/tablet_discovery.go b/go/vt/orchestrator/logic/tablet_discovery.go index 44c271628ee..6cd73f5bb98 100644 --- a/go/vt/orchestrator/logic/tablet_discovery.go +++ b/go/vt/orchestrator/logic/tablet_discovery.go @@ -252,8 +252,7 @@ func LockShard(ctx context.Context, instanceKey inst.InstanceKey) (context.Conte if err != nil { return nil, nil, err } - ctx, cancel := context.WithTimeout(ctx, time.Duration(config.Config.LockShardTimeoutSeconds)*time.Second) - defer cancel() + ctx, _ = context.WithTimeout(ctx, time.Duration(config.Config.LockShardTimeoutSeconds)*time.Second) atomic.AddInt32(&shardsLockCounter, 1) ctx, unlock, err := ts.LockShard(ctx, tablet.Keyspace, tablet.Shard, "Orc Recovery") if err != nil { From 281455fa99e67554c28da9b4351a89ca7e348695 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 16 Aug 2021 18:23:17 +0530 Subject: [PATCH 014/176] call the cancel function Signed-off-by: GuptaManan100 --- go/vt/orchestrator/logic/tablet_discovery.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go/vt/orchestrator/logic/tablet_discovery.go b/go/vt/orchestrator/logic/tablet_discovery.go index a40d7bff1a9..ae7062d7971 100644 --- a/go/vt/orchestrator/logic/tablet_discovery.go +++ b/go/vt/orchestrator/logic/tablet_discovery.go @@ -252,16 +252,18 @@ func LockShard(ctx context.Context, instanceKey inst.InstanceKey) (context.Conte if err != nil { return nil, nil, err } - ctx, _ = context.WithTimeout(ctx, time.Duration(config.Config.LockShardTimeoutSeconds)*time.Second) + ctx, cancel := context.WithTimeout(ctx, time.Duration(config.Config.LockShardTimeoutSeconds)*time.Second) atomic.AddInt32(&shardsLockCounter, 1) ctx, unlock, err := ts.LockShard(ctx, tablet.Keyspace, tablet.Shard, "Orc Recovery") if err != nil { + cancel() atomic.AddInt32(&shardsLockCounter, -1) return nil, nil, err } return ctx, func(e *error) { defer atomic.AddInt32(&shardsLockCounter, -1) unlock(e) + cancel() }, nil } From 9ada8aded376a50cf00909638dad7afb298c23d7 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 16 Aug 2021 18:24:16 +0530 Subject: [PATCH 015/176] remove dead code Signed-off-by: GuptaManan100 --- go/vt/orchestrator/logic/topology_recovery.go | 204 ------------------ 1 file changed, 204 deletions(-) diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index ac121bf41e2..2142af613cf 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -377,101 +377,6 @@ func executeProcesses(processes []string, description string, topologyRecovery * return err } -func recoverDeadMasterInBinlogServerTopology(topologyRecovery *TopologyRecovery) (promotedReplica *inst.Instance, err error) { - failedMasterKey := &topologyRecovery.AnalysisEntry.AnalyzedInstanceKey - - var promotedBinlogServer *inst.Instance - - _, promotedBinlogServer, err = inst.RegroupReplicasBinlogServers(failedMasterKey, true) - if err != nil { - return nil, log.Errore(err) - } - promotedBinlogServer, err = inst.StopReplication(&promotedBinlogServer.Key) - if err != nil { - return promotedReplica, log.Errore(err) - } - // Find candidate replica - promotedReplica, err = inst.GetCandidateReplicaOfBinlogServerTopology(&promotedBinlogServer.Key) - if err != nil { - return promotedReplica, log.Errore(err) - } - // Align it with binlog server coordinates - promotedReplica, err = inst.StopReplication(&promotedReplica.Key) - if err != nil { - return promotedReplica, log.Errore(err) - } - promotedReplica, err = inst.StartReplicationUntilMasterCoordinates(&promotedReplica.Key, &promotedBinlogServer.ExecBinlogCoordinates) - if err != nil { - return promotedReplica, log.Errore(err) - } - promotedReplica, err = inst.StopReplication(&promotedReplica.Key) - if err != nil { - return promotedReplica, log.Errore(err) - } - // Detach, flush binary logs forward - promotedReplica, err = inst.ResetReplication(&promotedReplica.Key) - if err != nil { - return promotedReplica, log.Errore(err) - } - promotedReplica, err = inst.FlushBinaryLogsTo(&promotedReplica.Key, promotedBinlogServer.ExecBinlogCoordinates.LogFile) - if err != nil { - return promotedReplica, log.Errore(err) - } - promotedReplica, err = inst.FlushBinaryLogs(&promotedReplica.Key, 1) - if err != nil { - return promotedReplica, log.Errore(err) - } - promotedReplica, err = inst.PurgeBinaryLogsToLatest(&promotedReplica.Key, false) - if err != nil { - return promotedReplica, log.Errore(err) - } - // Reconnect binlog servers to promoted replica (now primary): - promotedBinlogServer, err = inst.SkipToNextBinaryLog(&promotedBinlogServer.Key) - if err != nil { - return promotedReplica, log.Errore(err) - } - promotedBinlogServer, err = inst.Repoint(&promotedBinlogServer.Key, &promotedReplica.Key, inst.GTIDHintDeny) - if err != nil { - return nil, log.Errore(err) - } - - func() { - // Move binlog server replicas up to replicate from primary. - // This can only be done once a BLS has skipped to the next binlog - // We postpone this operation. The primary is already promoted and we're happy. - binlogServerReplicas, err := inst.ReadBinlogServerReplicaInstances(&promotedBinlogServer.Key) - if err != nil { - return - } - maxBinlogServersToPromote := 3 - for i, binlogServerReplica := range binlogServerReplicas { - binlogServerReplica := binlogServerReplica - if i >= maxBinlogServersToPromote { - return - } - postponedFunction := func() error { - binlogServerReplica, err := inst.StopReplication(&binlogServerReplica.Key) - if err != nil { - return err - } - // Make sure the BLS has the "next binlog" -- the one the primary flushed & purged to. Otherwise the BLS - // will request a binlog the primary does not have - if binlogServerReplica.ExecBinlogCoordinates.SmallerThan(&promotedBinlogServer.ExecBinlogCoordinates) { - binlogServerReplica, err = inst.StartReplicationUntilMasterCoordinates(&binlogServerReplica.Key, &promotedBinlogServer.ExecBinlogCoordinates) - if err != nil { - return err - } - } - _, err = inst.Repoint(&binlogServerReplica.Key, &promotedReplica.Key, inst.GTIDHintDeny) - return err - } - topologyRecovery.AddPostponedFunction(postponedFunction, fmt.Sprintf("recoverDeadMasterInBinlogServerTopology, moving binlog server %+v", binlogServerReplica.Key)) - } - }() - - return promotedReplica, err -} - func GetMasterRecoveryType(analysisEntry *inst.ReplicationAnalysis) (masterRecoveryType MasterRecoveryType) { masterRecoveryType = MasterRecoveryUnknown if analysisEntry.OracleGTIDImmediateTopology || analysisEntry.MariaDBGTIDImmediateTopology { @@ -482,115 +387,6 @@ func GetMasterRecoveryType(analysisEntry *inst.ReplicationAnalysis) (masterRecov return masterRecoveryType } -// recoverDeadMaster recovers a dead primary, complete logic inside -func recoverDeadMaster(topologyRecovery *TopologyRecovery, candidateInstanceKey *inst.InstanceKey, skipProcesses bool) (recoveryAttempted bool, promotedReplica *inst.Instance, lostReplicas [](*inst.Instance), err error) { - topologyRecovery.Type = MasterRecovery - analysisEntry := &topologyRecovery.AnalysisEntry - failedInstanceKey := &analysisEntry.AnalyzedInstanceKey - var cannotReplicateReplicas [](*inst.Instance) - postponedAll := false - - inst.AuditOperation("recover-dead-master", failedInstanceKey, "problem found; will recover") - if !skipProcesses { - if err := executeProcesses(config.Config.PreFailoverProcesses, "PreFailoverProcesses", topologyRecovery, true); err != nil { - return false, nil, lostReplicas, topologyRecovery.AddError(err) - } - } - - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("RecoverDeadMaster: will recover %+v", *failedInstanceKey)) - - err = TabletDemoteMaster(*failedInstanceKey) - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("RecoverDeadMaster: TabletDemoteMaster: %v", err)) - - topologyRecovery.RecoveryType = GetMasterRecoveryType(analysisEntry) - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("RecoverDeadMaster: masterRecoveryType=%+v", topologyRecovery.RecoveryType)) - - promotedReplicaIsIdeal := func(promoted *inst.Instance, hasBestPromotionRule bool) bool { - if promoted == nil { - return false - } - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("RecoverDeadMaster: promotedReplicaIsIdeal(%+v)", promoted.Key)) - if candidateInstanceKey != nil { //explicit request to promote a specific server - return promoted.Key.Equals(candidateInstanceKey) - } - if promoted.DataCenter == topologyRecovery.AnalysisEntry.AnalyzedInstanceDataCenter && - promoted.PhysicalEnvironment == topologyRecovery.AnalysisEntry.AnalyzedInstancePhysicalEnvironment { - if promoted.PromotionRule == inst.MustPromoteRule || promoted.PromotionRule == inst.PreferPromoteRule || - (hasBestPromotionRule && promoted.PromotionRule != inst.MustNotPromoteRule) { - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("RecoverDeadMaster: found %+v to be ideal candidate; will optimize recovery", promoted.Key)) - postponedAll = true - return true - } - } - return false - } - switch topologyRecovery.RecoveryType { - case MasterRecoveryUnknown: - { - return false, nil, lostReplicas, topologyRecovery.AddError(log.Errorf("RecoveryType unknown/unsupported")) - } - case MasterRecoveryGTID: - { - AuditTopologyRecovery(topologyRecovery, "RecoverDeadMaster: regrouping replicas via GTID") - lostReplicas, _, cannotReplicateReplicas, promotedReplica, err = inst.RegroupReplicasGTID(failedInstanceKey, true, nil, &topologyRecovery.PostponedFunctionsContainer, promotedReplicaIsIdeal) - } - case MasterRecoveryBinlogServer: - { - AuditTopologyRecovery(topologyRecovery, "RecoverDeadMaster: recovering via binlog servers") - promotedReplica, err = recoverDeadMasterInBinlogServerTopology(topologyRecovery) - } - } - topologyRecovery.AddError(err) - lostReplicas = append(lostReplicas, cannotReplicateReplicas...) - for _, replica := range lostReplicas { - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("RecoverDeadMaster: - lost replica: %+v", replica.Key)) - } - - if promotedReplica != nil && len(lostReplicas) > 0 && config.Config.DetachLostReplicasAfterMasterFailover { - postponedFunction := func() error { - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("RecoverDeadMaster: lost %+v replicas during recovery process; detaching them", len(lostReplicas))) - for _, replica := range lostReplicas { - replica := replica - inst.DetachReplicaMasterHost(&replica.Key) - } - return nil - } - topologyRecovery.AddPostponedFunction(postponedFunction, fmt.Sprintf("RecoverDeadMaster, detach %+v lost replicas", len(lostReplicas))) - } - - func() error { - // TODO(sougou): Commented out: this downtime feels a little aggressive. - //inst.BeginDowntime(inst.NewDowntime(failedInstanceKey, inst.GetMaintenanceOwner(), inst.DowntimeLostInRecoveryMessage, time.Duration(config.LostInRecoveryDowntimeSeconds)*time.Second)) - acknowledgeInstanceFailureDetection(&analysisEntry.AnalyzedInstanceKey) - for _, replica := range lostReplicas { - replica := replica - inst.BeginDowntime(inst.NewDowntime(&replica.Key, inst.GetMaintenanceOwner(), inst.DowntimeLostInRecoveryMessage, time.Duration(config.LostInRecoveryDowntimeSeconds)*time.Second)) - } - return nil - }() - - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("RecoverDeadMaster: %d postponed functions", topologyRecovery.PostponedFunctionsContainer.Len())) - - if promotedReplica != nil && !postponedAll { - promotedReplica, err = replacePromotedReplicaWithCandidate(topologyRecovery, &analysisEntry.AnalyzedInstanceKey, promotedReplica, candidateInstanceKey) - topologyRecovery.AddError(err) - } - - if promotedReplica == nil { - err := TabletUndoDemoteMaster(*failedInstanceKey) - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("RecoverDeadMaster: TabletUndoDemoteMaster: %v", err)) - message := "Failure: no replica promoted." - AuditTopologyRecovery(topologyRecovery, message) - inst.AuditOperation("recover-dead-master", failedInstanceKey, message) - return true, promotedReplica, lostReplicas, err - } - - message := fmt.Sprintf("promoted replica: %+v", promotedReplica.Key) - AuditTopologyRecovery(topologyRecovery, message) - inst.AuditOperation("recover-dead-master", failedInstanceKey, message) - return true, promotedReplica, lostReplicas, err -} - func MasterFailoverGeographicConstraintSatisfied(analysisEntry *inst.ReplicationAnalysis, suggestedInstance *inst.Instance) (satisfied bool, dissatisfiedReason string) { if config.Config.PreventCrossDataCenterMasterFailover { if suggestedInstance.DataCenter != analysisEntry.AnalyzedInstanceDataCenter { From 96b9fb5c65b38a538fdf44966e6133bcc404b2ce Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 16 Aug 2021 18:51:36 +0530 Subject: [PATCH 016/176] added creator functions for the reparenters and started switching tests Signed-off-by: GuptaManan100 --- .../orchestrator/logic/reparent_functions.go | 9 ++++++++ go/vt/orchestrator/logic/topology_recovery.go | 7 +------ .../reparentutil/emergency_reparenter_test.go | 21 +++++++++++-------- go/vt/vtctl/reparentutil/util.go | 6 ++++++ 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 878643ccf34..f47c3098df6 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -52,6 +52,15 @@ type VtOrcReparentFunctions struct { recoveryAttempted bool } +func NewVtorcReparentFunctions(analysisEntry inst.ReplicationAnalysis, candidateInstanceKey *inst.InstanceKey, skipProcesses bool, topologyRecovery *TopologyRecovery) *VtOrcReparentFunctions { + return &VtOrcReparentFunctions{ + analysisEntry: analysisEntry, + candidateInstanceKey: candidateInstanceKey, + skipProcesses: skipProcesses, + topologyRecovery: topologyRecovery, + } +} + // LockShard implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) LockShard(ctx context.Context) (context.Context, func(*error), error) { ctx, unlock, err := LockShard(ctx, vtorcReparent.analysisEntry.AnalyzedInstanceKey) diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 2142af613cf..a5a7cbb3e46 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -623,12 +623,7 @@ func checkAndRecoverDeadMaster(analysisEntry inst.ReplicationAnalysis, candidate } log.Infof("Analysis: %v, deadmaster %+v", analysisEntry.Analysis, analysisEntry.AnalyzedInstanceKey) - reparentFunctions := &VtOrcReparentFunctions{ - analysisEntry: analysisEntry, - candidateInstanceKey: candidateInstanceKey, - skipProcesses: skipProcesses, - topologyRecovery: topologyRecovery, - } + reparentFunctions := NewVtorcReparentFunctions(analysisEntry, candidateInstanceKey, skipProcesses, topologyRecovery) _, err = reparentutil.NewEmergencyReparenter2(tmclient.NewTabletManagerClient(), nil).ReparentShard(context.Background(), reparentFunctions) return reparentFunctions.recoveryAttempted, topologyRecovery, err diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index cea43ab129b..1f1801e36c4 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -92,7 +92,7 @@ func TestEmergencyReparenter_getLockAction(t *testing.T) { }, } - erp := &EmergencyReparenter{} + vtctlReparentFunctions := &VtctlReparentFunctions{} for _, tt := range tests { tt := tt @@ -100,7 +100,7 @@ func TestEmergencyReparenter_getLockAction(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - actual := erp.getLockAction(tt.alias) + actual := vtctlReparentFunctions.getLockAction(tt.alias) assert.Equal(t, tt.expected, actual, tt.msg) }) } @@ -1169,7 +1169,6 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { "zone1-0000000101": {}, }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - opts: EmergencyReparentOptions{}, shouldErr: true, }, { @@ -1207,7 +1206,6 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - opts: EmergencyReparentOptions{}, shouldErr: true, }, { @@ -1246,7 +1244,6 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - opts: EmergencyReparentOptions{}, shouldErr: true, }, { @@ -1287,7 +1284,6 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - opts: EmergencyReparentOptions{}, shouldErr: true, }, { @@ -1341,7 +1337,6 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - opts: EmergencyReparentOptions{}, shouldErr: true, }, { @@ -1488,9 +1483,17 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }() } - erp := NewEmergencyReparenter(tt.ts, tt.tmc, logger) + var err error + vtctlReparentFunctions := NewVtctlReparentFunctions(tt.ts) + vtctlReparentFunctions.keyspace = tt.keyspace + vtctlReparentFunctions.shard = tt.shard + vtctlReparentFunctions.winningPrimaryTabletAliasStr = tt.newPrimaryTabletAlias + vtctlReparentFunctions.tabletMap = tt.tabletMap + vtctlReparentFunctions.statusMap = tt.statusMap + vtctlReparentFunctions.IgnoreReplicas = tt.opts.IgnoreReplicas + vtctlReparentFunctions.WaitReplicasTimeout = tt.opts.WaitReplicasTimeout - err := erp.promoteNewPrimary(ctx, ev, tt.keyspace, tt.shard, tt.newPrimaryTabletAlias, tt.tabletMap, tt.statusMap, tt.opts) + err = vtctlReparentFunctions.promoteNewPrimary(ctx, ev, logger, tt.tmc) if tt.shouldErr { assert.Error(t, err) return diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 521063cfa8d..a752715083d 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -84,6 +84,12 @@ var ( _ ReparentFunctions = (*VtctlReparentFunctions)(nil) ) +func NewVtctlReparentFunctions(ts *topo.Server) *VtctlReparentFunctions { + return &VtctlReparentFunctions{ + ts: ts, + } +} + // LockShard implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) LockShard(ctx context.Context) (context.Context, func(*error), error) { vtctlReparent.lockAction = vtctlReparent.getLockAction(vtctlReparent.NewPrimaryAlias) From 8e5e41fae22870a980b723b4c6efd5d845baa895 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 16 Aug 2021 19:42:36 +0530 Subject: [PATCH 017/176] moved remaining unit tests to newer ers Signed-off-by: GuptaManan100 --- .../reparentutil/emergency_reparenter_test.go | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 1f1801e36c4..135a258c124 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -60,7 +60,7 @@ func TestNewEmergencyReparenter(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - er := NewEmergencyReparenter(nil, nil, tt.logger) + er := NewEmergencyReparenter2(nil, tt.logger) assert.NotNil(t, er.logger, "NewEmergencyReparenter should never result in a nil logger instance on the EmergencyReparenter") }) } @@ -1048,9 +1048,15 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { ctx = lctx // make the reparentShardLocked call use the lock ctx } - erp := NewEmergencyReparenter(tt.ts, tt.tmc, logger) + erp := NewEmergencyReparenter2(tt.tmc, logger) + vtctlReparentFunctions := NewVtctlReparentFunctions(tt.ts) + vtctlReparentFunctions.keyspace = tt.keyspace + vtctlReparentFunctions.shard = tt.shard + vtctlReparentFunctions.IgnoreReplicas = tt.opts.IgnoreReplicas + vtctlReparentFunctions.WaitReplicasTimeout = tt.opts.WaitReplicasTimeout + vtctlReparentFunctions.NewPrimaryAlias = tt.opts.NewPrimaryAlias - err := erp.reparentShardLocked(ctx, ev, tt.keyspace, tt.shard, tt.opts) + err := erp.reparentShardLocked(ctx, ev, vtctlReparentFunctions) if tt.shouldErr { assert.Error(t, err) return @@ -1740,8 +1746,12 @@ func TestEmergencyReparenter_waitForAllRelayLogsToApply(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - erp := NewEmergencyReparenter(nil, tt.tmc, logger) - err := erp.waitForAllRelayLogsToApply(ctx, tt.candidates, tt.tabletMap, tt.statusMap, opts) + vtctlReparentFunctions := NewVtctlReparentFunctions(nil) + vtctlReparentFunctions.tabletMap = tt.tabletMap + vtctlReparentFunctions.statusMap = tt.statusMap + vtctlReparentFunctions.WaitReplicasTimeout = opts.WaitReplicasTimeout + + err := vtctlReparentFunctions.waitForAllRelayLogsToApply(ctx, logger, tt.tmc, tt.candidates) if tt.shouldErr { assert.Error(t, err) return From eb3a127f939e1989ba99a0060c5e68e817a5adb3 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 16 Aug 2021 21:10:42 +0530 Subject: [PATCH 018/176] remove old ers Signed-off-by: GuptaManan100 --- go/vt/vtctl/grpcvtctldserver/server.go | 16 +- .../reparentutil/emergency_reparenter.go | 404 ------------------ .../reparentutil/emergency_reparenter_test.go | 264 ++++-------- go/vt/vtctl/reparentutil/util.go | 10 +- go/vt/wrangler/reparent.go | 17 +- 5 files changed, 112 insertions(+), 599 deletions(-) delete mode 100644 go/vt/vtctl/reparentutil/emergency_reparenter.go diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 6cbd824a365..f316658f1f2 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -588,15 +588,13 @@ func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldat logstream = append(logstream, e) }) - ev, err := reparentutil.NewEmergencyReparenter(s.ts, s.tmc, logger).ReparentShard(ctx, - req.Keyspace, - req.Shard, - reparentutil.EmergencyReparentOptions{ - NewPrimaryAlias: req.NewPrimary, - IgnoreReplicas: sets.NewString(ignoreReplicaAliases...), - WaitReplicasTimeout: waitReplicasTimeout, - }, - ) + ev, err := reparentutil.NewEmergencyReparenter2(s.tmc, logger).ReparentShard(ctx, + reparentutil.NewVtctlReparentFunctions(req.NewPrimary, + sets.NewString(ignoreReplicaAliases...), + waitReplicasTimeout, + req.Keyspace, + req.Shard, + s.ts)) resp := &vtctldatapb.EmergencyReparentShardResponse{ Keyspace: req.Keyspace, diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go deleted file mode 100644 index ae2f15e8241..00000000000 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ /dev/null @@ -1,404 +0,0 @@ -/* -Copyright 2021 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package reparentutil - -import ( - "context" - "fmt" - "sync" - "time" - - "google.golang.org/protobuf/proto" - - "k8s.io/apimachinery/pkg/util/sets" - - "vitess.io/vitess/go/event" - "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/vt/concurrency" - "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/topo" - "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/topotools/events" - "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/vt/vttablet/tmclient" - - logutilpb "vitess.io/vitess/go/vt/proto/logutil" - replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/proto/vtrpc" -) - -// EmergencyReparenter performs EmergencyReparentShard operations. -type EmergencyReparenter struct { - ts *topo.Server - tmc tmclient.TabletManagerClient - logger logutil.Logger -} - -// EmergencyReparentOptions provides optional parameters to -// EmergencyReparentShard operations. Options are passed by value, so it is safe -// for callers to mutate and reuse options structs for multiple calls. -type EmergencyReparentOptions struct { - NewPrimaryAlias *topodatapb.TabletAlias - IgnoreReplicas sets.String - WaitReplicasTimeout time.Duration - - // Private options managed internally. We use value passing to avoid leaking - // these details back out. - - lockAction string -} - -// NewEmergencyReparenter returns a new EmergencyReparenter object, ready to -// perform EmergencyReparentShard operations using the given topo.Server, -// TabletManagerClient, and logger. -// -// Providing a nil logger instance is allowed. -func NewEmergencyReparenter(ts *topo.Server, tmc tmclient.TabletManagerClient, logger logutil.Logger) *EmergencyReparenter { - erp := EmergencyReparenter{ - ts: ts, - tmc: tmc, - logger: logger, - } - - if erp.logger == nil { - // Create a no-op logger so we can call functions on er.logger without - // needed to constantly check for non-nil. - erp.logger = logutil.NewCallbackLogger(func(*logutilpb.Event) {}) - } - - return &erp -} - -// ReparentShard performs the EmergencyReparentShard operation on the given -// keyspace and shard. -func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace string, shard string, opts EmergencyReparentOptions) (*events.Reparent, error) { - opts.lockAction = erp.getLockAction(opts.NewPrimaryAlias) - - ctx, unlock, err := erp.ts.LockShard(ctx, keyspace, shard, opts.lockAction) - if err != nil { - return nil, err - } - - defer unlock(&err) - - ev := &events.Reparent{} - defer func() { - switch err { - case nil: - event.DispatchUpdate(ev, "finished EmergencyReparentShard") - default: - event.DispatchUpdate(ev, "failed EmergencyReparentShard: "+err.Error()) - } - }() - - err = erp.reparentShardLocked(ctx, ev, keyspace, shard, opts) - - return ev, err -} - -func (erp *EmergencyReparenter) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { - action := "EmergencyReparentShard" - - if newPrimaryAlias != nil { - action += fmt.Sprintf("(%v)", topoproto.TabletAliasString(newPrimaryAlias)) - } - - return action -} - -func (erp *EmergencyReparenter) promoteNewPrimary( - ctx context.Context, - ev *events.Reparent, - keyspace string, - shard string, - newPrimaryTabletAlias string, - tabletMap map[string]*topo.TabletInfo, - statusMap map[string]*replicationdatapb.StopReplicationStatus, - opts EmergencyReparentOptions, -) error { - erp.logger.Infof("promoting tablet %v to primary", newPrimaryTabletAlias) - event.DispatchUpdate(ev, "promoting replica") - - newPrimaryTabletInfo, ok := tabletMap[newPrimaryTabletAlias] - if !ok { - return vterrors.Errorf(vtrpc.Code_INTERNAL, "attempted to promote primary-elect %v that was not in the tablet map; this an impossible situation", newPrimaryTabletAlias) - } - - rp, err := erp.tmc.PromoteReplica(ctx, newPrimaryTabletInfo.Tablet) - if err != nil { - return vterrors.Wrapf(err, "primary-elect tablet %v failed to be upgraded to primary: %v", newPrimaryTabletAlias, err) - } - - if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { - return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) - } - - replCtx, replCancel := context.WithTimeout(ctx, opts.WaitReplicasTimeout) - defer replCancel() - - event.DispatchUpdate(ev, "reparenting all tablets") - - // Create a context and cancel function to watch for the first successful - // SetMaster call on a replica. We use a background context so that this - // context is only ever Done when its cancel is called by the background - // goroutine we're about to spin up. - // - // Similarly, create a context and cancel for the replica waiter goroutine - // to signal when all replica goroutines have finished. In the case where at - // least one replica succeeds, replSuccessCtx will be canceled first, while - // allReplicasDoneCtx is guaranteed to be canceled within - // opts.WaitReplicasTimeout plus some jitter. - replSuccessCtx, replSuccessCancel := context.WithCancel(context.Background()) - allReplicasDoneCtx, allReplicasDoneCancel := context.WithCancel(context.Background()) - - now := time.Now().UnixNano() - replWg := sync.WaitGroup{} - rec := concurrency.AllErrorRecorder{} - - handlePrimary := func(alias string, ti *topo.TabletInfo) error { - erp.logger.Infof("populating reparent journal on new primary %v", alias) - return erp.tmc.PopulateReparentJournal(replCtx, ti.Tablet, now, opts.lockAction, newPrimaryTabletInfo.Alias, rp) - } - - handleReplica := func(alias string, ti *topo.TabletInfo) { - defer replWg.Done() - erp.logger.Infof("setting new primary on replica %v", alias) - - forceStart := false - if status, ok := statusMap[alias]; ok { - fs, err := ReplicaWasRunning(status) - if err != nil { - err = vterrors.Wrapf(err, "tablet %v could not determine StopReplicationStatus: %v", alias, err) - rec.RecordError(err) - - return - } - - forceStart = fs - } - - err := erp.tmc.SetMaster(replCtx, ti.Tablet, newPrimaryTabletInfo.Alias, now, "", forceStart) - if err != nil { - err = vterrors.Wrapf(err, "tablet %v SetMaster failed: %v", alias, err) - rec.RecordError(err) - - return - } - - // Signal that at least one goroutine succeeded to SetMaster. - replSuccessCancel() - } - - numReplicas := 0 - - for alias, ti := range tabletMap { - switch { - case alias == newPrimaryTabletAlias: - continue - case !opts.IgnoreReplicas.Has(alias): - replWg.Add(1) - numReplicas++ - go handleReplica(alias, ti) - } - } - - // Spin up a background goroutine to wait until all replica goroutines - // finished. Polling this way allows us to have promoteNewPrimary return - // success as soon as (a) the primary successfully populates its reparent - // journal and (b) at least one replica successfully begins replicating. - // - // If we were to follow the more common pattern of blocking on replWg.Wait() - // in the main body of promoteNewPrimary, we would be bound to the - // time of slowest replica, instead of the time of the fastest successful - // replica, and we want ERS to be fast. - go func() { - replWg.Wait() - allReplicasDoneCancel() - }() - - primaryErr := handlePrimary(newPrimaryTabletAlias, newPrimaryTabletInfo) - if primaryErr != nil { - erp.logger.Warningf("primary failed to PopulateReparentJournal") - replCancel() - - return vterrors.Wrapf(primaryErr, "failed to PopulateReparentJournal on primary: %v", primaryErr) - } - - select { - case <-replSuccessCtx.Done(): - // At least one replica was able to SetMaster successfully - return nil - case <-allReplicasDoneCtx.Done(): - // There are certain timing issues between replSuccessCtx.Done firing - // and allReplicasDoneCtx.Done firing, so we check again if truly all - // replicas failed (where `numReplicas` goroutines recorded an error) or - // one or more actually managed to succeed. - errCount := len(rec.Errors) - - switch { - case errCount > numReplicas: - // Technically, rec.Errors should never be greater than numReplicas, - // but it's better to err on the side of caution here, but also - // we're going to be explicit that this is doubly unexpected. - return vterrors.Wrapf(rec.Error(), "received more errors (= %d) than replicas (= %d), which should be impossible: %v", errCount, numReplicas, rec.Error()) - case errCount == numReplicas: - return vterrors.Wrapf(rec.Error(), "%d replica(s) failed: %v", numReplicas, rec.Error()) - default: - return nil - } - } -} - -func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *events.Reparent, keyspace string, shard string, opts EmergencyReparentOptions) error { - shardInfo, err := erp.ts.GetShard(ctx, keyspace, shard) - if err != nil { - return err - } - - ev.ShardInfo = *shardInfo - - event.DispatchUpdate(ev, "reading all tablets") - - tabletMap, err := erp.ts.GetTabletMapForShard(ctx, keyspace, shard) - if err != nil { - return vterrors.Wrapf(err, "failed to get tablet map for %v/%v: %v", keyspace, shard, err) - } - - statusMap, primaryStatusMap, err := StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, tabletMap, opts.WaitReplicasTimeout, opts.IgnoreReplicas, erp.logger) - if err != nil { - return vterrors.Wrapf(err, "failed to stop replication and build status maps: %v", err) - } - - if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { - return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) - } - - validCandidates, err := FindValidEmergencyReparentCandidates(statusMap, primaryStatusMap) - if err != nil { - return err - } else if len(validCandidates) == 0 { - return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "no valid candidates for emergency reparent") - } - - // Wait for all candidates to apply relay logs - if err := erp.waitForAllRelayLogsToApply(ctx, validCandidates, tabletMap, statusMap, opts); err != nil { - return err - } - - // Elect the candidate with the most up-to-date position. - var ( - winningPosition mysql.Position - winningPrimaryTabletAliasStr string - ) - - for alias, position := range validCandidates { - if winningPosition.IsZero() || position.AtLeast(winningPosition) { - winningPosition = position - winningPrimaryTabletAliasStr = alias - } - } - - // If we were requested to elect a particular primary, verify it's a valid - // candidate (non-zero position, no errant GTIDs) and is at least as - // advanced as the winning position. - if opts.NewPrimaryAlias != nil { - winningPrimaryTabletAliasStr = topoproto.TabletAliasString(opts.NewPrimaryAlias) - pos, ok := validCandidates[winningPrimaryTabletAliasStr] - switch { - case !ok: - return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "primary elect %v has errant GTIDs", winningPrimaryTabletAliasStr) - case !pos.AtLeast(winningPosition): - return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "primary elect %v at position %v is not fully caught up. Winning position: %v", winningPrimaryTabletAliasStr, pos, winningPosition) - } - } - - // Check (again) we still have the topology lock. - if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { - return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) - } - - // Do the promotion. - if err := erp.promoteNewPrimary(ctx, ev, keyspace, shard, winningPrimaryTabletAliasStr, tabletMap, statusMap, opts); err != nil { - return err - } - - ev.NewPrimary = proto.Clone(tabletMap[winningPrimaryTabletAliasStr].Tablet).(*topodatapb.Tablet) - return nil -} - -func (erp *EmergencyReparenter) waitForAllRelayLogsToApply( - ctx context.Context, - validCandidates map[string]mysql.Position, - tabletMap map[string]*topo.TabletInfo, - statusMap map[string]*replicationdatapb.StopReplicationStatus, - opts EmergencyReparentOptions, -) error { - errCh := make(chan error) - defer close(errCh) - - groupCtx, groupCancel := context.WithTimeout(ctx, opts.WaitReplicasTimeout) - defer groupCancel() - - waiterCount := 0 - - for candidate := range validCandidates { - // When we called StopReplicationAndBuildStatusMaps, we got back two - // maps: (1) the StopReplicationStatus of any replicas that actually - // stopped replication; and (2) the MasterStatus of anything that - // returned ErrNotReplica, which is a tablet that is either the current - // primary or is stuck thinking it is a PRIMARY but is not in actuality. - // - // If we have a tablet in the validCandidates map that does not appear - // in the statusMap, then we have either (a) the current primary, which - // is not replicating, so it is not applying relay logs; or (b) a tablet - // that is stuck thinking it is PRIMARY but is not in actuality. In that - // second case - (b) - we will most likely find that the stuck PRIMARY - // does not have a winning position, and fail the ERS. If, on the other - // hand, it does have a winning position, we are trusting the operator - // to know what they are doing by emergency-reparenting onto that - // tablet. In either case, it does not make sense to wait for relay logs - // to apply on a tablet that was never applying relay logs in the first - // place, so we skip it, and log that we did. - status, ok := statusMap[candidate] - if !ok { - erp.logger.Infof("EmergencyReparent candidate %v not in replica status map; this means it was not running replication (because it was formerly PRIMARY), so skipping WaitForRelayLogsToApply step for this candidate", candidate) - continue - } - - go func(alias string, status *replicationdatapb.StopReplicationStatus) { - var err error - defer func() { errCh <- err }() - err = WaitForRelayLogsToApply(groupCtx, erp.tmc, tabletMap[alias], status) - }(candidate, status) - - waiterCount++ - } - - errgroup := concurrency.ErrorGroup{ - NumGoroutines: waiterCount, - NumRequiredSuccesses: waiterCount, - NumAllowedErrors: 0, - } - rec := errgroup.Wait(groupCancel, errCh) - - if len(rec.Errors) != 0 { - return vterrors.Wrapf(rec.Error(), "could not apply all relay logs within the provided WaitReplicasTimeout (%s): %v", opts.WaitReplicasTimeout, rec.Error()) - } - - return nil -} diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 135a258c124..7b8ab195018 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -110,23 +110,20 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { t.Parallel() tests := []struct { - name string + name string + vtctlReparentFunctions *VtctlReparentFunctions + tmc *testutil.TabletManagerClient // setup ts *topo.Server - tmc *testutil.TabletManagerClient unlockTopo bool shards []*vtctldatapb.Shard tablets []*topodatapb.Tablet - // params - keyspace string - shard string - opts EmergencyReparentOptions // results shouldErr bool }{ { - name: "success", - ts: memorytopo.NewServer("zone1"), + name: "success", + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000102": nil, @@ -222,16 +219,16 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Hostname: "most up-to-date position, wins election", }, }, - keyspace: "testkeyspace", - shard: "-", - opts: EmergencyReparentOptions{}, shouldErr: false, }, { // Here, all our tablets are tied, so we're going to explicitly pick // zone1-101. name: "success with requested primary-elect", - ts: memorytopo.NewServer("zone1"), + vtctlReparentFunctions: NewVtctlReparentFunctions(&topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000101": nil, @@ -326,19 +323,11 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Shard: "-", }, }, - keyspace: "testkeyspace", - shard: "-", - opts: EmergencyReparentOptions{ - NewPrimaryAlias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, shouldErr: false, }, { - name: "success with existing primary", - ts: memorytopo.NewServer("zone1"), + name: "success with existing primary", + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.PrimaryStatus @@ -436,25 +425,19 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Hostname: "most up-to-date position, wins election", }, }, - keyspace: "testkeyspace", - shard: "-", - opts: EmergencyReparentOptions{}, shouldErr: false, }, { - name: "shard not found", - ts: memorytopo.NewServer("zone1"), - tmc: &testutil.TabletManagerClient{}, - unlockTopo: true, // we shouldn't try to lock the nonexistent shard - shards: nil, - keyspace: "testkeyspace", - shard: "-", - opts: EmergencyReparentOptions{}, - shouldErr: true, + name: "shard not found", + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + tmc: &testutil.TabletManagerClient{}, + unlockTopo: true, // we shouldn't try to lock the nonexistent shard + shards: nil, + shouldErr: true, }, { - name: "cannot stop replication", - ts: memorytopo.NewServer("zone1"), + name: "cannot stop replication", + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -505,14 +488,11 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Shard: "-", }, }, - keyspace: "testkeyspace", - shard: "-", - opts: EmergencyReparentOptions{}, shouldErr: true, }, { - name: "lost topo lock", - ts: memorytopo.NewServer("zone1"), + name: "lost topo lock", + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -563,14 +543,11 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Shard: "-", }, }, - keyspace: "testkeyspace", - shard: "-", - opts: EmergencyReparentOptions{}, shouldErr: true, }, { - name: "cannot get reparent candidates", - ts: memorytopo.NewServer("zone1"), + name: "cannot get reparent candidates", + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -633,29 +610,24 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Hostname: "has a zero relay log position", }, }, - keyspace: "testkeyspace", - shard: "-", - opts: EmergencyReparentOptions{}, shouldErr: true, }, { - name: "zero valid reparent candidates", - ts: memorytopo.NewServer("zone1"), - tmc: &testutil.TabletManagerClient{}, + name: "zero valid reparent candidates", + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + tmc: &testutil.TabletManagerClient{}, shards: []*vtctldatapb.Shard{ { Keyspace: "testkeyspace", Name: "-", }, }, - keyspace: "testkeyspace", - shard: "-", - opts: EmergencyReparentOptions{}, shouldErr: true, }, { name: "error waiting for relay logs to apply", - ts: memorytopo.NewServer("zone1"), + // one replica is going to take a minute to apply relay logs + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, time.Millisecond*50, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -736,16 +708,14 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Hostname: "fails to apply relay logs", }, }, - keyspace: "testkeyspace", - shard: "-", - opts: EmergencyReparentOptions{ - WaitReplicasTimeout: time.Millisecond * 50, // one replica is going to take a minute to apply relay logs - }, shouldErr: true, }, { name: "requested primary-elect is not in tablet map", - ts: memorytopo.NewServer("zone1"), + vtctlReparentFunctions: NewVtctlReparentFunctions(&topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 200, + }, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -821,19 +791,15 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Shard: "-", }, }, - keyspace: "testkeyspace", - shard: "-", - opts: EmergencyReparentOptions{ - NewPrimaryAlias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 200, - }, - }, shouldErr: true, }, { name: "requested primary-elect is not winning primary-elect", - ts: memorytopo.NewServer("zone1"), + vtctlReparentFunctions: NewVtctlReparentFunctions(&topodatapb.TabletAlias{ // we're requesting a tablet that's behind in replication + Cell: "zone1", + Uid: 102, + }, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + ts: memorytopo.NewServer("zone1"), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -910,19 +876,18 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Hostname: "not most up-to-date position", }, }, - keyspace: "testkeyspace", - shard: "-", - opts: EmergencyReparentOptions{ - NewPrimaryAlias: &topodatapb.TabletAlias{ // we're requesting a tablet that's behind in replication - Cell: "zone1", - Uid: 102, - }, - }, shouldErr: true, }, { name: "cannot promote new primary", - ts: memorytopo.NewServer("zone1"), + vtctlReparentFunctions: NewVtctlReparentFunctions( // We're explicitly requesting a primary-elect in this test case + // because we don't care about the correctness of the selection + // code (it's covered by other test cases), and it simplifies + // the error mocking. + &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ PromoteReplicaResults: map[string]struct { Result string @@ -1007,18 +972,6 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Hostname: "not most up-to-date position", }, }, - keyspace: "testkeyspace", - shard: "-", - opts: EmergencyReparentOptions{ - // We're explicitly requesting a primary-elect in this test case - // because we don't care about the correctness of the selection - // code (it's covered by other test cases), and it simplifies - // the error mocking. - NewPrimaryAlias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - }, shouldErr: true, }, } @@ -1033,30 +986,24 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { logger := logutil.NewMemoryLogger() ev := &events.Reparent{} - testutil.AddShards(ctx, t, tt.ts, tt.shards...) - testutil.AddTablets(ctx, t, tt.ts, nil, tt.tablets...) + testutil.AddShards(ctx, t, tt.vtctlReparentFunctions.ts, tt.shards...) + testutil.AddTablets(ctx, t, tt.vtctlReparentFunctions.ts, nil, tt.tablets...) if !tt.unlockTopo { - lctx, unlock, lerr := tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "test lock") - require.NoError(t, lerr, "could not lock %s/%s for testing", tt.keyspace, tt.shard) + lctx, unlock, lerr := tt.vtctlReparentFunctions.ts.LockShard(ctx, tt.vtctlReparentFunctions.keyspace, tt.vtctlReparentFunctions.shard, "test lock") + require.NoError(t, lerr, "could not lock %s/%s for testing", tt.vtctlReparentFunctions.keyspace, tt.vtctlReparentFunctions.shard) defer func() { unlock(&lerr) - require.NoError(t, lerr, "could not unlock %s/%s after test", tt.keyspace, tt.shard) + require.NoError(t, lerr, "could not unlock %s/%s after test", tt.vtctlReparentFunctions.keyspace, tt.vtctlReparentFunctions.shard) }() ctx = lctx // make the reparentShardLocked call use the lock ctx } erp := NewEmergencyReparenter2(tt.tmc, logger) - vtctlReparentFunctions := NewVtctlReparentFunctions(tt.ts) - vtctlReparentFunctions.keyspace = tt.keyspace - vtctlReparentFunctions.shard = tt.shard - vtctlReparentFunctions.IgnoreReplicas = tt.opts.IgnoreReplicas - vtctlReparentFunctions.WaitReplicasTimeout = tt.opts.WaitReplicasTimeout - vtctlReparentFunctions.NewPrimaryAlias = tt.opts.NewPrimaryAlias - err := erp.reparentShardLocked(ctx, ev, vtctlReparentFunctions) + err := erp.reparentShardLocked(ctx, ev, tt.vtctlReparentFunctions) if tt.shouldErr { assert.Error(t, err) return @@ -1071,21 +1018,18 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { t.Parallel() tests := []struct { - name string - ts *topo.Server - tmc *testutil.TabletManagerClient - unlockTopo bool - keyspace string - shard string - newPrimaryTabletAlias string - tabletMap map[string]*topo.TabletInfo - statusMap map[string]*replicationdatapb.StopReplicationStatus - opts EmergencyReparentOptions - shouldErr bool + name string + vtctlReparentFunctions *VtctlReparentFunctions + tmc *testutil.TabletManagerClient + unlockTopo bool + newPrimaryTabletAlias string + tabletMap map[string]*topo.TabletInfo + statusMap map[string]*replicationdatapb.StopReplicationStatus + shouldErr bool }{ { - name: "success", - ts: memorytopo.NewServer("zone1"), + name: "success", + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, sets.NewString("zone1-0000000404"), 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1104,8 +1048,6 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { "zone1-0000000404": assert.AnError, // okay, because we're ignoring it. }, }, - keyspace: "testkeyspace", - shard: "-", newPrimaryTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { @@ -1158,18 +1100,13 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, }, - opts: EmergencyReparentOptions{ - IgnoreReplicas: sets.NewString("zone1-0000000404"), - }, shouldErr: false, }, { - name: "primary not in tablet map", - ts: memorytopo.NewServer("zone1"), - tmc: &testutil.TabletManagerClient{}, - keyspace: "testkeyspace", - shard: "-", - newPrimaryTabletAlias: "zone2-0000000200", + name: "primary not in tablet map", + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + tmc: &testutil.TabletManagerClient{}, + newPrimaryTabletAlias: "zone2-0000000200", tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": {}, "zone1-0000000101": {}, @@ -1178,8 +1115,8 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { shouldErr: true, }, { - name: "PromoteReplica error", - ts: memorytopo.NewServer("zone1"), + name: "PromoteReplica error", + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ PromoteReplicaResults: map[string]struct { Result string @@ -1190,8 +1127,6 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, }, - keyspace: "testkeyspace", - shard: "-", newPrimaryTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { @@ -1215,8 +1150,8 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { shouldErr: true, }, { - name: "lost topology lock", - ts: memorytopo.NewServer("zone1"), + name: "lost topology lock", + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ PromoteReplicaResults: map[string]struct { Result string @@ -1228,8 +1163,6 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, unlockTopo: true, - keyspace: "testkeyspace", - shard: "-", newPrimaryTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { @@ -1253,8 +1186,8 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { shouldErr: true, }, { - name: "cannot repopulate reparent journal on new primary", - ts: memorytopo.NewServer("zone1"), + name: "cannot repopulate reparent journal on new primary", + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": assert.AnError, @@ -1268,8 +1201,6 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, }, - keyspace: "testkeyspace", - shard: "-", newPrimaryTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { @@ -1293,8 +1224,8 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { shouldErr: true, }, { - name: "all replicas failing to SetMaster does fail the promotion", - ts: memorytopo.NewServer("zone1"), + name: "all replicas failing to SetMaster does fail the promotion", + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1313,8 +1244,6 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { "zone1-0000000102": assert.AnError, }, }, - keyspace: "testkeyspace", - shard: "-", newPrimaryTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { @@ -1346,8 +1275,8 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { shouldErr: true, }, { - name: "all replicas slow to SetMaster does fail the promotion", - ts: memorytopo.NewServer("zone1"), + name: "all replicas slow to SetMaster does fail the promotion", + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, time.Millisecond*10, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1370,8 +1299,6 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { "zone1-0000000102": nil, }, }, - keyspace: "testkeyspace", - shard: "-", newPrimaryTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { @@ -1400,14 +1327,11 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - opts: EmergencyReparentOptions{ - WaitReplicasTimeout: time.Millisecond * 10, - }, shouldErr: true, }, { - name: "one replica failing to SetMaster does not fail the promotion", - ts: memorytopo.NewServer("zone1"), + name: "one replica failing to SetMaster does not fail the promotion", + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1425,8 +1349,6 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { "zone1-0000000102": assert.AnError, }, }, - keyspace: "testkeyspace", - shard: "-", newPrimaryTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { @@ -1469,9 +1391,9 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { logger := logutil.NewMemoryLogger() ev := &events.Reparent{} - testutil.AddShards(ctx, t, tt.ts, &vtctldatapb.Shard{ - Keyspace: tt.keyspace, - Name: tt.shard, + testutil.AddShards(ctx, t, tt.vtctlReparentFunctions.ts, &vtctldatapb.Shard{ + Keyspace: tt.vtctlReparentFunctions.keyspace, + Name: tt.vtctlReparentFunctions.shard, }) if !tt.unlockTopo { @@ -1480,26 +1402,21 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { lerr error ) - ctx, unlock, lerr = tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "test lock") - require.NoError(t, lerr, "could not lock %s/%s for test", tt.keyspace, tt.shard) + ctx, unlock, lerr = tt.vtctlReparentFunctions.ts.LockShard(ctx, tt.vtctlReparentFunctions.keyspace, tt.vtctlReparentFunctions.shard, "test lock") + require.NoError(t, lerr, "could not lock %s/%s for test", tt.vtctlReparentFunctions.keyspace, tt.vtctlReparentFunctions.shard) defer func() { unlock(&lerr) - require.NoError(t, lerr, "could not unlock %s/%s after test", tt.keyspace, tt.shard) + require.NoError(t, lerr, "could not unlock %s/%s after test", tt.vtctlReparentFunctions.keyspace, tt.vtctlReparentFunctions.shard) }() } var err error - vtctlReparentFunctions := NewVtctlReparentFunctions(tt.ts) - vtctlReparentFunctions.keyspace = tt.keyspace - vtctlReparentFunctions.shard = tt.shard - vtctlReparentFunctions.winningPrimaryTabletAliasStr = tt.newPrimaryTabletAlias - vtctlReparentFunctions.tabletMap = tt.tabletMap - vtctlReparentFunctions.statusMap = tt.statusMap - vtctlReparentFunctions.IgnoreReplicas = tt.opts.IgnoreReplicas - vtctlReparentFunctions.WaitReplicasTimeout = tt.opts.WaitReplicasTimeout + tt.vtctlReparentFunctions.winningPrimaryTabletAliasStr = tt.newPrimaryTabletAlias + tt.vtctlReparentFunctions.tabletMap = tt.tabletMap + tt.vtctlReparentFunctions.statusMap = tt.statusMap - err = vtctlReparentFunctions.promoteNewPrimary(ctx, ev, logger, tt.tmc) + err = tt.vtctlReparentFunctions.promoteNewPrimary(ctx, ev, logger, tt.tmc) if tt.shouldErr { assert.Error(t, err) return @@ -1515,9 +1432,7 @@ func TestEmergencyReparenter_waitForAllRelayLogsToApply(t *testing.T) { ctx := context.Background() logger := logutil.NewMemoryLogger() - opts := EmergencyReparentOptions{ - WaitReplicasTimeout: time.Millisecond * 50, - } + waitReplicasTimeout := 50 * time.Millisecond tests := []struct { name string tmc *testutil.TabletManagerClient @@ -1746,10 +1661,9 @@ func TestEmergencyReparenter_waitForAllRelayLogsToApply(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - vtctlReparentFunctions := NewVtctlReparentFunctions(nil) + vtctlReparentFunctions := NewVtctlReparentFunctions(nil, nil, waitReplicasTimeout, "", "", nil) vtctlReparentFunctions.tabletMap = tt.tabletMap vtctlReparentFunctions.statusMap = tt.statusMap - vtctlReparentFunctions.WaitReplicasTimeout = opts.WaitReplicasTimeout err := vtctlReparentFunctions.waitForAllRelayLogsToApply(ctx, logger, tt.tmc, tt.candidates) if tt.shouldErr { diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index a752715083d..dbdb3e1c784 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -84,9 +84,15 @@ var ( _ ReparentFunctions = (*VtctlReparentFunctions)(nil) ) -func NewVtctlReparentFunctions(ts *topo.Server) *VtctlReparentFunctions { +// NewVtctlReparentFunctions creates a new VtctlReparentFunctions which is used in ERS ans PRS +func NewVtctlReparentFunctions(newPrimaryAlias *topodatapb.TabletAlias, ignoreReplicas sets.String, waitReplicasTimeout time.Duration, keyspace string, shard string, ts *topo.Server) *VtctlReparentFunctions { return &VtctlReparentFunctions{ - ts: ts, + NewPrimaryAlias: newPrimaryAlias, + IgnoreReplicas: ignoreReplicas, + WaitReplicasTimeout: waitReplicasTimeout, + keyspace: keyspace, + shard: shard, + ts: ts, } } diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index 2f8d465c0c4..833f1183bbf 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -146,16 +146,15 @@ func (wr *Wrangler) PlannedReparentShard(ctx context.Context, keyspace, shard st // EmergencyReparentShard will make the provided tablet the primary for // the shard, when the old primary is completely unreachable. func (wr *Wrangler) EmergencyReparentShard(ctx context.Context, keyspace, shard string, primaryElectTabletAlias *topodatapb.TabletAlias, waitReplicasTimeout time.Duration, ignoredTablets sets.String) (err error) { - _, err = reparentutil.NewEmergencyReparenter(wr.ts, wr.tmc, wr.logger).ReparentShard( + _, err = reparentutil.NewEmergencyReparenter2(wr.tmc, wr.logger).ReparentShard( ctx, - keyspace, - shard, - reparentutil.EmergencyReparentOptions{ - NewPrimaryAlias: primaryElectTabletAlias, - WaitReplicasTimeout: waitReplicasTimeout, - IgnoreReplicas: ignoredTablets, - }, - ) + reparentutil.NewVtctlReparentFunctions(primaryElectTabletAlias, + ignoredTablets, + waitReplicasTimeout, + keyspace, + shard, + wr.ts, + )) return err } From 873ca33c5d0382840320d991fc94033e36e8bd8c Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 16 Aug 2021 21:17:48 +0530 Subject: [PATCH 019/176] rename the new ERS Signed-off-by: GuptaManan100 --- go/vt/orchestrator/logic/topology_recovery.go | 2 +- go/vt/vtctl/grpcvtctldserver/server.go | 2 +- ...y_reparenter_2.go => emergency_reparenter.go} | 16 ++++++++-------- .../reparentutil/emergency_reparenter_test.go | 4 ++-- go/vt/wrangler/reparent.go | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) rename go/vt/vtctl/reparentutil/{emergency_reparenter_2.go => emergency_reparenter.go} (83%) diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index a5a7cbb3e46..ba2fe319d24 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -624,7 +624,7 @@ func checkAndRecoverDeadMaster(analysisEntry inst.ReplicationAnalysis, candidate log.Infof("Analysis: %v, deadmaster %+v", analysisEntry.Analysis, analysisEntry.AnalyzedInstanceKey) reparentFunctions := NewVtorcReparentFunctions(analysisEntry, candidateInstanceKey, skipProcesses, topologyRecovery) - _, err = reparentutil.NewEmergencyReparenter2(tmclient.NewTabletManagerClient(), nil).ReparentShard(context.Background(), reparentFunctions) + _, err = reparentutil.NewEmergencyReparenter(tmclient.NewTabletManagerClient(), nil).ReparentShard(context.Background(), reparentFunctions) return reparentFunctions.recoveryAttempted, topologyRecovery, err } diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index f316658f1f2..6e74dea37e7 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -588,7 +588,7 @@ func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldat logstream = append(logstream, e) }) - ev, err := reparentutil.NewEmergencyReparenter2(s.tmc, logger).ReparentShard(ctx, + ev, err := reparentutil.NewEmergencyReparenter(s.tmc, logger).ReparentShard(ctx, reparentutil.NewVtctlReparentFunctions(req.NewPrimary, sets.NewString(ignoreReplicaAliases...), waitReplicasTimeout, diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_2.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go similarity index 83% rename from go/vt/vtctl/reparentutil/emergency_reparenter_2.go rename to go/vt/vtctl/reparentutil/emergency_reparenter.go index 316035240da..5f613cb5134 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_2.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -34,19 +34,19 @@ import ( "vitess.io/vitess/go/vt/vttablet/tmclient" ) -// EmergencyReparenter2 performs EmergencyReparentShard operations. -type EmergencyReparenter2 struct { +// EmergencyReparenter performs EmergencyReparentShard operations. +type EmergencyReparenter struct { tmc tmclient.TabletManagerClient logger logutil.Logger } -// NewEmergencyReparenter2 returns a new EmergencyReparenter object, ready to -// perform EmergencyReparentShard operations using the given topo.Server, +// NewEmergencyReparenter returns a new EmergencyReparenter object, ready to +// perform EmergencyReparentShard operations using the given // TabletManagerClient, and logger. // // Providing a nil logger instance is allowed. -func NewEmergencyReparenter2(tmc tmclient.TabletManagerClient, logger logutil.Logger) *EmergencyReparenter2 { - erp := EmergencyReparenter2{ +func NewEmergencyReparenter(tmc tmclient.TabletManagerClient, logger logutil.Logger) *EmergencyReparenter { + erp := EmergencyReparenter{ tmc: tmc, logger: logger, } @@ -62,7 +62,7 @@ func NewEmergencyReparenter2(tmc tmclient.TabletManagerClient, logger logutil.Lo // ReparentShard performs the EmergencyReparentShard operation on the given // keyspace and shard. -func (erp *EmergencyReparenter2) ReparentShard(ctx context.Context, reparentFunctions ReparentFunctions) (*events.Reparent, error) { +func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, reparentFunctions ReparentFunctions) (*events.Reparent, error) { ctx, unlock, err := reparentFunctions.LockShard(ctx) if err != nil { return nil, err @@ -84,7 +84,7 @@ func (erp *EmergencyReparenter2) ReparentShard(ctx context.Context, reparentFunc return ev, err } -func (erp *EmergencyReparenter2) reparentShardLocked(ctx context.Context, ev *events.Reparent, reparentFunctions ReparentFunctions) error { +func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *events.Reparent, reparentFunctions ReparentFunctions) error { if reparentFunctions.CheckIfFixed() { return nil diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 7b8ab195018..03d9499b8c0 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -60,7 +60,7 @@ func TestNewEmergencyReparenter(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - er := NewEmergencyReparenter2(nil, tt.logger) + er := NewEmergencyReparenter(nil, tt.logger) assert.NotNil(t, er.logger, "NewEmergencyReparenter should never result in a nil logger instance on the EmergencyReparenter") }) } @@ -1001,7 +1001,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { ctx = lctx // make the reparentShardLocked call use the lock ctx } - erp := NewEmergencyReparenter2(tt.tmc, logger) + erp := NewEmergencyReparenter(tt.tmc, logger) err := erp.reparentShardLocked(ctx, ev, tt.vtctlReparentFunctions) if tt.shouldErr { diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index 833f1183bbf..8ceb0c19cba 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -146,7 +146,7 @@ func (wr *Wrangler) PlannedReparentShard(ctx context.Context, keyspace, shard st // EmergencyReparentShard will make the provided tablet the primary for // the shard, when the old primary is completely unreachable. func (wr *Wrangler) EmergencyReparentShard(ctx context.Context, keyspace, shard string, primaryElectTabletAlias *topodatapb.TabletAlias, waitReplicasTimeout time.Duration, ignoredTablets sets.String) (err error) { - _, err = reparentutil.NewEmergencyReparenter2(wr.tmc, wr.logger).ReparentShard( + _, err = reparentutil.NewEmergencyReparenter(wr.tmc, wr.logger).ReparentShard( ctx, reparentutil.NewVtctlReparentFunctions(primaryElectTabletAlias, ignoredTablets, From 266aba2b8b4f63419b9f4503ce4764dee87dbdbd Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Tue, 17 Aug 2021 14:54:04 +0530 Subject: [PATCH 020/176] added a failing test for ERS which promotes rdonly Signed-off-by: GuptaManan100 --- go/test/endtoend/reparent/reparent_test.go | 29 +++++++++++++++++++++- go/test/endtoend/reparent/utils_test.go | 14 ++++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/go/test/endtoend/reparent/reparent_test.go b/go/test/endtoend/reparent/reparent_test.go index 736d86bc7e1..b3b0ea7359a 100644 --- a/go/test/endtoend/reparent/reparent_test.go +++ b/go/test/endtoend/reparent/reparent_test.go @@ -139,7 +139,7 @@ func TestReparentIgnoreReplicas(t *testing.T) { require.NotNil(t, err, out) // Now let's run it again, but set the command to ignore the unreachable replica. - out, err = ersIgnoreTablet(t, nil, "30s", tab3) + out, err = ersIgnoreTablet(t, nil, "30s", []*cluster.Vttablet{tab3}) require.Nil(t, err, out) // We'll bring back the replica we took down. @@ -159,6 +159,33 @@ func TestReparentIgnoreReplicas(t *testing.T) { resurrectTablet(ctx, t, tab1) } +// TestERSPromoteRdonly tests that we never end up promoting a rdonly instance as the primary +func TestERSPromoteRdonly(t *testing.T) { + defer cluster.PanicHandler(t) + setupReparentCluster(t) + defer teardownCluster() + var err error + + err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", tab2.Alias, "rdonly") + require.NoError(t, err) + + err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", tab3.Alias, "rdonly") + require.NoError(t, err) + + confirmReplication(t, tab1, []*cluster.Vttablet{tab2, tab3, tab4}) + + // Make the current primary agent and database unavailable. + stopTablet(t, tab1, true) + + // We expect this one to fail because we have ignored all the replicas and have only the rdonly's which should not be promoted + out, err := ersIgnoreTablet(t, nil, "30s", []*cluster.Vttablet{tab4}) + require.NotNil(t, err, out) + + out, err = clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetShard", keyspaceShard) + require.NoError(t, err) + require.Contains(t, out, `"uid": 101`, "the primary should still be 101 in the shard info") +} + func TestReparentCrossCell(t *testing.T) { defer cluster.PanicHandler(t) setupReparentCluster(t) diff --git a/go/test/endtoend/reparent/utils_test.go b/go/test/endtoend/reparent/utils_test.go index defc308e3c3..8f9704774c5 100644 --- a/go/test/endtoend/reparent/utils_test.go +++ b/go/test/endtoend/reparent/utils_test.go @@ -249,7 +249,7 @@ func ers(t *testing.T, tab *cluster.Vttablet, timeout string) (string, error) { return ersIgnoreTablet(t, tab, timeout, nil) } -func ersIgnoreTablet(t *testing.T, tab *cluster.Vttablet, timeout string, tabToIgnore *cluster.Vttablet) (string, error) { +func ersIgnoreTablet(t *testing.T, tab *cluster.Vttablet, timeout string, tabletsToIgnore []*cluster.Vttablet) (string, error) { args := []string{"EmergencyReparentShard", "-keyspace_shard", fmt.Sprintf("%s/%s", keyspaceName, shardName)} if tab != nil { args = append(args, "-new_primary", tab.Alias) @@ -257,8 +257,16 @@ func ersIgnoreTablet(t *testing.T, tab *cluster.Vttablet, timeout string, tabToI if timeout != "" { args = append(args, "-wait_replicas_timeout", "30s") } - if tabToIgnore != nil { - args = append(args, "-ignore_replicas", tabToIgnore.Alias) + if len(tabletsToIgnore) != 0 { + tabsString := "" + for _, vttablet := range tabletsToIgnore { + if tabsString == "" { + tabsString = vttablet.Alias + } else { + tabsString = tabsString + "," + vttablet.Alias + } + } + args = append(args, "-ignore_replicas", tabsString) } return clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput(args...) } From e8cc7d9da672b236cfb6997224c0767954354d6e Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Tue, 17 Aug 2021 15:03:25 +0530 Subject: [PATCH 021/176] added a failing test for vtorc which promotes a crossCellReplica Signed-off-by: GuptaManan100 --- .../endtoend/vtorc/primary_failure_test.go | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index d95274da68c..e54b065e0c4 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -20,6 +20,9 @@ import ( "testing" "time" + "vitess.io/vitess/go/json2" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -90,6 +93,41 @@ func TestCrossDataCenterFailure(t *testing.T) { checkPrimaryTablet(t, clusterInstance, replicaInSameCell) } +// Failover should not be cross data centers, according to the configuration file +// In case of no viable candidates, we should error out +func TestCrossDataCenterFailureError(t *testing.T) { + defer cluster.PanicHandler(t) + setupVttabletsAndVtorc(t, 1, 1, nil, "test_config.json") + keyspace := &clusterInstance.Keyspaces[0] + shard0 := &keyspace.Shards[0] + // find primary from topo + curPrimary := shardPrimaryTablet(t, clusterInstance, keyspace, shard0) + assert.NotNil(t, curPrimary, "should have elected a primary") + + crossCellReplica := startVttablet(t, cell2, false) + // newly started tablet does not replicate from anyone yet, we will allow orchestrator to fix this too + checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{crossCellReplica}, 25*time.Second) + + // Make the current primary database unavailable. + err := curPrimary.MysqlctlProcess.Stop() + require.NoError(t, err) + defer func() { + // we remove the tablet from our global list since its mysqlctl process has stopped and cannot be reused for other tests + permanentlyRemoveVttablet(curPrimary) + }() + + // Give vtorc time to repair + time.Sleep(15 * time.Second) + + // we should not have promoted the crossCellReplica + result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", crossCellReplica.Alias) + require.NoError(t, err) + var tabletInfo topodatapb.Tablet + err = json2.Unmarshal([]byte(result), &tabletInfo) + require.NoError(t, err) + require.NotEqual(t, topodatapb.TabletType_PRIMARY, tabletInfo.GetType()) +} + // Failover will sometimes lead to a replica which can no longer replicate. // covers part of the test case master-failover-lost-replicas from orchestrator func TestLostReplicasOnPrimaryFailure(t *testing.T) { From 383d78f546f7cf52832d5ea9077e445e119326fe Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 18 Aug 2021 11:54:25 +0530 Subject: [PATCH 022/176] stop replication via common code Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 13 ++++++----- .../reparentutil/emergency_reparenter.go | 10 +++++++-- go/vt/vtctl/reparentutil/util.go | 22 +++++++------------ 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index f47c3098df6..f24441ab00b 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -21,6 +21,8 @@ import ( "fmt" "time" + "k8s.io/apimachinery/pkg/util/sets" + "vitess.io/vitess/go/vt/orchestrator/attributes" "vitess.io/vitess/go/vt/orchestrator/kv" "vitess.io/vitess/go/vt/vtctl/reparentutil" @@ -94,7 +96,7 @@ func (vtorcReparent *VtOrcReparentFunctions) GetShard() string { func (vtorcReparent *VtOrcReparentFunctions) CheckIfFixed() bool { // Check if someone else fixed the problem. tablet, err := TabletRefresh(vtorcReparent.analysisEntry.AnalyzedInstanceKey) - if err == nil && tablet.Type != topodatapb.TabletType_MASTER { + if err == nil && tablet.Type != topodatapb.TabletType_PRIMARY { // TODO(sougou); use a version that only refreshes the current shard. RefreshTablets() AuditTopologyRecovery(vtorcReparent.topologyRecovery, "another agent seems to have fixed the problem") @@ -119,10 +121,11 @@ func (vtorcReparent *VtOrcReparentFunctions) PreRecoveryProcesses(ctx context.Co return nil } -// StopReplicationAndBuildStatusMaps implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) StopReplicationAndBuildStatusMaps(context.Context, tmclient.TabletManagerClient, *events.Reparent, logutil.Logger) error { - err := TabletDemoteMaster(vtorcReparent.analysisEntry.AnalyzedInstanceKey) - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: TabletDemoteMaster: %v", err)) +func (vtorcReparent *VtOrcReparentFunctions) GetWaitReplicasTimeout() time.Duration { + return time.Duration(config.Config.LockShardTimeoutSeconds) * time.Second +} + +func (vtorcReparent *VtOrcReparentFunctions) GetIgnoreReplicas() sets.String { return nil } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 5f613cb5134..274841300f3 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -104,8 +104,14 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } - if err := reparentFunctions.StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, erp.logger); err != nil { - return err + tabletMap, err := ts.GetTabletMapForShard(ctx, keyspace, shard) + if err != nil { + return vterrors.Wrapf(err, "failed to get tablet map for %v/%v: %v", keyspace, shard, err) + } + + _, _, err = StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, tabletMap, reparentFunctions.GetWaitReplicasTimeout(), reparentFunctions.GetIgnoreReplicas(), erp.logger) + if err != nil { + return vterrors.Wrapf(err, "failed to stop replication and build status maps: %v", err) } if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index dbdb3e1c784..1e7c5219fa2 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -54,7 +54,8 @@ type ( GetShard() string CheckIfFixed() bool PreRecoveryProcesses(context.Context) error - StopReplicationAndBuildStatusMaps(context.Context, tmclient.TabletManagerClient, *events.Reparent, logutil.Logger) error + GetWaitReplicasTimeout() time.Duration + GetIgnoreReplicas() sets.String CheckPrimaryRecoveryType() error FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient) error CheckIfNeedToOverridePrimary() error @@ -125,22 +126,15 @@ func (vtctlReparent *VtctlReparentFunctions) CheckIfFixed() bool { // PreRecoveryProcesses implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) PreRecoveryProcesses(ctx context.Context) error { - var err error - vtctlReparent.tabletMap, err = vtctlReparent.ts.GetTabletMapForShard(ctx, vtctlReparent.keyspace, vtctlReparent.shard) - if err != nil { - return vterrors.Wrapf(err, "failed to get tablet map for %v/%v: %v", vtctlReparent.keyspace, vtctlReparent.shard, err) - } return nil } -// StopReplicationAndBuildStatusMaps implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) StopReplicationAndBuildStatusMaps(ctx context.Context, tmc tmclient.TabletManagerClient, ev *events.Reparent, logger logutil.Logger) error { - var err error - vtctlReparent.statusMap, vtctlReparent.primaryStatusMap, err = StopReplicationAndBuildStatusMaps(ctx, tmc, ev, vtctlReparent.tabletMap, vtctlReparent.WaitReplicasTimeout, vtctlReparent.IgnoreReplicas, logger) - if err != nil { - return vterrors.Wrapf(err, "failed to stop replication and build status maps: %v", err) - } - return nil +func (vtctlReparentFunctions *VtctlReparentFunctions) GetWaitReplicasTimeout() time.Duration { + return vtctlReparentFunctions.WaitReplicasTimeout +} + +func (vtctlReparentFunctions *VtctlReparentFunctions) GetIgnoreReplicas() sets.String { + return vtctlReparentFunctions.IgnoreReplicas } // CheckPrimaryRecoveryType implements the ReparentFunctions interface From 19d3544a109c63a9e510ddad4285a883985ad389 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 18 Aug 2021 17:44:06 +0530 Subject: [PATCH 023/176] use the candidate list generated from the topo server instead of local database in vtorc Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/main_test.go | 2 +- go/vt/orchestrator/inst/instance_topology.go | 32 ++-- .../inst/instance_topology_test.go | 40 ++--- .../orchestrator/logic/reparent_functions.go | 141 +++++++++++++++++- .../reparentutil/emergency_reparenter.go | 23 ++- .../reparentutil/emergency_reparenter_test.go | 6 +- go/vt/vtctl/reparentutil/util.go | 36 ++--- 7 files changed, 212 insertions(+), 68 deletions(-) diff --git a/go/test/endtoend/vtorc/main_test.go b/go/test/endtoend/vtorc/main_test.go index 2b4dd740e8f..c7de3c44fc9 100644 --- a/go/test/endtoend/vtorc/main_test.go +++ b/go/test/endtoend/vtorc/main_test.go @@ -324,7 +324,7 @@ func TestMain(m *testing.M) { // setup cellInfos before creating the cluster cellInfos = append(cellInfos, &cellInfo{ cellName: cell1, - numReplicas: 10, + numReplicas: 11, numRdonly: 1, uidBase: 100, }) diff --git a/go/vt/orchestrator/inst/instance_topology.go b/go/vt/orchestrator/inst/instance_topology.go index 9c45b83b9d4..91869338861 100644 --- a/go/vt/orchestrator/inst/instance_topology.go +++ b/go/vt/orchestrator/inst/instance_topology.go @@ -618,9 +618,9 @@ func MoveBelowGTID(instanceKey, otherKey *InstanceKey) (*Instance, error) { return moveInstanceBelowViaGTID(instance, other) } -// moveReplicasViaGTID moves a list of replicas under another instance via GTID, returning those replicas +// MoveReplicasViaGTID moves a list of replicas under another instance via GTID, returning those replicas // that could not be moved (do not use GTID or had GTID errors) -func moveReplicasViaGTID(replicas [](*Instance), other *Instance, postponedFunctionsContainer *PostponedFunctionsContainer) (movedReplicas [](*Instance), unmovedReplicas [](*Instance), err error, errs []error) { +func MoveReplicasViaGTID(replicas [](*Instance), other *Instance, postponedFunctionsContainer *PostponedFunctionsContainer) (movedReplicas [](*Instance), unmovedReplicas [](*Instance), err error, errs []error) { replicas = RemoveNilInstances(replicas) replicas = RemoveInstance(replicas, &other.Key) if len(replicas) == 0 { @@ -628,7 +628,7 @@ func moveReplicasViaGTID(replicas [](*Instance), other *Instance, postponedFunct return movedReplicas, unmovedReplicas, nil, errs } - log.Infof("moveReplicasViaGTID: Will move %+v replicas below %+v via GTID, max concurrency: %v", + log.Infof("MoveReplicasViaGTID: Will move %+v replicas below %+v via GTID, max concurrency: %v", len(replicas), other.Key, config.Config.MaxConcurrentReplicaOperations) @@ -679,7 +679,7 @@ func moveReplicasViaGTID(replicas [](*Instance), other *Instance, postponedFunct if len(errs) == len(replicas) { // All returned with error - return movedReplicas, unmovedReplicas, fmt.Errorf("moveReplicasViaGTID: Error on all %+v operations", len(errs)), errs + return movedReplicas, unmovedReplicas, fmt.Errorf("MoveReplicasViaGTID: Error on all %+v operations", len(errs)), errs } AuditOperation("move-replicas-gtid", &other.Key, fmt.Sprintf("moved %d/%d replicas below %+v via GTID", len(movedReplicas), len(replicas), other.Key)) @@ -700,7 +700,7 @@ func MoveReplicasGTID(masterKey *InstanceKey, belowKey *InstanceKey, pattern str return movedReplicas, unmovedReplicas, err, errs } replicas = filterInstancesByPattern(replicas, pattern) - movedReplicas, unmovedReplicas, err, errs = moveReplicasViaGTID(replicas, belowInstance, nil) + movedReplicas, unmovedReplicas, err, errs = MoveReplicasViaGTID(replicas, belowInstance, nil) if err != nil { log.Errore(err) } @@ -1504,13 +1504,13 @@ Cleanup: } // sortInstances shuffles given list of instances according to some logic -func sortInstancesDataCenterHint(instances [](*Instance), dataCenterHint string) { +func SortInstancesDataCenterHint(instances [](*Instance), dataCenterHint string) { sort.Sort(sort.Reverse(NewInstancesSorterByExec(instances, dataCenterHint))) } // sortInstances shuffles given list of instances according to some logic func sortInstances(instances [](*Instance)) { - sortInstancesDataCenterHint(instances, "") + SortInstancesDataCenterHint(instances, "") } // getReplicasForSorting returns a list of replicas of a given primary potentially for candidate choosing @@ -1538,7 +1538,7 @@ func sortedReplicasDataCenterHint(replicas [](*Instance), stopReplicationMethod replicas = StopReplicas(replicas, stopReplicationMethod, time.Duration(config.Config.InstanceBulkOperationsWaitTimeoutSeconds)*time.Second) replicas = RemoveNilInstances(replicas) - sortInstancesDataCenterHint(replicas, dataCenterHint) + SortInstancesDataCenterHint(replicas, dataCenterHint) for _, replica := range replicas { log.Debugf("- sorted replica: %+v %+v", replica.Key, replica.ExecBinlogCoordinates) } @@ -1659,10 +1659,10 @@ func getPriorityBinlogFormatForCandidate(replicas [](*Instance)) (priorityBinlog return sorted.First(), nil } -// chooseCandidateReplica -func chooseCandidateReplica(replicas [](*Instance)) (candidateReplica *Instance, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas [](*Instance), err error) { +// ChooseCandidateReplica +func ChooseCandidateReplica(replicas [](*Instance)) (candidateReplica *Instance, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas [](*Instance), err error) { if len(replicas) == 0 { - return candidateReplica, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, fmt.Errorf("No replicas found given in chooseCandidateReplica") + return candidateReplica, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, fmt.Errorf("No replicas found given in ChooseCandidateReplica") } priorityMajorVersion, _ := getPriorityMajorVersionForCandidate(replicas) priorityBinlogFormat, _ := getPriorityBinlogFormatForCandidate(replicas) @@ -1692,7 +1692,7 @@ func chooseCandidateReplica(replicas [](*Instance)) (candidateReplica *Instance, if candidateReplica != nil { replicas = RemoveInstance(replicas, &candidateReplica.Key) } - return candidateReplica, replicas, equalReplicas, laterReplicas, cannotReplicateReplicas, fmt.Errorf("chooseCandidateReplica: no candidate replica found") + return candidateReplica, replicas, equalReplicas, laterReplicas, cannotReplicateReplicas, fmt.Errorf("ChooseCandidateReplica: no candidate replica found") } replicas = RemoveInstance(replicas, &candidateReplica.Key) for _, replica := range replicas { @@ -1701,7 +1701,7 @@ func chooseCandidateReplica(replicas [](*Instance)) (candidateReplica *Instance, // lost due to inability to replicate cannotReplicateReplicas = append(cannotReplicateReplicas, replica) if err != nil { - log.Errorf("chooseCandidateReplica(): error checking CanReplicateFrom(). replica: %v; error: %v", replica.Key, err) + log.Errorf("ChooseCandidateReplica(): error checking CanReplicateFrom(). replica: %v; error: %v", replica.Key, err) } } else if replica.ExecBinlogCoordinates.SmallerThan(&candidateReplica.ExecBinlogCoordinates) { laterReplicas = append(laterReplicas, replica) @@ -1742,7 +1742,7 @@ func GetCandidateReplica(masterKey *InstanceKey, forRematchPurposes bool) (*Inst if len(replicas) == 0 { return candidateReplica, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, fmt.Errorf("No replicas found for %+v", *masterKey) } - candidateReplica, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err = chooseCandidateReplica(replicas) + candidateReplica, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err = ChooseCandidateReplica(replicas) if err != nil { return candidateReplica, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err } @@ -1845,7 +1845,7 @@ func RegroupReplicasGTID( moveGTIDFunc := func() error { log.Debugf("RegroupReplicasGTID: working on %d replicas", len(replicasToMove)) - movedReplicas, unmovedReplicas, err, _ = moveReplicasViaGTID(replicasToMove, candidateReplica, postponedFunctionsContainer) + movedReplicas, unmovedReplicas, err, _ = MoveReplicasViaGTID(replicasToMove, candidateReplica, postponedFunctionsContainer) unmovedReplicas = append(unmovedReplicas, aheadReplicas...) return log.Errore(err) } @@ -2090,7 +2090,7 @@ func relocateReplicasInternal(replicas [](*Instance), instance, other *Instance) } // GTID { - movedReplicas, unmovedReplicas, err, errs := moveReplicasViaGTID(replicas, other, nil) + movedReplicas, unmovedReplicas, err, errs := MoveReplicasViaGTID(replicas, other, nil) if len(movedReplicas) == len(replicas) { // Moved (or tried moving) everything via GTID diff --git a/go/vt/orchestrator/inst/instance_topology_test.go b/go/vt/orchestrator/inst/instance_topology_test.go index ab2f2f04557..daafec6c9a9 100644 --- a/go/vt/orchestrator/inst/instance_topology_test.go +++ b/go/vt/orchestrator/inst/instance_topology_test.go @@ -99,7 +99,7 @@ func TestSortInstancesDataCenterHint(t *testing.T) { instance.DataCenter = "somedc" } instancesMap[i810Key.StringCode()].DataCenter = "localdc" - sortInstancesDataCenterHint(instances, "localdc") + SortInstancesDataCenterHint(instances, "localdc") test.S(t).ExpectEquals(instances[0].Key, i810Key) } @@ -277,7 +277,7 @@ func TestChooseCandidateReplicaNoCandidateReplica(t *testing.T) { instance.LogBinEnabled = true instance.LogReplicationUpdatesEnabled = false } - _, _, _, _, _, err := chooseCandidateReplica(instances) + _, _, _, _, _, err := ChooseCandidateReplica(instances) test.S(t).ExpectNotNil(err) } @@ -285,7 +285,7 @@ func TestChooseCandidateReplica(t *testing.T) { instances, _ := generateTestInstances() applyGeneralGoodToGoReplicationParams(instances) instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i830Key) test.S(t).ExpectEquals(len(aheadReplicas), 0) @@ -300,7 +300,7 @@ func TestChooseCandidateReplica2(t *testing.T) { instancesMap[i830Key.StringCode()].LogReplicationUpdatesEnabled = false instancesMap[i820Key.StringCode()].LogBinEnabled = false instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i810Key) test.S(t).ExpectEquals(len(aheadReplicas), 2) @@ -318,7 +318,7 @@ func TestChooseCandidateReplicaSameCoordinatesDifferentVersions(t *testing.T) { instancesMap[i810Key.StringCode()].Version = "5.5.1" instancesMap[i720Key.StringCode()].Version = "5.7.8" instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i810Key) test.S(t).ExpectEquals(len(aheadReplicas), 0) @@ -332,7 +332,7 @@ func TestChooseCandidateReplicaPriorityVersionNoLoss(t *testing.T) { applyGeneralGoodToGoReplicationParams(instances) instancesMap[i830Key.StringCode()].Version = "5.5.1" instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i830Key) test.S(t).ExpectEquals(len(aheadReplicas), 0) @@ -346,7 +346,7 @@ func TestChooseCandidateReplicaPriorityVersionLosesOne(t *testing.T) { applyGeneralGoodToGoReplicationParams(instances) instancesMap[i830Key.StringCode()].Version = "5.7.8" instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i820Key) test.S(t).ExpectEquals(len(aheadReplicas), 1) @@ -361,7 +361,7 @@ func TestChooseCandidateReplicaPriorityVersionLosesTwo(t *testing.T) { instancesMap[i830Key.StringCode()].Version = "5.7.8" instancesMap[i820Key.StringCode()].Version = "5.7.18" instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i810Key) test.S(t).ExpectEquals(len(aheadReplicas), 2) @@ -378,7 +378,7 @@ func TestChooseCandidateReplicaPriorityVersionHigherVersionOverrides(t *testing. instancesMap[i810Key.StringCode()].Version = "5.7.5" instancesMap[i730Key.StringCode()].Version = "5.7.30" instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i830Key) test.S(t).ExpectEquals(len(aheadReplicas), 0) @@ -396,7 +396,7 @@ func TestChooseCandidateReplicaLosesOneDueToBinlogFormat(t *testing.T) { instancesMap[i730Key.StringCode()].Binlog_format = "STATEMENT" instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i830Key) test.S(t).ExpectEquals(len(aheadReplicas), 0) @@ -413,7 +413,7 @@ func TestChooseCandidateReplicaPriorityBinlogFormatNoLoss(t *testing.T) { } instancesMap[i830Key.StringCode()].Binlog_format = "STATEMENT" instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i830Key) test.S(t).ExpectEquals(len(aheadReplicas), 0) @@ -427,7 +427,7 @@ func TestChooseCandidateReplicaPriorityBinlogFormatLosesOne(t *testing.T) { applyGeneralGoodToGoReplicationParams(instances) instancesMap[i830Key.StringCode()].Binlog_format = "ROW" instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i820Key) test.S(t).ExpectEquals(len(aheadReplicas), 1) @@ -442,7 +442,7 @@ func TestChooseCandidateReplicaPriorityBinlogFormatLosesTwo(t *testing.T) { instancesMap[i830Key.StringCode()].Binlog_format = "ROW" instancesMap[i820Key.StringCode()].Binlog_format = "ROW" instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i810Key) test.S(t).ExpectEquals(len(aheadReplicas), 2) @@ -459,7 +459,7 @@ func TestChooseCandidateReplicaPriorityBinlogFormatRowOverrides(t *testing.T) { instancesMap[i810Key.StringCode()].Binlog_format = "ROW" instancesMap[i730Key.StringCode()].Binlog_format = "ROW" instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i830Key) test.S(t).ExpectEquals(len(aheadReplicas), 0) @@ -473,7 +473,7 @@ func TestChooseCandidateReplicaMustNotPromoteRule(t *testing.T) { applyGeneralGoodToGoReplicationParams(instances) instancesMap[i830Key.StringCode()].PromotionRule = MustNotPromoteRule instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i820Key) test.S(t).ExpectEquals(len(aheadReplicas), 1) @@ -488,7 +488,7 @@ func TestChooseCandidateReplicaPreferNotPromoteRule(t *testing.T) { instancesMap[i830Key.StringCode()].PromotionRule = MustNotPromoteRule instancesMap[i820Key.StringCode()].PromotionRule = PreferNotPromoteRule instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i820Key) test.S(t).ExpectEquals(len(aheadReplicas), 1) @@ -505,7 +505,7 @@ func TestChooseCandidateReplicaPreferNotPromoteRule2(t *testing.T) { } instancesMap[i830Key.StringCode()].PromotionRule = MustNotPromoteRule instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i820Key) test.S(t).ExpectEquals(len(aheadReplicas), 1) @@ -523,7 +523,7 @@ func TestChooseCandidateReplicaPromoteRuleOrdering(t *testing.T) { } instancesMap[i830Key.StringCode()].PromotionRule = PreferPromoteRule instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i830Key) test.S(t).ExpectEquals(len(aheadReplicas), 0) @@ -541,7 +541,7 @@ func TestChooseCandidateReplicaPromoteRuleOrdering2(t *testing.T) { } instancesMap[i820Key.StringCode()].PromotionRule = MustPromoteRule instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i820Key) test.S(t).ExpectEquals(len(aheadReplicas), 0) @@ -561,7 +561,7 @@ func TestChooseCandidateReplicaPromoteRuleOrdering3(t *testing.T) { instancesMap[i810Key.StringCode()].PromotionRule = PreferPromoteRule instancesMap[i830Key.StringCode()].PromotionRule = PreferNotPromoteRule instances = sortedReplicas(instances, NoStopReplication) - candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := chooseCandidateReplica(instances) + candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) test.S(t).ExpectEquals(candidate.Key, i730Key) test.S(t).ExpectEquals(len(aheadReplicas), 0) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index f24441ab00b..c6e8f4b2067 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -21,6 +21,13 @@ import ( "fmt" "time" + replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" + + "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" + + "vitess.io/vitess/go/mysql" + "k8s.io/apimachinery/pkg/util/sets" "vitess.io/vitess/go/vt/orchestrator/attributes" @@ -140,7 +147,7 @@ func (vtorcReparent *VtOrcReparentFunctions) CheckPrimaryRecoveryType() error { } // FindPrimaryCandidates implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient) error { +func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) error { postponedAll := false promotedReplicaIsIdeal := func(promoted *inst.Instance, hasBestPromotionRule bool) bool { if promoted == nil { @@ -163,9 +170,8 @@ func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.C } AuditTopologyRecovery(vtorcReparent.topologyRecovery, "RecoverDeadMaster: regrouping replicas via GTID") - lostReplicas, _, cannotReplicateReplicas, promotedReplica, err := inst.RegroupReplicasGTID(&vtorcReparent.analysisEntry.AnalyzedInstanceKey, true, nil, &vtorcReparent.topologyRecovery.PostponedFunctionsContainer, promotedReplicaIsIdeal) + lostReplicas, promotedReplica, err := ChooseCandidate(tmc, &vtorcReparent.analysisEntry.AnalyzedInstanceKey, &vtorcReparent.topologyRecovery.PostponedFunctionsContainer, promotedReplicaIsIdeal, validCandidates, tabletMap) vtorcReparent.topologyRecovery.AddError(err) - lostReplicas = append(lostReplicas, cannotReplicateReplicas...) for _, replica := range lostReplicas { AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: - lost replica: %+v", replica.Key)) } @@ -206,6 +212,132 @@ func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.C return nil } +// ChooseCandidate will choose a candidate replica of a given instance, and take its siblings using GTID +func ChooseCandidate( + tmc tmclient.TabletManagerClient, + masterKey *inst.InstanceKey, + postponedFunctionsContainer *inst.PostponedFunctionsContainer, + postponeAllMatchOperations func(*inst.Instance, bool) bool, + validCandidates map[string]mysql.Position, + tabletMap map[string]*topo.TabletInfo, +) ( + lostReplicas [](*inst.Instance), + candidateReplica *inst.Instance, + err error, +) { + var emptyReplicas [](*inst.Instance) + var unmovedReplicas [](*inst.Instance) + var movedReplicas [](*inst.Instance) + + dataCenterHint := "" + if master, _, _ := inst.ReadInstance(masterKey); master != nil { + dataCenterHint = master.DataCenter + } + + var replicas [](*inst.Instance) + log.Errorf("started Manan's new function") + + for candidate := range validCandidates { + candidateInfo, ok := tabletMap[candidate] + if !ok { + return emptyReplicas, candidateReplica, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) + } + candidateInstance, _, err := inst.ReadInstance(&inst.InstanceKey{ + Hostname: candidateInfo.MysqlHostname, + Port: int(candidateInfo.MysqlPort), + }) + if err != nil { + log.Errorf("%v", err) + return emptyReplicas, candidateReplica, err + } + replicas = append(replicas, candidateInstance) + } + + inst.SortInstancesDataCenterHint(replicas, dataCenterHint) + for _, replica := range replicas { + log.Debugf("- sorted replica: %+v %+v", replica.Key, replica.ExecBinlogCoordinates) + } + + candidateReplica, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := inst.ChooseCandidateReplica(replicas) + if err != nil { + return emptyReplicas, candidateReplica, err + } + if candidateReplica != nil { + mostUpToDateReplica := replicas[0] + if candidateReplica.ExecBinlogCoordinates.SmallerThan(&mostUpToDateReplica.ExecBinlogCoordinates) { + log.Warningf("GetCandidateReplica: chosen replica: %+v is behind most-up-to-date replica: %+v", candidateReplica.Key, mostUpToDateReplica.Key) + } + } + log.Debugf("GetCandidateReplica: candidate: %+v, ahead: %d, equal: %d, late: %d, break: %d", candidateReplica.Key, len(aheadReplicas), len(equalReplicas), len(laterReplicas), len(cannotReplicateReplicas)) + + replicasToMove := append(equalReplicas, laterReplicas...) + hasBestPromotionRule := true + if candidateReplica != nil { + for _, replica := range replicasToMove { + if replica.PromotionRule.BetterThan(candidateReplica.PromotionRule) { + hasBestPromotionRule = false + } + } + } + + // TODO: Use the set replication source functionality instead of the moveReplicasViaGTID + //now := time.Now().UnixNano() + if err := inst.SwitchMaster(candidateReplica.Key, *masterKey); err != nil { + return emptyReplicas, candidateReplica, err + } + + //candidateReplicaTablet, err := inst.ReadTablet(candidateReplica.Key) + //if err != nil { + // return emptyReplicas, candidateReplica, err + //} + + //moveGTIDFunc := func() error { + // log.Debugf("RegroupReplicasGTID: working on %d replicas", len(replicasToMove)) + // + // for _, instance := range replicasToMove { + // tablet, err := inst.ReadTablet(instance.Key) + // if err != nil { + // return err + // } + // + // err = tmc.SetReplicationSource(context.Background(), tablet, candidateReplicaTablet.Alias, now, "", false) + // if err != nil { + // unmovedReplicas = append(unmovedReplicas, instance) + // err = vterrors.Wrapf(err, "tablet %v SetMaster failed: %v", tablet.Alias, err) + // return err + // } + // movedReplicas = append(movedReplicas, instance) + // } + // + // unmovedReplicas = append(unmovedReplicas, aheadReplicas...) + // unmovedReplicas = append(unmovedReplicas, cannotReplicateReplicas...) + // return log.Errore(err) + //} + moveGTIDFunc := func() error { + log.Debugf("RegroupReplicasGTID: working on %d replicas", len(replicasToMove)) + + movedReplicas, unmovedReplicas, err, _ = inst.MoveReplicasViaGTID(replicasToMove, candidateReplica, postponedFunctionsContainer) + unmovedReplicas = append(unmovedReplicas, aheadReplicas...) + return log.Errore(err) + } + if postponedFunctionsContainer != nil && postponeAllMatchOperations != nil && postponeAllMatchOperations(candidateReplica, hasBestPromotionRule) { + postponedFunctionsContainer.AddPostponedFunction(moveGTIDFunc, fmt.Sprintf("regroup-replicas-gtid %+v", candidateReplica.Key)) + } else { + err = moveGTIDFunc() + } + if postponedFunctionsContainer != nil && postponeAllMatchOperations != nil && postponeAllMatchOperations(candidateReplica, hasBestPromotionRule) { + postponedFunctionsContainer.AddPostponedFunction(moveGTIDFunc, fmt.Sprintf("regroup-replicas-gtid %+v", candidateReplica.Key)) + } else { + err = moveGTIDFunc() + } + + inst.StartReplication(&candidateReplica.Key) + + log.Debugf("RegroupReplicasGTID: done") + inst.AuditOperation("regroup-replicas-gtid", masterKey, fmt.Sprintf("regrouped replicas of %+v via GTID; promoted %+v", *masterKey, candidateReplica.Key)) + return unmovedReplicas, candidateReplica, err +} + // CheckIfNeedToOverridePrimary implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) CheckIfNeedToOverridePrimary() error { if vtorcReparent.promotedReplica == nil { @@ -345,3 +477,6 @@ func (vtorcReparent *VtOrcReparentFunctions) GetNewPrimary() *topodatapb.Tablet tablet, _ := inst.ReadTablet(vtorcReparent.promotedReplica.Key) return tablet } + +func (vtorcReparent *VtOrcReparentFunctions) SetMaps(tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, primaryStatusMap map[string]*replicationdatapb.PrimaryStatus) { +} diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 274841300f3..288734e5b83 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -19,6 +19,8 @@ package reparentutil import ( "context" + "vitess.io/vitess/go/vt/proto/vtrpc" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" "google.golang.org/protobuf/proto" @@ -104,12 +106,16 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } + if err := reparentFunctions.CheckPrimaryRecoveryType(); err != nil { + return err + } + tabletMap, err := ts.GetTabletMapForShard(ctx, keyspace, shard) if err != nil { return vterrors.Wrapf(err, "failed to get tablet map for %v/%v: %v", keyspace, shard, err) } - _, _, err = StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, tabletMap, reparentFunctions.GetWaitReplicasTimeout(), reparentFunctions.GetIgnoreReplicas(), erp.logger) + statusMap, primaryStatusMap, err := StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, tabletMap, reparentFunctions.GetWaitReplicasTimeout(), reparentFunctions.GetIgnoreReplicas(), erp.logger) if err != nil { return vterrors.Wrapf(err, "failed to stop replication and build status maps: %v", err) } @@ -118,11 +124,22 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } - if err := reparentFunctions.CheckPrimaryRecoveryType(); err != nil { + // TODO: remove this entirely + reparentFunctions.SetMaps(tabletMap, statusMap, primaryStatusMap) + + validCandidates, err := FindValidEmergencyReparentCandidates(statusMap, primaryStatusMap) + if err != nil { + return err + } else if len(validCandidates) == 0 { + return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "no valid candidates for emergency reparent") + } + + // Wait for all candidates to apply relay logs + if err := waitForAllRelayLogsToApply(ctx, erp.logger, erp.tmc, validCandidates, tabletMap, statusMap, reparentFunctions.GetWaitReplicasTimeout()); err != nil { return err } - if err := reparentFunctions.FindPrimaryCandidates(ctx, erp.logger, erp.tmc); err != nil { + if err := reparentFunctions.FindPrimaryCandidates(ctx, erp.logger, erp.tmc, validCandidates, tabletMap); err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 03d9499b8c0..c4e40de6804 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -1661,11 +1661,7 @@ func TestEmergencyReparenter_waitForAllRelayLogsToApply(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - vtctlReparentFunctions := NewVtctlReparentFunctions(nil, nil, waitReplicasTimeout, "", "", nil) - vtctlReparentFunctions.tabletMap = tt.tabletMap - vtctlReparentFunctions.statusMap = tt.statusMap - - err := vtctlReparentFunctions.waitForAllRelayLogsToApply(ctx, logger, tt.tmc, tt.candidates) + err := waitForAllRelayLogsToApply(ctx, logger, tt.tmc, tt.candidates, tt.tabletMap, tt.statusMap, waitReplicasTimeout) if tt.shouldErr { assert.Error(t, err) return diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 1e7c5219fa2..820f73e1d1e 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -57,10 +57,13 @@ type ( GetWaitReplicasTimeout() time.Duration GetIgnoreReplicas() sets.String CheckPrimaryRecoveryType() error - FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient) error + FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) error CheckIfNeedToOverridePrimary() error StartReplication(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) error GetNewPrimary() *topodatapb.Tablet + + // TODO: remove this + SetMaps(map[string]*topo.TabletInfo, map[string]*replicationdatapb.StopReplicationStatus, map[string]*replicationdatapb.PrimaryStatus) } // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions @@ -143,21 +146,8 @@ func (vtctlReparent *VtctlReparentFunctions) CheckPrimaryRecoveryType() error { } // FindPrimaryCandidates implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient) error { - validCandidates, err := FindValidEmergencyReparentCandidates(vtctlReparent.statusMap, vtctlReparent.primaryStatusMap) - if err != nil { - return err - } else if len(validCandidates) == 0 { - return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "no valid candidates for emergency reparent") - } - - // Wait for all candidates to apply relay logs - if err := vtctlReparent.waitForAllRelayLogsToApply(ctx, logger, tmc, validCandidates); err != nil { - return err - } - +func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, m map[string]*topo.TabletInfo) error { // Elect the candidate with the most up-to-date position. - for alias, position := range validCandidates { if vtctlReparent.winningPosition.IsZero() || position.AtLeast(vtctlReparent.winningPosition) { vtctlReparent.winningPosition = position @@ -199,6 +189,12 @@ func (vtctlReparent *VtctlReparentFunctions) GetNewPrimary() *topodatapb.Tablet return vtctlReparent.tabletMap[vtctlReparent.winningPrimaryTabletAliasStr].Tablet } +func (vtctlReparent *VtctlReparentFunctions) SetMaps(tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, primaryStatusMap map[string]*replicationdatapb.PrimaryStatus) { + vtctlReparent.tabletMap = tabletMap + vtctlReparent.statusMap = statusMap + vtctlReparent.primaryStatusMap = primaryStatusMap +} + func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { action := "EmergencyReparentShard" @@ -209,11 +205,11 @@ func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topo return action } -func (vtctlReparent *VtctlReparentFunctions) waitForAllRelayLogsToApply(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position) error { +func waitForAllRelayLogsToApply(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, waitReplicasTimeout time.Duration) error { errCh := make(chan error) defer close(errCh) - groupCtx, groupCancel := context.WithTimeout(ctx, vtctlReparent.WaitReplicasTimeout) + groupCtx, groupCancel := context.WithTimeout(ctx, waitReplicasTimeout) defer groupCancel() waiterCount := 0 @@ -236,7 +232,7 @@ func (vtctlReparent *VtctlReparentFunctions) waitForAllRelayLogsToApply(ctx cont // tablet. In either case, it does not make sense to wait for relay logs // to apply on a tablet that was never applying relay logs in the first // place, so we skip it, and log that we did. - status, ok := vtctlReparent.statusMap[candidate] + status, ok := statusMap[candidate] if !ok { logger.Infof("EmergencyReparent candidate %v not in replica status map; this means it was not running replication (because it was formerly MASTER), so skipping WaitForRelayLogsToApply step for this candidate", candidate) continue @@ -245,7 +241,7 @@ func (vtctlReparent *VtctlReparentFunctions) waitForAllRelayLogsToApply(ctx cont go func(alias string, status *replicationdatapb.StopReplicationStatus) { var err error defer func() { errCh <- err }() - err = WaitForRelayLogsToApply(groupCtx, tmc, vtctlReparent.tabletMap[alias], status) + err = WaitForRelayLogsToApply(groupCtx, tmc, tabletMap[alias], status) }(candidate, status) waiterCount++ @@ -259,7 +255,7 @@ func (vtctlReparent *VtctlReparentFunctions) waitForAllRelayLogsToApply(ctx cont rec := errgroup.Wait(groupCancel, errCh) if len(rec.Errors) != 0 { - return vterrors.Wrapf(rec.Error(), "could not apply all relay logs within the provided WaitReplicasTimeout (%s): %v", vtctlReparent.WaitReplicasTimeout, rec.Error()) + return vterrors.Wrapf(rec.Error(), "could not apply all relay logs within the provided waitReplicasTimeout (%s): %v", waitReplicasTimeout, rec.Error()) } return nil From 5ef4ab67e4035a93a81d62af017fd6674af428bb Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 19 Aug 2021 10:36:41 +0530 Subject: [PATCH 024/176] add function to restrict the candidates based on the type Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 29 +++++++++++++++++++ .../reparentutil/emergency_reparenter.go | 4 +++ go/vt/vtctl/reparentutil/util.go | 21 ++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index c6e8f4b2067..04d7b54e45e 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -146,6 +146,35 @@ func (vtorcReparent *VtOrcReparentFunctions) CheckPrimaryRecoveryType() error { return nil } +// RestrictValidCandidates implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) RestrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { + restrictedValidCandidates := make(map[string]mysql.Position) + + for candidate, position := range validCandidates { + candidateInfo, ok := tabletMap[candidate] + if !ok { + return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) + } + + candidateInstance, _, err := inst.ReadInstance(&inst.InstanceKey{ + Hostname: candidateInfo.MysqlHostname, + Port: int(candidateInfo.MysqlPort), + }) + + if err != nil { + return nil, err + } + + if candidateInstance.PromotionRule == inst.MustNotPromoteRule { + continue + } + + restrictedValidCandidates[candidate] = position + } + + return restrictedValidCandidates, nil +} + // FindPrimaryCandidates implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) error { postponedAll := false diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 288734e5b83..8c6e7b5ede5 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -128,6 +128,10 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve reparentFunctions.SetMaps(tabletMap, statusMap, primaryStatusMap) validCandidates, err := FindValidEmergencyReparentCandidates(statusMap, primaryStatusMap) + if err != nil { + return err + } + validCandidates, err = reparentFunctions.RestrictValidCandidates(validCandidates, tabletMap) if err != nil { return err } else if len(validCandidates) == 0 { diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 820f73e1d1e..bc7c31232db 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -57,6 +57,7 @@ type ( GetWaitReplicasTimeout() time.Duration GetIgnoreReplicas() sets.String CheckPrimaryRecoveryType() error + RestrictValidCandidates(map[string]mysql.Position, map[string]*topo.TabletInfo) (map[string]mysql.Position, error) FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) error CheckIfNeedToOverridePrimary() error StartReplication(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) error @@ -145,6 +146,26 @@ func (vtctlReparent *VtctlReparentFunctions) CheckPrimaryRecoveryType() error { return nil } +// RestrictValidCandidates implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) RestrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { + restrictedValidCandidates := make(map[string]mysql.Position) + + for candidate, position := range validCandidates { + candidateInfo, ok := tabletMap[candidate] + if !ok { + return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) + } + + if candidateInfo.Type != topodatapb.TabletType_PRIMARY && candidateInfo.Type != topodatapb.TabletType_REPLICA { + continue + } + + restrictedValidCandidates[candidate] = position + } + + return restrictedValidCandidates, nil +} + // FindPrimaryCandidates implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, m map[string]*topo.TabletInfo) error { // Elect the candidate with the most up-to-date position. From 49cf202886ea348f03f96e5ebc4424538adb39d0 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 19 Aug 2021 11:14:36 +0530 Subject: [PATCH 025/176] added logger to vtorc Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/topology_recovery.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index ba2fe319d24..b28d70e82a5 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -27,6 +27,9 @@ import ( "sync/atomic" "time" + "vitess.io/vitess/go/vt/logutil" + logutilpb "vitess.io/vitess/go/vt/proto/logutil" + "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/vttablet/tmclient" @@ -624,7 +627,18 @@ func checkAndRecoverDeadMaster(analysisEntry inst.ReplicationAnalysis, candidate log.Infof("Analysis: %v, deadmaster %+v", analysisEntry.Analysis, analysisEntry.AnalyzedInstanceKey) reparentFunctions := NewVtorcReparentFunctions(analysisEntry, candidateInstanceKey, skipProcesses, topologyRecovery) - _, err = reparentutil.NewEmergencyReparenter(tmclient.NewTabletManagerClient(), nil).ReparentShard(context.Background(), reparentFunctions) + _, err = reparentutil.NewEmergencyReparenter(tmclient.NewTabletManagerClient(), logutil.NewCallbackLogger(func(event *logutilpb.Event) { + level := event.GetLevel() + value := event.GetValue() + switch level { + case logutilpb.Level_WARNING: + log.Warningf("ERP - %s", value) + case logutilpb.Level_ERROR: + log.Errorf("ERP - %s", value) + default: + log.Infof("ERP - %s", value) + } + })).ReparentShard(context.Background(), reparentFunctions) return reparentFunctions.recoveryAttempted, topologyRecovery, err } From 558d7b3e7d49f4951d77abb14d6e8da59e3fe826 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 19 Aug 2021 16:40:57 +0530 Subject: [PATCH 026/176] fix test name and comments Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/primary_failure_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index e54b065e0c4..9ec739777da 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -128,9 +128,9 @@ func TestCrossDataCenterFailureError(t *testing.T) { require.NotEqual(t, topodatapb.TabletType_PRIMARY, tabletInfo.GetType()) } -// Failover will sometimes lead to a replica which can no longer replicate. +// Failover will sometimes lead to a rdonly which can no longer replicate. // covers part of the test case master-failover-lost-replicas from orchestrator -func TestLostReplicasOnPrimaryFailure(t *testing.T) { +func TestLostRdonlyOnPrimaryFailure(t *testing.T) { defer cluster.PanicHandler(t) setupVttabletsAndVtorc(t, 2, 1, nil, "test_config.json") keyspace := &clusterInstance.Keyspaces[0] @@ -181,7 +181,7 @@ func TestLostReplicasOnPrimaryFailure(t *testing.T) { // vtorc must promote the lagging replica and not the rdonly, since it has a MustNotPromoteRule promotion rule checkPrimaryTablet(t, clusterInstance, replica) - // check that the rdonly replica is lost. The lost replica has is detached and its host is prepended with `//` + // check that the rdonly is lost. The lost replica has is detached and its host is prepended with `//` out, err := runSQL(t, "SELECT HOST FROM performance_schema.replication_connection_configuration", rdonly, "") require.NoError(t, err) require.Equal(t, "//localhost", out.Rows[0][0].ToString()) From 073df19d7638ad776a8a7f1b384717bd2b91f98d Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 19 Aug 2021 20:24:18 +0530 Subject: [PATCH 027/176] fixed the test so that it does not introduce errant GTIDs Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/main_test.go | 14 ----------- .../endtoend/vtorc/primary_failure_test.go | 25 ++++++++++++------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/go/test/endtoend/vtorc/main_test.go b/go/test/endtoend/vtorc/main_test.go index c7de3c44fc9..63f7f73f2c1 100644 --- a/go/test/endtoend/vtorc/main_test.go +++ b/go/test/endtoend/vtorc/main_test.go @@ -662,17 +662,3 @@ func permanentlyRemoveVttablet(tablet *cluster.Vttablet) { } } } - -func changePrivileges(t *testing.T, sql string, tablet *cluster.Vttablet, user string) { - _, err := runSQL(t, sql, tablet, "") - require.NoError(t, err) - - res, err := runSQL(t, fmt.Sprintf("SELECT id FROM INFORMATION_SCHEMA.PROCESSLIST WHERE user = '%s'", user), tablet, "") - require.NoError(t, err) - for _, row := range res.Rows { - id, err := row[0].ToInt64() - require.NoError(t, err) - _, err = runSQL(t, fmt.Sprintf("kill %d", id), tablet, "") - require.NoError(t, err) - } -} diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index 9ec739777da..3eb08d67309 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -157,16 +157,26 @@ func TestLostRdonlyOnPrimaryFailure(t *testing.T) { // check that replication is setup correctly checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{rdonly, replica}, 15*time.Second) - // revoke super privileges from vtorc on replica so that it is unable to repair the replication - changePrivileges(t, `REVOKE SUPER ON *.* FROM 'orc_client_user'@'%'`, replica, "orc_client_user") + // make the replica lag by setting the source_delay to 20 seconds + runSQL(t, "STOP SLAVE", replica, "") + runSQL(t, "CHANGE MASTER TO MASTER_DELAY = 20", replica, "") + runSQL(t, "START SLAVE", replica, "") - // stop replication on the replica. - err := clusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", replica.Alias) - require.NoError(t, err) + defer func() { + // fix the crossCell replica back so that no other tests see this as a side effect + runSQL(t, "STOP SLAVE", replica, "") + runSQL(t, "CHANGE MASTER TO MASTER_DELAY = 0", replica, "") + runSQL(t, "START SLAVE", replica, "") + }() // check that rdonly is able to replicate. We also want to add some queries to rdonly which will not be there in replica runAdditionalCommands(t, curPrimary, []*cluster.Vttablet{rdonly}, 15*time.Second) + // assert that the replica is indeed lagging and does not have the new insertion by checking the count of rows in the table + out, err := runSQL(t, "SELECT * FROM vt_insert_test", replica, "vt_ks") + require.NoError(t, err) + require.Equal(t, 1, len(out.Rows)) + // Make the current primary database unavailable. err = curPrimary.MysqlctlProcess.Stop() require.NoError(t, err) @@ -175,14 +185,11 @@ func TestLostRdonlyOnPrimaryFailure(t *testing.T) { permanentlyRemoveVttablet(curPrimary) }() - // grant super privileges back to vtorc on replica so that it can repair - changePrivileges(t, `GRANT SUPER ON *.* TO 'orc_client_user'@'%'`, replica, "orc_client_user") - // vtorc must promote the lagging replica and not the rdonly, since it has a MustNotPromoteRule promotion rule checkPrimaryTablet(t, clusterInstance, replica) // check that the rdonly is lost. The lost replica has is detached and its host is prepended with `//` - out, err := runSQL(t, "SELECT HOST FROM performance_schema.replication_connection_configuration", rdonly, "") + out, err = runSQL(t, "SELECT HOST FROM performance_schema.replication_connection_configuration", rdonly, "") require.NoError(t, err) require.Equal(t, "//localhost", out.Rows[0][0].ToString()) } From 058901cf64373d52b18703729ff29fc5a3781fd7 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 19 Aug 2021 20:24:44 +0530 Subject: [PATCH 028/176] bug fix Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 29 +++---------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 04d7b54e45e..6d4303d952f 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -148,31 +148,10 @@ func (vtorcReparent *VtOrcReparentFunctions) CheckPrimaryRecoveryType() error { // RestrictValidCandidates implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) RestrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { - restrictedValidCandidates := make(map[string]mysql.Position) - - for candidate, position := range validCandidates { - candidateInfo, ok := tabletMap[candidate] - if !ok { - return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) - } - - candidateInstance, _, err := inst.ReadInstance(&inst.InstanceKey{ - Hostname: candidateInfo.MysqlHostname, - Port: int(candidateInfo.MysqlPort), - }) - - if err != nil { - return nil, err - } - - if candidateInstance.PromotionRule == inst.MustNotPromoteRule { - continue - } - - restrictedValidCandidates[candidate] = position - } - - return restrictedValidCandidates, nil + // we do not restrict the valid candidates for VtOrc for 2 reasons - + // any candidate that can no longer replicate from the new primary is detached later on + // we already restrict which candidate can become the primary via inst.IsBannedFromBeingCandidateReplica when we choose the candidate + return validCandidates, nil } // FindPrimaryCandidates implements the ReparentFunctions interface From ae49168d4cc064c85bc1ba1daa166c113b86fe65 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 23 Aug 2021 15:00:00 +0530 Subject: [PATCH 029/176] handle a TODO Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 75 ++++++++++--------- go/vt/vtctl/reparentutil/util.go | 11 +-- 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 6d4303d952f..51161d1908c 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -243,7 +243,6 @@ func ChooseCandidate( } var replicas [](*inst.Instance) - log.Errorf("started Manan's new function") for candidate := range validCandidates { candidateInfo, ok := tabletMap[candidate] @@ -288,51 +287,53 @@ func ChooseCandidate( } } - // TODO: Use the set replication source functionality instead of the moveReplicasViaGTID - //now := time.Now().UnixNano() + now := time.Now().UnixNano() if err := inst.SwitchMaster(candidateReplica.Key, *masterKey); err != nil { return emptyReplicas, candidateReplica, err } - //candidateReplicaTablet, err := inst.ReadTablet(candidateReplica.Key) - //if err != nil { - // return emptyReplicas, candidateReplica, err - //} - - //moveGTIDFunc := func() error { - // log.Debugf("RegroupReplicasGTID: working on %d replicas", len(replicasToMove)) - // - // for _, instance := range replicasToMove { - // tablet, err := inst.ReadTablet(instance.Key) - // if err != nil { - // return err - // } - // - // err = tmc.SetReplicationSource(context.Background(), tablet, candidateReplicaTablet.Alias, now, "", false) - // if err != nil { - // unmovedReplicas = append(unmovedReplicas, instance) - // err = vterrors.Wrapf(err, "tablet %v SetMaster failed: %v", tablet.Alias, err) - // return err - // } - // movedReplicas = append(movedReplicas, instance) - // } - // - // unmovedReplicas = append(unmovedReplicas, aheadReplicas...) - // unmovedReplicas = append(unmovedReplicas, cannotReplicateReplicas...) - // return log.Errore(err) - //} + candidateReplicaTablet, err := inst.ReadTablet(candidateReplica.Key) + if err != nil { + return emptyReplicas, candidateReplica, err + } + moveGTIDFunc := func() error { log.Debugf("RegroupReplicasGTID: working on %d replicas", len(replicasToMove)) - movedReplicas, unmovedReplicas, err, _ = inst.MoveReplicasViaGTID(replicasToMove, candidateReplica, postponedFunctionsContainer) + for _, instance := range replicasToMove { + tablet, err := inst.ReadTablet(instance.Key) + if err != nil { + goto Cleanup + } + + if maintenanceToken, merr := inst.BeginMaintenance(&instance.Key, inst.GetMaintenanceOwner(), fmt.Sprintf("move below %+v", candidateReplica.Key)); merr != nil { + err = fmt.Errorf("Cannot begin maintenance on %+v: %v", instance.Key, merr) + goto Cleanup + } else { + defer inst.EndMaintenance(maintenanceToken) + } + + err = tmc.SetReplicationSource(context.Background(), tablet, candidateReplicaTablet.Alias, now, "", false) + if err != nil { + unmovedReplicas = append(unmovedReplicas, instance) + err = vterrors.Wrapf(err, "tablet %v SetMaster failed: %v", tablet.Alias, err) + goto Cleanup + } + movedReplicas = append(movedReplicas, instance) + + Cleanup: + tmc.StartReplication(context.Background(), tablet) + return err + } + unmovedReplicas = append(unmovedReplicas, aheadReplicas...) - return log.Errore(err) - } - if postponedFunctionsContainer != nil && postponeAllMatchOperations != nil && postponeAllMatchOperations(candidateReplica, hasBestPromotionRule) { - postponedFunctionsContainer.AddPostponedFunction(moveGTIDFunc, fmt.Sprintf("regroup-replicas-gtid %+v", candidateReplica.Key)) - } else { - err = moveGTIDFunc() + unmovedReplicas = append(unmovedReplicas, cannotReplicateReplicas...) + if err != nil { + log.Errore(err) + } + return nil } + if postponedFunctionsContainer != nil && postponeAllMatchOperations != nil && postponeAllMatchOperations(candidateReplica, hasBestPromotionRule) { postponedFunctionsContainer.AddPostponedFunction(moveGTIDFunc, fmt.Sprintf("regroup-replicas-gtid %+v", candidateReplica.Key)) } else { diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index bc7c31232db..5835cb699d1 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -178,11 +178,6 @@ func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.C vtctlReparent.validCandidates = validCandidates - return nil -} - -// CheckIfNeedToOverridePrimary implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePrimary() error { // If we were requested to elect a particular primary, verify it's a valid // candidate (non-zero position, no errant GTIDs) and is at least as // advanced as the winning position. @@ -196,6 +191,12 @@ func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePrimary() erro return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v at position %v is not fully caught up. Winning position: %v", vtctlReparent.winningPrimaryTabletAliasStr, pos, vtctlReparent.winningPosition) } } + + return nil +} + +// CheckIfNeedToOverridePrimary implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePrimary() error { return nil } From 7572ba9c1e86f7c5e9c5d15cc27c38953f0dc6ed Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 23 Aug 2021 18:26:49 +0530 Subject: [PATCH 030/176] create a new function for promoting primary Signed-off-by: Manan Gupta --- .../testutil/test_tmclient.go | 26 +++++ go/vt/vtctl/reparentutil/util.go | 110 ++++++++++++++---- 2 files changed, 111 insertions(+), 25 deletions(-) diff --git a/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go b/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go index 34a1b850c6a..15c6705b835 100644 --- a/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go +++ b/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go @@ -435,6 +435,32 @@ func (fake *TabletManagerClient) SetMaster(ctx context.Context, tablet *topodata return assert.AnError } +// SetReplicationSource is part of the tmclient.TabletManagerClient interface. +func (fake *TabletManagerClient) SetReplicationSource(ctx context.Context, tablet *topodatapb.Tablet, parent *topodatapb.TabletAlias, timeCreatedNS int64, waitPosition string, forceStartReplication bool) error { + if fake.SetMasterResults == nil { + return assert.AnError + } + + key := topoproto.TabletAliasString(tablet.Alias) + + if fake.SetMasterDelays != nil { + if delay, ok := fake.SetMasterDelays[key]; ok { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + // proceed to results + } + } + } + + if result, ok := fake.SetMasterResults[key]; ok { + return result + } + + return assert.AnError +} + // SetReadWrite is part of the tmclient.TabletManagerClient interface. func (fake *TabletManagerClient) SetReadWrite(ctx context.Context, tablet *topodatapb.Tablet) error { if fake.SetReadWriteResults == nil { diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 5835cb699d1..0bcbe317696 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -22,6 +22,8 @@ import ( "sync" "time" + "google.golang.org/protobuf/proto" + "vitess.io/vitess/go/event" "vitess.io/vitess/go/vt/concurrency" @@ -203,7 +205,8 @@ func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePrimary() erro // StartReplication implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) StartReplication(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) error { // Do the promotion. - return vtctlReparent.promoteNewPrimary(ctx, ev, logger, tmc) + //return vtctlReparent.promoteNewPrimary(ctx, ev, logger, tmc) + return nil } // GetNewPrimary implements the ReparentFunctions interface @@ -301,7 +304,13 @@ func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Conte return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } - replCtx, replCancel := context.WithTimeout(ctx, vtctlReparent.WaitReplicasTimeout) + return reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.lockAction, rp, vtctlReparent.tabletMap, vtctlReparent) +} + +func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent, logger logutil.Logger, + tmc tmclient.TabletManagerClient, newPrimaryTablet *topodatapb.Tablet, lockAction, rp string, + tabletMap map[string]*topo.TabletInfo, reparentFunction ReparentFunctions) error { + replCtx, replCancel := context.WithTimeout(ctx, reparentFunction.GetWaitReplicasTimeout()) defer replCancel() event.DispatchUpdate(ev, "reparenting all tablets") @@ -323,47 +332,34 @@ func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Conte replWg := sync.WaitGroup{} rec := concurrency.AllErrorRecorder{} - handlePrimary := func(alias string, ti *topo.TabletInfo) error { + handlePrimary := func(alias string, ti *topodatapb.Tablet) error { logger.Infof("populating reparent journal on new master %v", alias) - return tmc.PopulateReparentJournal(replCtx, ti.Tablet, now, vtctlReparent.lockAction, newPrimaryTabletInfo.Alias, rp) + return tmc.PopulateReparentJournal(replCtx, ti, now, lockAction, newPrimaryTablet.Alias, rp) } handleReplica := func(alias string, ti *topo.TabletInfo) { defer replWg.Done() logger.Infof("setting new master on replica %v", alias) - forceStart := false - if status, ok := vtctlReparent.statusMap[alias]; ok { - fs, err := ReplicaWasRunning(status) - if err != nil { - err = vterrors.Wrapf(err, "tablet %v could not determine StopReplicationStatus: %v", alias, err) - rec.RecordError(err) - - return - } - - forceStart = fs - } - - err := tmc.SetMaster(replCtx, ti.Tablet, newPrimaryTabletInfo.Alias, now, "", forceStart) + err := tmc.SetReplicationSource(replCtx, ti.Tablet, newPrimaryTablet.Alias, now, "", true) if err != nil { - err = vterrors.Wrapf(err, "tablet %v SetMaster failed: %v", alias, err) + err = vterrors.Wrapf(err, "tablet %v SetReplicationSource failed: %v", alias, err) rec.RecordError(err) return } - // Signal that at least one goroutine succeeded to SetMaster. + // Signal that at least one goroutine succeeded to SetReplicationSource. replSuccessCancel() } numReplicas := 0 - for alias, ti := range vtctlReparent.tabletMap { + for alias, ti := range tabletMap { switch { - case alias == vtctlReparent.winningPrimaryTabletAliasStr: + case alias == topoproto.TabletAliasString(newPrimaryTablet.Alias): continue - case !vtctlReparent.IgnoreReplicas.Has(alias): + case !reparentFunction.GetIgnoreReplicas().Has(alias): replWg.Add(1) numReplicas++ go handleReplica(alias, ti) @@ -371,7 +367,7 @@ func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Conte } // Spin up a background goroutine to wait until all replica goroutines - // finished. Polling this way allows us to have promoteNewPrimary return + // finished. Polling this way allows us to have reparentReplicasAndPopulateJournal return // success as soon as (a) the primary successfully populates its reparent // journal and (b) at least one replica successfully begins replicating. // @@ -384,7 +380,7 @@ func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Conte allReplicasDoneCancel() }() - primaryErr := handlePrimary(vtctlReparent.winningPrimaryTabletAliasStr, newPrimaryTabletInfo) + primaryErr := handlePrimary(topoproto.TabletAliasString(newPrimaryTablet.Alias), newPrimaryTablet) if primaryErr != nil { logger.Warningf("master failed to PopulateReparentJournal") replCancel() @@ -478,6 +474,70 @@ func ChooseNewPrimary( return nil, nil } +// PromotePrimaryCandidateAndStartReplication promotes the primary candidate that we have, but it does not set to start accepting writes +func PromotePrimaryCandidateAndStartReplication(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, + lockAction, rp string, tabletMap map[string]*topo.TabletInfo, reparentFunctions ReparentFunctions) error { + if err := promotePrimary(ctx, tmc, ts, logger, newPrimary); err != nil { + return err + } + + if err := reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, rp, tabletMap, reparentFunctions); err != nil { + return err + } + + // TODO := add as a postponed function + //if postponedFunctionsContainer != nil && postponeAllMatchOperations != nil && postponeAllMatchOperations(candidateReplica, hasBestPromotionRule) { + // postponedFunctionsContainer.AddPostponedFunction(moveGTIDFunc, fmt.Sprintf("regroup-replicas-gtid %+v", candidateReplica.Key)) + //} else { + // err = moveGTIDFunc() + //} + + err := tmc.StartReplication(ctx, newPrimary) + if err != nil { + return err + } + + //log.Debugf("RegroupReplicasGTID: done") + //inst.AuditOperation("regroup-replicas-gtid", masterKey, fmt.Sprintf("regrouped replicas of %+v via GTID; promoted %+v", *masterKey, candidateReplica.Key)) + return nil //unmovedReplicas, candidateReplica, err +} + +// promotePrimary makes the new tablet the primary and proactively performs +// the necessary propagation to the old primary. The propagation is best +// effort. If it fails, the tablet's shard sync will eventually converge. +func promotePrimary(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, logger logutil.Logger, newPrimary *topodatapb.Tablet) error { + err := tmc.ChangeType(ctx, newPrimary, topodatapb.TabletType_PRIMARY) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) + defer cancel() + _, err = ts.UpdateShardFields(ctx, newPrimary.Keyspace, newPrimary.Shard, func(si *topo.ShardInfo) error { + if proto.Equal(si.PrimaryAlias, newPrimary.Alias) && proto.Equal(si.PrimaryTermStartTime, newPrimary.PrimaryTermStartTime) { + return topo.NewError(topo.NoUpdateNeeded, "") + } + + // We just successfully reparented. We should check timestamps, but always overwrite. + lastTerm := si.GetPrimaryTermStartTime() + newTerm := logutil.ProtoToTime(newPrimary.PrimaryTermStartTime) + if !newTerm.After(lastTerm) { + logger.Errorf("Possible clock skew. New master start time is before previous one: %v vs %v", newTerm, lastTerm) + } + + aliasStr := topoproto.TabletAliasString(newPrimary.Alias) + logger.Infof("Updating shard record: master_alias=%v, primary_term_start_time=%v", aliasStr, newTerm) + si.PrimaryAlias = newPrimary.Alias + si.PrimaryTermStartTime = newPrimary.PrimaryTermStartTime + return nil + }) + // Log any error but do not abort + if err != nil { + logger.Error(err) + return nil + } + return nil +} + // FindCurrentPrimary returns the current primary tablet of a shard, if any. The // current primary is whichever tablet of type PRIMARY (if any) has the most // recent PrimaryTermStartTime, which is the same rule that vtgate uses to route From 96e72447db33e2013ef1f6cc5aef7c47b5f65dd5 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 23 Aug 2021 18:50:46 +0530 Subject: [PATCH 031/176] used the newly created function Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 164 ++++++++++-------- .../reparentutil/emergency_reparenter.go | 8 +- go/vt/vtctl/reparentutil/util.go | 18 +- 3 files changed, 104 insertions(+), 86 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 51161d1908c..5daf1ae2e79 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -59,6 +59,7 @@ type VtOrcReparentFunctions struct { promotedReplica *inst.Instance lostReplicas [](*inst.Instance) recoveryAttempted bool + postponedAll bool } func NewVtorcReparentFunctions(analysisEntry inst.ReplicationAnalysis, candidateInstanceKey *inst.InstanceKey, skipProcesses bool, topologyRecovery *TopologyRecovery) *VtOrcReparentFunctions { @@ -155,8 +156,8 @@ func (vtorcReparent *VtOrcReparentFunctions) RestrictValidCandidates(validCandid } // FindPrimaryCandidates implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) error { - postponedAll := false +func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, error) { + vtorcReparent.postponedAll = false promotedReplicaIsIdeal := func(promoted *inst.Instance, hasBestPromotionRule bool) bool { if promoted == nil { return false @@ -170,7 +171,7 @@ func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.C if promoted.PromotionRule == inst.MustPromoteRule || promoted.PromotionRule == inst.PreferPromoteRule || (hasBestPromotionRule && promoted.PromotionRule != inst.MustNotPromoteRule) { AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: found %+v to be ideal candidate; will optimize recovery", promoted.Key)) - postponedAll = true + vtorcReparent.postponedAll = true return true } } @@ -180,6 +181,14 @@ func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.C AuditTopologyRecovery(vtorcReparent.topologyRecovery, "RecoverDeadMaster: regrouping replicas via GTID") lostReplicas, promotedReplica, err := ChooseCandidate(tmc, &vtorcReparent.analysisEntry.AnalyzedInstanceKey, &vtorcReparent.topologyRecovery.PostponedFunctionsContainer, promotedReplicaIsIdeal, validCandidates, tabletMap) vtorcReparent.topologyRecovery.AddError(err) + if err != nil { + return nil, err + } + newPrimary, err := inst.ReadTablet(promotedReplica.Key) + if err != nil { + return nil, err + } + for _, replica := range lostReplicas { AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: - lost replica: %+v", replica.Key)) } @@ -209,15 +218,10 @@ func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.C AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: %d postponed functions", vtorcReparent.topologyRecovery.PostponedFunctionsContainer.Len())) - if promotedReplica != nil && !postponedAll { - promotedReplica, err = replacePromotedReplicaWithCandidate(vtorcReparent.topologyRecovery, &vtorcReparent.analysisEntry.AnalyzedInstanceKey, promotedReplica, vtorcReparent.candidateInstanceKey) - vtorcReparent.topologyRecovery.AddError(err) - } - vtorcReparent.promotedReplica = promotedReplica vtorcReparent.lostReplicas = lostReplicas vtorcReparent.recoveryAttempted = true - return nil + return newPrimary, nil } // ChooseCandidate will choose a candidate replica of a given instance, and take its siblings using GTID @@ -235,7 +239,7 @@ func ChooseCandidate( ) { var emptyReplicas [](*inst.Instance) var unmovedReplicas [](*inst.Instance) - var movedReplicas [](*inst.Instance) + //var movedReplicas [](*inst.Instance) dataCenterHint := "" if master, _, _ := inst.ReadInstance(masterKey); master != nil { @@ -277,78 +281,84 @@ func ChooseCandidate( } log.Debugf("GetCandidateReplica: candidate: %+v, ahead: %d, equal: %d, late: %d, break: %d", candidateReplica.Key, len(aheadReplicas), len(equalReplicas), len(laterReplicas), len(cannotReplicateReplicas)) - replicasToMove := append(equalReplicas, laterReplicas...) - hasBestPromotionRule := true - if candidateReplica != nil { - for _, replica := range replicasToMove { - if replica.PromotionRule.BetterThan(candidateReplica.PromotionRule) { - hasBestPromotionRule = false - } - } - } - - now := time.Now().UnixNano() - if err := inst.SwitchMaster(candidateReplica.Key, *masterKey); err != nil { - return emptyReplicas, candidateReplica, err - } - - candidateReplicaTablet, err := inst.ReadTablet(candidateReplica.Key) - if err != nil { - return emptyReplicas, candidateReplica, err - } - - moveGTIDFunc := func() error { - log.Debugf("RegroupReplicasGTID: working on %d replicas", len(replicasToMove)) - - for _, instance := range replicasToMove { - tablet, err := inst.ReadTablet(instance.Key) - if err != nil { - goto Cleanup - } - - if maintenanceToken, merr := inst.BeginMaintenance(&instance.Key, inst.GetMaintenanceOwner(), fmt.Sprintf("move below %+v", candidateReplica.Key)); merr != nil { - err = fmt.Errorf("Cannot begin maintenance on %+v: %v", instance.Key, merr) - goto Cleanup - } else { - defer inst.EndMaintenance(maintenanceToken) - } - - err = tmc.SetReplicationSource(context.Background(), tablet, candidateReplicaTablet.Alias, now, "", false) - if err != nil { - unmovedReplicas = append(unmovedReplicas, instance) - err = vterrors.Wrapf(err, "tablet %v SetMaster failed: %v", tablet.Alias, err) - goto Cleanup - } - movedReplicas = append(movedReplicas, instance) - - Cleanup: - tmc.StartReplication(context.Background(), tablet) - return err - } - - unmovedReplicas = append(unmovedReplicas, aheadReplicas...) - unmovedReplicas = append(unmovedReplicas, cannotReplicateReplicas...) - if err != nil { - log.Errore(err) - } - return nil - } - - if postponedFunctionsContainer != nil && postponeAllMatchOperations != nil && postponeAllMatchOperations(candidateReplica, hasBestPromotionRule) { - postponedFunctionsContainer.AddPostponedFunction(moveGTIDFunc, fmt.Sprintf("regroup-replicas-gtid %+v", candidateReplica.Key)) - } else { - err = moveGTIDFunc() - } - - inst.StartReplication(&candidateReplica.Key) - - log.Debugf("RegroupReplicasGTID: done") - inst.AuditOperation("regroup-replicas-gtid", masterKey, fmt.Sprintf("regrouped replicas of %+v via GTID; promoted %+v", *masterKey, candidateReplica.Key)) + //replicasToMove := append(equalReplicas, laterReplicas...) + //hasBestPromotionRule := true + //if candidateReplica != nil { + // for _, replica := range replicasToMove { + // if replica.PromotionRule.BetterThan(candidateReplica.PromotionRule) { + // hasBestPromotionRule = false + // } + // } + //} + + //now := time.Now().UnixNano() + //if err := inst.SwitchMaster(candidateReplica.Key, *masterKey); err != nil { + // return emptyReplicas, candidateReplica, err + //} + // + //candidateReplicaTablet, err := inst.ReadTablet(candidateReplica.Key) + //if err != nil { + // return emptyReplicas, candidateReplica, err + //} + // + //moveGTIDFunc := func() error { + // log.Debugf("RegroupReplicasGTID: working on %d replicas", len(replicasToMove)) + // + // for _, instance := range replicasToMove { + // tablet, err := inst.ReadTablet(instance.Key) + // if err != nil { + // goto Cleanup + // } + // + // if maintenanceToken, merr := inst.BeginMaintenance(&instance.Key, inst.GetMaintenanceOwner(), fmt.Sprintf("move below %+v", candidateReplica.Key)); merr != nil { + // err = fmt.Errorf("Cannot begin maintenance on %+v: %v", instance.Key, merr) + // goto Cleanup + // } else { + // defer inst.EndMaintenance(maintenanceToken) + // } + // + // err = tmc.SetReplicationSource(context.Background(), tablet, candidateReplicaTablet.Alias, now, "", false) + // if err != nil { + // unmovedReplicas = append(unmovedReplicas, instance) + // err = vterrors.Wrapf(err, "tablet %v SetMaster failed: %v", tablet.Alias, err) + // goto Cleanup + // } + // movedReplicas = append(movedReplicas, instance) + // + // Cleanup: + // tmc.StartReplication(context.Background(), tablet) + // return err + // } + // + unmovedReplicas = append(unmovedReplicas, aheadReplicas...) + unmovedReplicas = append(unmovedReplicas, cannotReplicateReplicas...) + // if err != nil { + // log.Errore(err) + // } + // return nil + //} + // + //if postponedFunctionsContainer != nil && postponeAllMatchOperations != nil && postponeAllMatchOperations(candidateReplica, hasBestPromotionRule) { + // postponedFunctionsContainer.AddPostponedFunction(moveGTIDFunc, fmt.Sprintf("regroup-replicas-gtid %+v", candidateReplica.Key)) + //} else { + // err = moveGTIDFunc() + //} + // + //inst.StartReplication(&candidateReplica.Key) + // + //log.Debugf("RegroupReplicasGTID: done") + //inst.AuditOperation("regroup-replicas-gtid", masterKey, fmt.Sprintf("regrouped replicas of %+v via GTID; promoted %+v", *masterKey, candidateReplica.Key)) return unmovedReplicas, candidateReplica, err } // CheckIfNeedToOverridePrimary implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) CheckIfNeedToOverridePrimary() error { + if vtorcReparent.promotedReplica != nil && !vtorcReparent.postponedAll { + var err error + vtorcReparent.promotedReplica, err = replacePromotedReplicaWithCandidate(vtorcReparent.topologyRecovery, &vtorcReparent.analysisEntry.AnalyzedInstanceKey, vtorcReparent.promotedReplica, vtorcReparent.candidateInstanceKey) + vtorcReparent.topologyRecovery.AddError(err) + } + if vtorcReparent.promotedReplica == nil { err := TabletUndoDemoteMaster(vtorcReparent.analysisEntry.AnalyzedInstanceKey) AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: TabletUndoDemoteMaster: %v", err)) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 8c6e7b5ede5..66aeeabf3b8 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -143,7 +143,13 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } - if err := reparentFunctions.FindPrimaryCandidates(ctx, erp.logger, erp.tmc, validCandidates, tabletMap); err != nil { + newPrimary, err := reparentFunctions.FindPrimaryCandidates(ctx, erp.logger, erp.tmc, validCandidates, tabletMap) + if err != nil { + return err + } + + // TODO := LockAction and RP + if err := PromotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, newPrimary, "", "", tabletMap, reparentFunctions); err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 0bcbe317696..1cbf0484879 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -60,7 +60,7 @@ type ( GetIgnoreReplicas() sets.String CheckPrimaryRecoveryType() error RestrictValidCandidates(map[string]mysql.Position, map[string]*topo.TabletInfo) (map[string]mysql.Position, error) - FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) error + FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) (*topodatapb.Tablet, error) CheckIfNeedToOverridePrimary() error StartReplication(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) error GetNewPrimary() *topodatapb.Tablet @@ -169,7 +169,7 @@ func (vtctlReparent *VtctlReparentFunctions) RestrictValidCandidates(validCandid } // FindPrimaryCandidates implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, m map[string]*topo.TabletInfo) error { +func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, error) { // Elect the candidate with the most up-to-date position. for alias, position := range validCandidates { if vtctlReparent.winningPosition.IsZero() || position.AtLeast(vtctlReparent.winningPosition) { @@ -188,13 +188,15 @@ func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.C pos, ok := vtctlReparent.validCandidates[vtctlReparent.winningPrimaryTabletAliasStr] switch { case !ok: - return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v has errant GTIDs", vtctlReparent.winningPrimaryTabletAliasStr) + return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v has errant GTIDs", vtctlReparent.winningPrimaryTabletAliasStr) case !pos.AtLeast(vtctlReparent.winningPosition): - return vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v at position %v is not fully caught up. Winning position: %v", vtctlReparent.winningPrimaryTabletAliasStr, pos, vtctlReparent.winningPosition) + return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v at position %v is not fully caught up. Winning position: %v", vtctlReparent.winningPrimaryTabletAliasStr, pos, vtctlReparent.winningPosition) } } - return nil + // TODO:= handle not found error + newPrimaryAlias := tabletMap[vtctlReparent.winningPrimaryTabletAliasStr] + return newPrimaryAlias.Tablet, nil } // CheckIfNeedToOverridePrimary implements the ReparentFunctions interface @@ -309,8 +311,8 @@ func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Conte func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, newPrimaryTablet *topodatapb.Tablet, lockAction, rp string, - tabletMap map[string]*topo.TabletInfo, reparentFunction ReparentFunctions) error { - replCtx, replCancel := context.WithTimeout(ctx, reparentFunction.GetWaitReplicasTimeout()) + tabletMap map[string]*topo.TabletInfo, reparentFunctions ReparentFunctions) error { + replCtx, replCancel := context.WithTimeout(ctx, reparentFunctions.GetWaitReplicasTimeout()) defer replCancel() event.DispatchUpdate(ev, "reparenting all tablets") @@ -359,7 +361,7 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent switch { case alias == topoproto.TabletAliasString(newPrimaryTablet.Alias): continue - case !reparentFunction.GetIgnoreReplicas().Has(alias): + case !reparentFunctions.GetIgnoreReplicas().Has(alias): replWg.Add(1) numReplicas++ go handleReplica(alias, ti) From b91f34ca205ec5e39461c30fb11327866de5eb4a Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 24 Aug 2021 18:38:57 +0530 Subject: [PATCH 032/176] added the function to check whether the selected primary candidate is ideal or not Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 102 ++++++++++++------ .../reparentutil/emergency_reparenter.go | 4 +- go/vt/vtctl/reparentutil/util.go | 50 +++++++-- 3 files changed, 114 insertions(+), 42 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 5daf1ae2e79..cd39ee9a00f 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -59,6 +59,7 @@ type VtOrcReparentFunctions struct { promotedReplica *inst.Instance lostReplicas [](*inst.Instance) recoveryAttempted bool + hasBestPromotionRule bool postponedAll bool } @@ -158,29 +159,30 @@ func (vtorcReparent *VtOrcReparentFunctions) RestrictValidCandidates(validCandid // FindPrimaryCandidates implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, error) { vtorcReparent.postponedAll = false - promotedReplicaIsIdeal := func(promoted *inst.Instance, hasBestPromotionRule bool) bool { - if promoted == nil { - return false - } - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: promotedReplicaIsIdeal(%+v)", promoted.Key)) - if vtorcReparent.candidateInstanceKey != nil { //explicit request to promote a specific server - return promoted.Key.Equals(vtorcReparent.candidateInstanceKey) - } - if promoted.DataCenter == vtorcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstanceDataCenter && - promoted.PhysicalEnvironment == vtorcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstancePhysicalEnvironment { - if promoted.PromotionRule == inst.MustPromoteRule || promoted.PromotionRule == inst.PreferPromoteRule || - (hasBestPromotionRule && promoted.PromotionRule != inst.MustNotPromoteRule) { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: found %+v to be ideal candidate; will optimize recovery", promoted.Key)) - vtorcReparent.postponedAll = true - return true - } - } - return false - } + //promotedReplicaIsIdeal := func(promoted *inst.Instance, hasBestPromotionRule bool) bool { + // if promoted == nil { + // return false + // } + // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: promotedReplicaIsIdeal(%+v)", promoted.Key)) + // if vtorcReparent.candidateInstanceKey != nil { //explicit request to promote a specific server + // return promoted.Key.Equals(vtorcReparent.candidateInstanceKey) + // } + // if promoted.DataCenter == vtorcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstanceDataCenter && + // promoted.PhysicalEnvironment == vtorcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstancePhysicalEnvironment { + // if promoted.PromotionRule == inst.MustPromoteRule || promoted.PromotionRule == inst.PreferPromoteRule || + // (hasBestPromotionRule && promoted.PromotionRule != inst.MustNotPromoteRule) { + // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: found %+v to be ideal candidate; will optimize recovery", promoted.Key)) + // vtorcReparent.postponedAll = true + // return true + // } + // } + // return false + //} AuditTopologyRecovery(vtorcReparent.topologyRecovery, "RecoverDeadMaster: regrouping replicas via GTID") - lostReplicas, promotedReplica, err := ChooseCandidate(tmc, &vtorcReparent.analysisEntry.AnalyzedInstanceKey, &vtorcReparent.topologyRecovery.PostponedFunctionsContainer, promotedReplicaIsIdeal, validCandidates, tabletMap) + lostReplicas, promotedReplica, hasBestPromotionRule, err := ChooseCandidate(tmc, &vtorcReparent.analysisEntry.AnalyzedInstanceKey, validCandidates, tabletMap) vtorcReparent.topologyRecovery.AddError(err) + vtorcReparent.hasBestPromotionRule = hasBestPromotionRule if err != nil { return nil, err } @@ -228,13 +230,12 @@ func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.C func ChooseCandidate( tmc tmclient.TabletManagerClient, masterKey *inst.InstanceKey, - postponedFunctionsContainer *inst.PostponedFunctionsContainer, - postponeAllMatchOperations func(*inst.Instance, bool) bool, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, ) ( lostReplicas [](*inst.Instance), candidateReplica *inst.Instance, + hasBestPromotionRule bool, err error, ) { var emptyReplicas [](*inst.Instance) @@ -251,7 +252,7 @@ func ChooseCandidate( for candidate := range validCandidates { candidateInfo, ok := tabletMap[candidate] if !ok { - return emptyReplicas, candidateReplica, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) + return emptyReplicas, candidateReplica, false, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) } candidateInstance, _, err := inst.ReadInstance(&inst.InstanceKey{ Hostname: candidateInfo.MysqlHostname, @@ -259,7 +260,7 @@ func ChooseCandidate( }) if err != nil { log.Errorf("%v", err) - return emptyReplicas, candidateReplica, err + return emptyReplicas, candidateReplica, false, err } replicas = append(replicas, candidateInstance) } @@ -271,7 +272,7 @@ func ChooseCandidate( candidateReplica, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := inst.ChooseCandidateReplica(replicas) if err != nil { - return emptyReplicas, candidateReplica, err + return emptyReplicas, candidateReplica, false, err } if candidateReplica != nil { mostUpToDateReplica := replicas[0] @@ -281,15 +282,15 @@ func ChooseCandidate( } log.Debugf("GetCandidateReplica: candidate: %+v, ahead: %d, equal: %d, late: %d, break: %d", candidateReplica.Key, len(aheadReplicas), len(equalReplicas), len(laterReplicas), len(cannotReplicateReplicas)) - //replicasToMove := append(equalReplicas, laterReplicas...) - //hasBestPromotionRule := true - //if candidateReplica != nil { - // for _, replica := range replicasToMove { - // if replica.PromotionRule.BetterThan(candidateReplica.PromotionRule) { - // hasBestPromotionRule = false - // } - // } - //} + replicasToMove := append(equalReplicas, laterReplicas...) + hasBestPromotionRule = true + if candidateReplica != nil { + for _, replica := range replicasToMove { + if replica.PromotionRule.BetterThan(candidateReplica.PromotionRule) { + hasBestPromotionRule = false + } + } + } //now := time.Now().UnixNano() //if err := inst.SwitchMaster(candidateReplica.Key, *masterKey); err != nil { @@ -348,7 +349,38 @@ func ChooseCandidate( // //log.Debugf("RegroupReplicasGTID: done") //inst.AuditOperation("regroup-replicas-gtid", masterKey, fmt.Sprintf("regrouped replicas of %+v via GTID; promoted %+v", *masterKey, candidateReplica.Key)) - return unmovedReplicas, candidateReplica, err + return unmovedReplicas, candidateReplica, hasBestPromotionRule, err +} + +// PromotedReplicaIsIdeal implements the ReparentFunctions interface +func (vtOrcReparent *VtOrcReparentFunctions) PromotedReplicaIsIdeal(newPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, primaryStatus map[string]*replicationdatapb.PrimaryStatus, validCandidates map[string]mysql.Position) bool { + AuditTopologyRecovery(vtOrcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: promotedReplicaIsIdeal(%+v)", newPrimary.Alias)) + newPrimaryKey := &inst.InstanceKey{ + Hostname: newPrimary.MysqlHostname, + Port: int(newPrimary.MysqlPort), + } + newPrimaryInst, _, _ := inst.ReadInstance(newPrimaryKey) + if vtOrcReparent.candidateInstanceKey != nil { //explicit request to promote a specific server + return newPrimaryKey.Equals(vtOrcReparent.candidateInstanceKey) + } + if newPrimaryInst.DataCenter == vtOrcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstanceDataCenter && + newPrimaryInst.PhysicalEnvironment == vtOrcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstancePhysicalEnvironment { + if newPrimaryInst.PromotionRule == inst.MustPromoteRule || newPrimaryInst.PromotionRule == inst.PreferPromoteRule || + (vtOrcReparent.hasBestPromotionRule && newPrimaryInst.PromotionRule != inst.MustNotPromoteRule) { + AuditTopologyRecovery(vtOrcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: found %+v to be ideal candidate; will optimize recovery", newPrimaryInst.Key)) + vtOrcReparent.postponedAll = true + return true + } + } + return false +} + +// PostReplicationChangeHook implements the ReparentFunctions interface +func (vtOrcReparent *VtOrcReparentFunctions) PostReplicationChangeHook(tablet *topodatapb.Tablet) { + inst.ReadTopologyInstance(&inst.InstanceKey{ + Hostname: tablet.MysqlHostname, + Port: int(tablet.MysqlPort), + }) } // CheckIfNeedToOverridePrimary implements the ReparentFunctions interface diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 66aeeabf3b8..401fa150fb8 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -148,8 +148,10 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } + isIdeal := reparentFunctions.PromotedReplicaIsIdeal(newPrimary, tabletMap, primaryStatusMap, validCandidates) + // TODO := LockAction and RP - if err := PromotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, newPrimary, "", "", tabletMap, reparentFunctions); err != nil { + if err := PromotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, newPrimary, "", "", tabletMap, reparentFunctions, isIdeal); err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 1cbf0484879..30abd54ab8f 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -61,6 +61,8 @@ type ( CheckPrimaryRecoveryType() error RestrictValidCandidates(map[string]mysql.Position, map[string]*topo.TabletInfo) (map[string]mysql.Position, error) FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) (*topodatapb.Tablet, error) + PromotedReplicaIsIdeal(*topodatapb.Tablet, map[string]*topo.TabletInfo, map[string]*replicationdatapb.PrimaryStatus, map[string]mysql.Position) bool + PostReplicationChangeHook(*topodatapb.Tablet) CheckIfNeedToOverridePrimary() error StartReplication(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) error GetNewPrimary() *topodatapb.Tablet @@ -199,6 +201,34 @@ func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.C return newPrimaryAlias.Tablet, nil } +// PostReplicationChangeHook implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) PostReplicationChangeHook(*topodatapb.Tablet) { + return +} + +// PromotedReplicaIsIdeal implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) PromotedReplicaIsIdeal(newPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, primaryStatus map[string]*replicationdatapb.PrimaryStatus, validCandidates map[string]mysql.Position) bool { + if vtctlReparent.NewPrimaryAlias != nil { + //explicit request to promote a specific tablet + return true + } + if len(primaryStatus) == 1 { + var prevPrimary *topo.TabletInfo + for tablet := range primaryStatus { + prevPrimary = tabletMap[tablet] + } + if (newPrimary.Type == topodatapb.TabletType_PRIMARY || newPrimary.Type == topodatapb.TabletType_REPLICA) && newPrimary.Alias.Cell == prevPrimary.Alias.Cell { + return true + } + return false + } + + if newPrimary.Type == topodatapb.TabletType_PRIMARY || newPrimary.Type == topodatapb.TabletType_REPLICA { + return true + } + return false +} + // CheckIfNeedToOverridePrimary implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePrimary() error { return nil @@ -306,12 +336,12 @@ func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Conte return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } - return reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.lockAction, rp, vtctlReparent.tabletMap, vtctlReparent) + return reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.lockAction, rp, vtctlReparent.tabletMap, vtctlReparent, false) } func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, newPrimaryTablet *topodatapb.Tablet, lockAction, rp string, - tabletMap map[string]*topo.TabletInfo, reparentFunctions ReparentFunctions) error { + tabletMap map[string]*topo.TabletInfo, reparentFunctions ReparentFunctions, waitForAllReplicas bool) error { replCtx, replCancel := context.WithTimeout(ctx, reparentFunctions.GetWaitReplicasTimeout()) defer replCancel() @@ -343,7 +373,7 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent defer replWg.Done() logger.Infof("setting new master on replica %v", alias) - err := tmc.SetReplicationSource(replCtx, ti.Tablet, newPrimaryTablet.Alias, now, "", true) + err := tmc.SetReplicationSource(replCtx, ti.Tablet, newPrimaryTablet.Alias, 0, "", true) if err != nil { err = vterrors.Wrapf(err, "tablet %v SetReplicationSource failed: %v", alias, err) rec.RecordError(err) @@ -351,8 +381,13 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent return } + reparentFunctions.PostReplicationChangeHook(ti.Tablet) + // Signal that at least one goroutine succeeded to SetReplicationSource. - replSuccessCancel() + // We do this only when we do not want to wair for all the replicas + if !waitForAllReplicas { + replSuccessCancel() + } } numReplicas := 0 @@ -478,12 +513,13 @@ func ChooseNewPrimary( // PromotePrimaryCandidateAndStartReplication promotes the primary candidate that we have, but it does not set to start accepting writes func PromotePrimaryCandidateAndStartReplication(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, - lockAction, rp string, tabletMap map[string]*topo.TabletInfo, reparentFunctions ReparentFunctions) error { + lockAction, rp string, tabletMap map[string]*topo.TabletInfo, reparentFunctions ReparentFunctions, isIdeal bool) error { if err := promotePrimary(ctx, tmc, ts, logger, newPrimary); err != nil { return err } - if err := reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, rp, tabletMap, reparentFunctions); err != nil { + // if the promoted primary is not ideal then we wait for all the replicas so that we choose a better candidate from them later + if err := reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, rp, tabletMap, reparentFunctions, !isIdeal); err != nil { return err } @@ -512,6 +548,8 @@ func promotePrimary(ctx context.Context, tmc tmclient.TabletManagerClient, ts *t if err != nil { return err } + ti, err := ts.GetTablet(ctx, newPrimary.Alias) + newPrimary = ti.Tablet ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() _, err = ts.UpdateShardFields(ctx, newPrimary.Keyspace, newPrimary.Shard, func(si *topo.ShardInfo) error { From e46a4d01766ad3f8108c608e3a137e733c336d28 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 24 Aug 2021 18:47:48 +0530 Subject: [PATCH 033/176] fixed unit tests Signed-off-by: Manan Gupta --- go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go | 7 +++++++ go/vt/vtctl/reparentutil/emergency_reparenter_test.go | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go b/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go index 15c6705b835..01c83a0700a 100644 --- a/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go +++ b/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go @@ -435,6 +435,13 @@ func (fake *TabletManagerClient) SetMaster(ctx context.Context, tablet *topodata return assert.AnError } +func (fake *TabletManagerClient) StartReplication(ctx context.Context, tablet *topodatapb.Tablet) error { + if tablet == nil { + return assert.AnError + } + return nil +} + // SetReplicationSource is part of the tmclient.TabletManagerClient interface. func (fake *TabletManagerClient) SetReplicationSource(ctx context.Context, tablet *topodatapb.Tablet, parent *topodatapb.TabletAlias, timeCreatedNS int64, waitPosition string, forceStartReplication bool) error { if fake.SetMasterResults == nil { diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index c4e40de6804..17868b471ee 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -986,6 +986,12 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { logger := logutil.NewMemoryLogger() ev := &events.Reparent{} + for i, tablet := range tt.tablets { + tablet.Type = topodatapb.TabletType_REPLICA + tt.tablets[i] = tablet + } + tt.tmc.TopoServer = tt.vtctlReparentFunctions.ts + testutil.AddShards(ctx, t, tt.vtctlReparentFunctions.ts, tt.shards...) testutil.AddTablets(ctx, t, tt.vtctlReparentFunctions.ts, nil, tt.tablets...) From 4ee197d3c70ad252b32d7399dca94a329e2bb7b9 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 25 Aug 2021 14:26:54 +0530 Subject: [PATCH 034/176] create a function to get a better candidate Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 4 ++ .../reparentutil/emergency_reparenter.go | 22 ++++-- go/vt/vtctl/reparentutil/util.go | 72 +++++++++++++++---- 3 files changed, 78 insertions(+), 20 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index cd39ee9a00f..07b07cabbc6 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -383,6 +383,10 @@ func (vtOrcReparent *VtOrcReparentFunctions) PostReplicationChangeHook(tablet *t }) } +func (vtorcReparent *VtOrcReparentFunctions) GetBetterCandidate(newPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, primaryStatus map[string]*replicationdatapb.PrimaryStatus, tabletMap map[string]*topo.TabletInfo) *topodatapb.Tablet { + return newPrimary +} + // CheckIfNeedToOverridePrimary implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) CheckIfNeedToOverridePrimary() error { if vtorcReparent.promotedReplica != nil && !vtorcReparent.postponedAll { diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 401fa150fb8..a32acb045cb 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -19,6 +19,8 @@ package reparentutil import ( "context" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/proto/vtrpc" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -151,11 +153,8 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve isIdeal := reparentFunctions.PromotedReplicaIsIdeal(newPrimary, tabletMap, primaryStatusMap, validCandidates) // TODO := LockAction and RP - if err := PromotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, newPrimary, "", "", tabletMap, reparentFunctions, isIdeal); err != nil { - return err - } - - if err := reparentFunctions.CheckIfNeedToOverridePrimary(); err != nil { + validReplacementCandidates, err := PromotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, newPrimary, "", "", tabletMap, statusMap, reparentFunctions, isIdeal) + if err != nil { return err } @@ -164,6 +163,19 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } + betterCandidate := newPrimary + if !isIdeal { + betterCandidate = reparentFunctions.GetBetterCandidate(newPrimary, validReplacementCandidates, primaryStatusMap, tabletMap) + } + + if !topoproto.TabletAliasEqual(betterCandidate.Alias, newPrimary.Alias) { + + } + + if err := reparentFunctions.CheckIfNeedToOverridePrimary(); err != nil { + return err + } + if err := reparentFunctions.StartReplication(ctx, ev, erp.logger, erp.tmc); err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 30abd54ab8f..0a07b56f720 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -63,6 +63,7 @@ type ( FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) (*topodatapb.Tablet, error) PromotedReplicaIsIdeal(*topodatapb.Tablet, map[string]*topo.TabletInfo, map[string]*replicationdatapb.PrimaryStatus, map[string]mysql.Position) bool PostReplicationChangeHook(*topodatapb.Tablet) + GetBetterCandidate(*topodatapb.Tablet, []*topodatapb.Tablet, map[string]*replicationdatapb.PrimaryStatus, map[string]*topo.TabletInfo) *topodatapb.Tablet CheckIfNeedToOverridePrimary() error StartReplication(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) error GetNewPrimary() *topodatapb.Tablet @@ -229,6 +230,28 @@ func (vtctlReparent *VtctlReparentFunctions) PromotedReplicaIsIdeal(newPrimary * return false } +// GetBetterCandidate implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) GetBetterCandidate(newPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, primaryStatus map[string]*replicationdatapb.PrimaryStatus, tabletMap map[string]*topo.TabletInfo) *topodatapb.Tablet { + if len(primaryStatus) == 1 { + var prevPrimary *topo.TabletInfo + for tablet := range primaryStatus { + prevPrimary = tabletMap[tablet] + } + // find one which is of the correct type and matches the cell of the previous primary + for _, candidate := range validCandidates { + if (candidate.Type == topodatapb.TabletType_PRIMARY || candidate.Type == topodatapb.TabletType_REPLICA) && prevPrimary.Alias.Cell == candidate.Alias.Cell { + return candidate + } + } + } + for _, candidate := range validCandidates { + if candidate.Type == topodatapb.TabletType_PRIMARY || candidate.Type == topodatapb.TabletType_REPLICA { + return candidate + } + } + return newPrimary +} + // CheckIfNeedToOverridePrimary implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePrimary() error { return nil @@ -336,12 +359,16 @@ func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Conte return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } - return reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.lockAction, rp, vtctlReparent.tabletMap, vtctlReparent, false) + _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.lockAction, rp, vtctlReparent.tabletMap, vtctlReparent.statusMap, vtctlReparent, false) + return err } func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, newPrimaryTablet *topodatapb.Tablet, lockAction, rp string, - tabletMap map[string]*topo.TabletInfo, reparentFunctions ReparentFunctions, waitForAllReplicas bool) error { + tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions, waitForAllReplicas bool) ([]*topodatapb.Tablet, error) { + + var replicasStartedReplication []*topodatapb.Tablet + replCtx, replCancel := context.WithTimeout(ctx, reparentFunctions.GetWaitReplicasTimeout()) defer replCancel() @@ -373,7 +400,20 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent defer replWg.Done() logger.Infof("setting new master on replica %v", alias) - err := tmc.SetReplicationSource(replCtx, ti.Tablet, newPrimaryTablet.Alias, 0, "", true) + forceStart := false + if status, ok := statusMap[alias]; ok { + fs, err := ReplicaWasRunning(status) + if err != nil { + err = vterrors.Wrapf(err, "tablet %v could not determine StopReplicationStatus: %v", alias, err) + rec.RecordError(err) + + return + } + + forceStart = fs + } + + err := tmc.SetReplicationSource(replCtx, ti.Tablet, newPrimaryTablet.Alias, 0, "", forceStart) if err != nil { err = vterrors.Wrapf(err, "tablet %v SetReplicationSource failed: %v", alias, err) rec.RecordError(err) @@ -381,6 +421,7 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent return } + replicasStartedReplication = append(replicasStartedReplication, ti.Tablet) reparentFunctions.PostReplicationChangeHook(ti.Tablet) // Signal that at least one goroutine succeeded to SetReplicationSource. @@ -422,13 +463,13 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent logger.Warningf("master failed to PopulateReparentJournal") replCancel() - return vterrors.Wrapf(primaryErr, "failed to PopulateReparentJournal on master: %v", primaryErr) + return nil, vterrors.Wrapf(primaryErr, "failed to PopulateReparentJournal on master: %v", primaryErr) } select { case <-replSuccessCtx.Done(): // At least one replica was able to SetMaster successfully - return nil + return replicasStartedReplication, nil case <-allReplicasDoneCtx.Done(): // There are certain timing issues between replSuccessCtx.Done firing // and allReplicasDoneCtx.Done firing, so we check again if truly all @@ -441,11 +482,11 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent // Technically, rec.Errors should never be greater than numReplicas, // but it's better to err on the side of caution here, but also // we're going to be explicit that this is doubly unexpected. - return vterrors.Wrapf(rec.Error(), "received more errors (= %d) than replicas (= %d), which should be impossible: %v", errCount, numReplicas, rec.Error()) + return nil, vterrors.Wrapf(rec.Error(), "received more errors (= %d) than replicas (= %d), which should be impossible: %v", errCount, numReplicas, rec.Error()) case errCount == numReplicas: - return vterrors.Wrapf(rec.Error(), "%d replica(s) failed: %v", numReplicas, rec.Error()) + return nil, vterrors.Wrapf(rec.Error(), "%d replica(s) failed: %v", numReplicas, rec.Error()) default: - return nil + return replicasStartedReplication, nil } } } @@ -513,14 +554,15 @@ func ChooseNewPrimary( // PromotePrimaryCandidateAndStartReplication promotes the primary candidate that we have, but it does not set to start accepting writes func PromotePrimaryCandidateAndStartReplication(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, - lockAction, rp string, tabletMap map[string]*topo.TabletInfo, reparentFunctions ReparentFunctions, isIdeal bool) error { + lockAction, rp string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions, isIdeal bool) ([]*topodatapb.Tablet, error) { if err := promotePrimary(ctx, tmc, ts, logger, newPrimary); err != nil { - return err + return nil, err } // if the promoted primary is not ideal then we wait for all the replicas so that we choose a better candidate from them later - if err := reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, rp, tabletMap, reparentFunctions, !isIdeal); err != nil { - return err + replicasStartedReplication, err := reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, rp, tabletMap, statusMap, reparentFunctions, !isIdeal) + if err != nil { + return nil, err } // TODO := add as a postponed function @@ -530,14 +572,14 @@ func PromotePrimaryCandidateAndStartReplication(ctx context.Context, tmc tmclien // err = moveGTIDFunc() //} - err := tmc.StartReplication(ctx, newPrimary) + err = tmc.StartReplication(ctx, newPrimary) if err != nil { - return err + return nil, err } //log.Debugf("RegroupReplicasGTID: done") //inst.AuditOperation("regroup-replicas-gtid", masterKey, fmt.Sprintf("regrouped replicas of %+v via GTID; promoted %+v", *masterKey, candidateReplica.Key)) - return nil //unmovedReplicas, candidateReplica, err + return replicasStartedReplication, nil //unmovedReplicas, candidateReplica, err } // promotePrimary makes the new tablet the primary and proactively performs From cd4f548394a3dee0d6c1634dac76e3ff436c37b1 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 25 Aug 2021 16:53:21 +0530 Subject: [PATCH 035/176] choose candidate using validCandidate list Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 172 +++++++++++++++++- 1 file changed, 166 insertions(+), 6 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 07b07cabbc6..758c371e5a1 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -21,6 +21,8 @@ import ( "fmt" "time" + "vitess.io/vitess/go/vt/topo/topoproto" + replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" "vitess.io/vitess/go/vt/proto/vtrpc" @@ -384,17 +386,175 @@ func (vtOrcReparent *VtOrcReparentFunctions) PostReplicationChangeHook(tablet *t } func (vtorcReparent *VtOrcReparentFunctions) GetBetterCandidate(newPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, primaryStatus map[string]*replicationdatapb.PrimaryStatus, tabletMap map[string]*topo.TabletInfo) *topodatapb.Tablet { + if vtorcReparent.candidateInstanceKey != nil { + candidateTablet, _ := inst.ReadTablet(*vtorcReparent.candidateInstanceKey) + // return the requested candidate as long as it is valid + for _, validCandidate := range validCandidates { + if topoproto.TabletAliasEqual(validCandidate.Alias, candidateTablet.Alias) { + return validCandidate + } + } + } + var oldPrimary *topodatapb.Tablet + if len(primaryStatus) == 1 { + for tablet := range primaryStatus { + ti := tabletMap[tablet] + oldPrimary = ti.Tablet + } + } + replacementCandidate := getReplacementForPromotedReplica(vtorcReparent.topologyRecovery, newPrimary, oldPrimary, validCandidates) + vtorcReparent.promotedReplica = getInstanceFromTablet(replacementCandidate) + + return replacementCandidate +} + +func getReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, newPrimary, oldPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet) *topodatapb.Tablet { + var preferredCandidates []*topodatapb.Tablet + var neutralReplicas []*topodatapb.Tablet + for _, candidate := range validCandidates { + promotionRule := inst.PromotionRule(candidate) + if promotionRule == inst.MustPromoteRule || promotionRule == inst.PreferPromoteRule { + preferredCandidates = append(preferredCandidates, candidate) + } + if promotionRule == inst.NeutralPromoteRule { + neutralReplicas = append(neutralReplicas, candidate) + } + } + + // So we've already promoted a replica. + // However, can we improve on our choice? Are there any replicas marked with "is_candidate"? + // Maybe we actually promoted such a replica. Does that mean we should keep it? + // Maybe we promoted a "neutral", and some "prefer" server is available. + // Maybe we promoted a "prefer_not" + // Maybe we promoted a server in a different DC than the primary + // There's many options. We may wish to replace the server we promoted with a better one. + AuditTopologyRecovery(topologyRecovery, "checking if should replace promoted replica with a better candidate") + AuditTopologyRecovery(topologyRecovery, "+ checking if promoted replica is the ideal candidate") + if oldPrimary != nil { + for _, candidateReplica := range preferredCandidates { + if topoproto.TabletAliasEqual(newPrimary.Alias, candidateReplica.Alias) && + newPrimary.Alias.Cell == oldPrimary.Alias.Cell { + // Seems like we promoted a candidate in the same cell as dead IM! Ideal! We're happy! + AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("promoted replica %+v is the ideal candidate", newPrimary.Alias)) + return newPrimary + } + } + } + // We didn't pick the ideal candidate; let's see if we can replace with a candidate from same DC and ENV + + // Try a candidate replica that is in same DC & env as the dead instance + AuditTopologyRecovery(topologyRecovery, "+ searching for an ideal candidate") + if oldPrimary != nil { + for _, candidateReplica := range preferredCandidates { + if canTakeOverPromotedServerAsMaster(getInstanceFromTablet(candidateReplica), getInstanceFromTablet(newPrimary)) && + candidateReplica.Alias.Cell == oldPrimary.Alias.Cell { + // This would make a great candidate + AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("orchestrator picks %+v as candidate replacement, based on being in same cell as failed instance", candidateReplica.Alias)) + return candidateReplica + } + } + } + + // We cannot find a candidate in same DC and ENV as dead primary + AuditTopologyRecovery(topologyRecovery, "+ checking if promoted replica is an OK candidate") + for _, candidateReplica := range preferredCandidates { + if topoproto.TabletAliasEqual(newPrimary.Alias, candidateReplica.Alias) { + // Seems like we promoted a candidate replica (though not in same DC and ENV as dead primary) + if satisfied, reason := MasterFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(candidateReplica)); satisfied { + // Good enough. No further action required. + AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("promoted replica %+v is a good candidate", newPrimary.Alias)) + return newPrimary + } else { + AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("skipping %+v; %s", candidateReplica.Alias, reason)) + } + } + } + + // Still nothing? + // Try a candidate replica that is in same DC & env as the promoted replica (our promoted replica is not an "is_candidate") + AuditTopologyRecovery(topologyRecovery, "+ searching for a candidate") + for _, candidateReplica := range preferredCandidates { + if canTakeOverPromotedServerAsMaster(getInstanceFromTablet(candidateReplica), getInstanceFromTablet(newPrimary)) && + newPrimary.Alias.Cell == candidateReplica.Alias.Cell { + // OK, better than nothing + AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("no candidate was offered for %+v but orchestrator picks %+v as candidate replacement, based on being in same DC & env as promoted instance", newPrimary.Alias, candidateReplica.Alias)) + return candidateReplica + } + } + + // Still nothing? + // Try a candidate replica (our promoted replica is not an "is_candidate") + AuditTopologyRecovery(topologyRecovery, "+ searching for a candidate") + for _, candidateReplica := range preferredCandidates { + if canTakeOverPromotedServerAsMaster(getInstanceFromTablet(candidateReplica), getInstanceFromTablet(newPrimary)) { + if satisfied, reason := MasterFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(candidateReplica)); satisfied { + // OK, better than nothing + AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("no candidate was offered for %+v but orchestrator picks %+v as candidate replacement", newPrimary.Alias, candidateReplica.Alias)) + return candidateReplica + } else { + AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("skipping %+v; %s", candidateReplica.Alias, reason)) + } + } + } + + keepSearchingHint := "" + if satisfied, reason := MasterFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(newPrimary)); !satisfied { + keepSearchingHint = fmt.Sprintf("Will keep searching; %s", reason) + } else if inst.PromotionRule(newPrimary) == inst.PreferNotPromoteRule { + keepSearchingHint = fmt.Sprintf("Will keep searching because we have promoted a server with prefer_not rule: %+v", newPrimary.Alias) + } + if keepSearchingHint != "" { + AuditTopologyRecovery(topologyRecovery, keepSearchingHint) + // Still nothing? Then we didn't find a replica marked as "candidate". OK, further down the stream we have: + // find neutral instance in same dv&env as dead primary + if oldPrimary != nil { + AuditTopologyRecovery(topologyRecovery, "+ searching for a neutral server to replace promoted server, in same DC and env as dead master") + for _, neutralReplica := range neutralReplicas { + if canTakeOverPromotedServerAsMaster(getInstanceFromTablet(neutralReplica), getInstanceFromTablet(newPrimary)) && + oldPrimary.Alias.Cell == neutralReplica.Alias.Cell { + AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("no candidate was offered for %+v but orchestrator picks %+v as candidate replacement, based on being in same DC & env as dead master", newPrimary.Alias, neutralReplica.Alias)) + return neutralReplica + } + } + } + + // find neutral instance in same dv&env as promoted replica + AuditTopologyRecovery(topologyRecovery, "+ searching for a neutral server to replace promoted server, in same DC and env as promoted replica") + for _, neutralReplica := range neutralReplicas { + if canTakeOverPromotedServerAsMaster(getInstanceFromTablet(neutralReplica), getInstanceFromTablet(newPrimary)) && + newPrimary.Alias.Cell == neutralReplica.Alias.Cell { + AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("no candidate was offered for %+v but orchestrator picks %+v as candidate replacement, based on being in same DC & env as promoted instance", newPrimary.Alias, neutralReplica.Alias)) + return neutralReplica + } + } + + AuditTopologyRecovery(topologyRecovery, "+ searching for a neutral server to replace a prefer_not") + for _, neutralReplica := range neutralReplicas { + if canTakeOverPromotedServerAsMaster(getInstanceFromTablet(neutralReplica), getInstanceFromTablet(newPrimary)) { + if satisfied, reason := MasterFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(neutralReplica)); satisfied { + // OK, better than nothing + AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("no candidate was offered for %+v but orchestrator picks %+v as candidate replacement, based on promoted instance having prefer_not promotion rule", newPrimary.Alias, neutralReplica.Alias)) + return neutralReplica + } else { + AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("skipping %+v; %s", neutralReplica.Alias, reason)) + } + } + } + } + return newPrimary } +func getInstanceFromTablet(tablet *topodatapb.Tablet) *inst.Instance { + instance, _, _ := inst.ReadInstance(&inst.InstanceKey{ + Hostname: tablet.MysqlHostname, + Port: int(tablet.MysqlPort), + }) + return instance +} + // CheckIfNeedToOverridePrimary implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) CheckIfNeedToOverridePrimary() error { - if vtorcReparent.promotedReplica != nil && !vtorcReparent.postponedAll { - var err error - vtorcReparent.promotedReplica, err = replacePromotedReplicaWithCandidate(vtorcReparent.topologyRecovery, &vtorcReparent.analysisEntry.AnalyzedInstanceKey, vtorcReparent.promotedReplica, vtorcReparent.candidateInstanceKey) - vtorcReparent.topologyRecovery.AddError(err) - } - if vtorcReparent.promotedReplica == nil { err := TabletUndoDemoteMaster(vtorcReparent.analysisEntry.AnalyzedInstanceKey) AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: TabletUndoDemoteMaster: %v", err)) From 2b7b14242ac25173362276fc1caffce8de206636 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 25 Aug 2021 21:01:07 +0530 Subject: [PATCH 036/176] refactor code into a new file Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter.go | 2 +- .../vtctl/reparentutil/reparent_functions.go | 299 ++++++++++++++++++ go/vt/vtctl/reparentutil/util.go | 267 +--------------- 3 files changed, 302 insertions(+), 266 deletions(-) create mode 100644 go/vt/vtctl/reparentutil/reparent_functions.go diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index a32acb045cb..2b6b71f32ce 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -153,7 +153,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve isIdeal := reparentFunctions.PromotedReplicaIsIdeal(newPrimary, tabletMap, primaryStatusMap, validCandidates) // TODO := LockAction and RP - validReplacementCandidates, err := PromotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, newPrimary, "", "", tabletMap, statusMap, reparentFunctions, isIdeal) + validReplacementCandidates, err := promotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, newPrimary, "", "", tabletMap, statusMap, reparentFunctions, isIdeal) if err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go new file mode 100644 index 00000000000..e8cab458638 --- /dev/null +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -0,0 +1,299 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reparentutil + +import ( + "context" + "fmt" + "time" + + "vitess.io/vitess/go/event" + + "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/vterrors" + + "k8s.io/apimachinery/pkg/util/sets" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/vt/logutil" + replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topotools/events" + "vitess.io/vitess/go/vt/vttablet/tmclient" +) + +type ( + // ReparentFunctions is an interface which has all the functions implementation required for re-parenting + ReparentFunctions interface { + LockShard(context.Context) (context.Context, func(*error), error) + GetTopoServer() *topo.Server + GetKeyspace() string + GetShard() string + CheckIfFixed() bool + PreRecoveryProcesses(context.Context) error + GetWaitReplicasTimeout() time.Duration + GetIgnoreReplicas() sets.String + CheckPrimaryRecoveryType() error + RestrictValidCandidates(map[string]mysql.Position, map[string]*topo.TabletInfo) (map[string]mysql.Position, error) + FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) (*topodatapb.Tablet, error) + PromotedReplicaIsIdeal(*topodatapb.Tablet, map[string]*topo.TabletInfo, map[string]*replicationdatapb.PrimaryStatus, map[string]mysql.Position) bool + PostReplicationChangeHook(*topodatapb.Tablet) + GetBetterCandidate(*topodatapb.Tablet, []*topodatapb.Tablet, map[string]*replicationdatapb.PrimaryStatus, map[string]*topo.TabletInfo) *topodatapb.Tablet + CheckIfNeedToOverridePrimary() error + StartReplication(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) error + GetNewPrimary() *topodatapb.Tablet + + // TODO: remove this + SetMaps(map[string]*topo.TabletInfo, map[string]*replicationdatapb.StopReplicationStatus, map[string]*replicationdatapb.PrimaryStatus) + } + + // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions + VtctlReparentFunctions struct { + NewPrimaryAlias *topodatapb.TabletAlias + IgnoreReplicas sets.String + WaitReplicasTimeout time.Duration + keyspace string + shard string + ts *topo.Server + lockAction string + tabletMap map[string]*topo.TabletInfo + statusMap map[string]*replicationdatapb.StopReplicationStatus + primaryStatusMap map[string]*replicationdatapb.PrimaryStatus + validCandidates map[string]mysql.Position + winningPosition mysql.Position + winningPrimaryTabletAliasStr string + } +) + +var ( + _ ReparentFunctions = (*VtctlReparentFunctions)(nil) +) + +// NewVtctlReparentFunctions creates a new VtctlReparentFunctions which is used in ERS ans PRS +func NewVtctlReparentFunctions(newPrimaryAlias *topodatapb.TabletAlias, ignoreReplicas sets.String, waitReplicasTimeout time.Duration, keyspace string, shard string, ts *topo.Server) *VtctlReparentFunctions { + return &VtctlReparentFunctions{ + NewPrimaryAlias: newPrimaryAlias, + IgnoreReplicas: ignoreReplicas, + WaitReplicasTimeout: waitReplicasTimeout, + keyspace: keyspace, + shard: shard, + ts: ts, + } +} + +// LockShard implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) LockShard(ctx context.Context) (context.Context, func(*error), error) { + vtctlReparent.lockAction = vtctlReparent.getLockAction(vtctlReparent.NewPrimaryAlias) + + return vtctlReparent.ts.LockShard(ctx, vtctlReparent.keyspace, vtctlReparent.shard, vtctlReparent.lockAction) +} + +// GetTopoServer implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) GetTopoServer() *topo.Server { + return vtctlReparent.ts +} + +// GetKeyspace implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) GetKeyspace() string { + return vtctlReparent.keyspace +} + +// GetShard implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) GetShard() string { + return vtctlReparent.shard +} + +// CheckIfFixed implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) CheckIfFixed() bool { + return false +} + +// PreRecoveryProcesses implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) PreRecoveryProcesses(ctx context.Context) error { + return nil +} + +func (vtctlReparentFunctions *VtctlReparentFunctions) GetWaitReplicasTimeout() time.Duration { + return vtctlReparentFunctions.WaitReplicasTimeout +} + +func (vtctlReparentFunctions *VtctlReparentFunctions) GetIgnoreReplicas() sets.String { + return vtctlReparentFunctions.IgnoreReplicas +} + +// CheckPrimaryRecoveryType implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) CheckPrimaryRecoveryType() error { + return nil +} + +// RestrictValidCandidates implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) RestrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { + restrictedValidCandidates := make(map[string]mysql.Position) + + for candidate, position := range validCandidates { + candidateInfo, ok := tabletMap[candidate] + if !ok { + return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) + } + + if candidateInfo.Type != topodatapb.TabletType_PRIMARY && candidateInfo.Type != topodatapb.TabletType_REPLICA { + continue + } + + restrictedValidCandidates[candidate] = position + } + + return restrictedValidCandidates, nil +} + +// FindPrimaryCandidates implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, error) { + // Elect the candidate with the most up-to-date position. + for alias, position := range validCandidates { + if vtctlReparent.winningPosition.IsZero() || position.AtLeast(vtctlReparent.winningPosition) { + vtctlReparent.winningPosition = position + vtctlReparent.winningPrimaryTabletAliasStr = alias + } + } + + vtctlReparent.validCandidates = validCandidates + + // If we were requested to elect a particular primary, verify it's a valid + // candidate (non-zero position, no errant GTIDs) and is at least as + // advanced as the winning position. + if vtctlReparent.NewPrimaryAlias != nil { + vtctlReparent.winningPrimaryTabletAliasStr = topoproto.TabletAliasString(vtctlReparent.NewPrimaryAlias) + pos, ok := vtctlReparent.validCandidates[vtctlReparent.winningPrimaryTabletAliasStr] + switch { + case !ok: + return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v has errant GTIDs", vtctlReparent.winningPrimaryTabletAliasStr) + case !pos.AtLeast(vtctlReparent.winningPosition): + return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v at position %v is not fully caught up. Winning position: %v", vtctlReparent.winningPrimaryTabletAliasStr, pos, vtctlReparent.winningPosition) + } + } + + // TODO:= handle not found error + newPrimaryAlias := tabletMap[vtctlReparent.winningPrimaryTabletAliasStr] + return newPrimaryAlias.Tablet, nil +} + +// PostReplicationChangeHook implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) PostReplicationChangeHook(*topodatapb.Tablet) { + return +} + +// PromotedReplicaIsIdeal implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) PromotedReplicaIsIdeal(newPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, primaryStatus map[string]*replicationdatapb.PrimaryStatus, validCandidates map[string]mysql.Position) bool { + if vtctlReparent.NewPrimaryAlias != nil { + //explicit request to promote a specific tablet + return true + } + if len(primaryStatus) == 1 { + var prevPrimary *topo.TabletInfo + for tablet := range primaryStatus { + prevPrimary = tabletMap[tablet] + } + if (newPrimary.Type == topodatapb.TabletType_PRIMARY || newPrimary.Type == topodatapb.TabletType_REPLICA) && newPrimary.Alias.Cell == prevPrimary.Alias.Cell { + return true + } + return false + } + + if newPrimary.Type == topodatapb.TabletType_PRIMARY || newPrimary.Type == topodatapb.TabletType_REPLICA { + return true + } + return false +} + +// GetBetterCandidate implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) GetBetterCandidate(newPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, primaryStatus map[string]*replicationdatapb.PrimaryStatus, tabletMap map[string]*topo.TabletInfo) *topodatapb.Tablet { + if len(primaryStatus) == 1 { + var prevPrimary *topo.TabletInfo + for tablet := range primaryStatus { + prevPrimary = tabletMap[tablet] + } + // find one which is of the correct type and matches the cell of the previous primary + for _, candidate := range validCandidates { + if (candidate.Type == topodatapb.TabletType_PRIMARY || candidate.Type == topodatapb.TabletType_REPLICA) && prevPrimary.Alias.Cell == candidate.Alias.Cell { + return candidate + } + } + } + for _, candidate := range validCandidates { + if candidate.Type == topodatapb.TabletType_PRIMARY || candidate.Type == topodatapb.TabletType_REPLICA { + return candidate + } + } + return newPrimary +} + +// CheckIfNeedToOverridePrimary implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePrimary() error { + return nil +} + +// StartReplication implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) StartReplication(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) error { + // Do the promotion. + //return vtctlReparent.promoteNewPrimary(ctx, ev, logger, tmc) + return nil +} + +// GetNewPrimary implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) GetNewPrimary() *topodatapb.Tablet { + return vtctlReparent.tabletMap[vtctlReparent.winningPrimaryTabletAliasStr].Tablet +} + +func (vtctlReparent *VtctlReparentFunctions) SetMaps(tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, primaryStatusMap map[string]*replicationdatapb.PrimaryStatus) { + vtctlReparent.tabletMap = tabletMap + vtctlReparent.statusMap = statusMap + vtctlReparent.primaryStatusMap = primaryStatusMap +} + +func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { + action := "EmergencyReparentShard" + + if newPrimaryAlias != nil { + action += fmt.Sprintf("(%v)", topoproto.TabletAliasString(newPrimaryAlias)) + } + + return action +} + +func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) error { + logger.Infof("promoting tablet %v to master", vtctlReparent.winningPrimaryTabletAliasStr) + event.DispatchUpdate(ev, "promoting replica") + + newPrimaryTabletInfo, ok := vtctlReparent.tabletMap[vtctlReparent.winningPrimaryTabletAliasStr] + if !ok { + return vterrors.Errorf(vtrpc.Code_INTERNAL, "attempted to promote master-elect %v that was not in the tablet map; this an impossible situation", vtctlReparent.winningPrimaryTabletAliasStr) + } + + rp, err := tmc.PromoteReplica(ctx, newPrimaryTabletInfo.Tablet) + if err != nil { + return vterrors.Wrapf(err, "master-elect tablet %v failed to be upgraded to master: %v", vtctlReparent.winningPrimaryTabletAliasStr, err) + } + + if err := topo.CheckShardLocked(ctx, vtctlReparent.keyspace, vtctlReparent.shard); err != nil { + return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) + } + + _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.lockAction, rp, vtctlReparent.tabletMap, vtctlReparent.statusMap, vtctlReparent, false) + return err +} diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 0a07b56f720..30792417454 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -18,7 +18,6 @@ package reparentutil import ( "context" - "fmt" "sync" "time" @@ -34,8 +33,6 @@ import ( "vitess.io/vitess/go/vt/topotools/events" - "k8s.io/apimachinery/pkg/util/sets" - "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" @@ -47,244 +44,6 @@ import ( "vitess.io/vitess/go/vt/proto/vtrpc" ) -type ( - // ReparentFunctions is an interface which has all the functions implementation required for re-parenting - ReparentFunctions interface { - LockShard(context.Context) (context.Context, func(*error), error) - GetTopoServer() *topo.Server - GetKeyspace() string - GetShard() string - CheckIfFixed() bool - PreRecoveryProcesses(context.Context) error - GetWaitReplicasTimeout() time.Duration - GetIgnoreReplicas() sets.String - CheckPrimaryRecoveryType() error - RestrictValidCandidates(map[string]mysql.Position, map[string]*topo.TabletInfo) (map[string]mysql.Position, error) - FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) (*topodatapb.Tablet, error) - PromotedReplicaIsIdeal(*topodatapb.Tablet, map[string]*topo.TabletInfo, map[string]*replicationdatapb.PrimaryStatus, map[string]mysql.Position) bool - PostReplicationChangeHook(*topodatapb.Tablet) - GetBetterCandidate(*topodatapb.Tablet, []*topodatapb.Tablet, map[string]*replicationdatapb.PrimaryStatus, map[string]*topo.TabletInfo) *topodatapb.Tablet - CheckIfNeedToOverridePrimary() error - StartReplication(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) error - GetNewPrimary() *topodatapb.Tablet - - // TODO: remove this - SetMaps(map[string]*topo.TabletInfo, map[string]*replicationdatapb.StopReplicationStatus, map[string]*replicationdatapb.PrimaryStatus) - } - - // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions - VtctlReparentFunctions struct { - NewPrimaryAlias *topodatapb.TabletAlias - IgnoreReplicas sets.String - WaitReplicasTimeout time.Duration - keyspace string - shard string - ts *topo.Server - lockAction string - tabletMap map[string]*topo.TabletInfo - statusMap map[string]*replicationdatapb.StopReplicationStatus - primaryStatusMap map[string]*replicationdatapb.PrimaryStatus - validCandidates map[string]mysql.Position - winningPosition mysql.Position - winningPrimaryTabletAliasStr string - } -) - -var ( - _ ReparentFunctions = (*VtctlReparentFunctions)(nil) -) - -// NewVtctlReparentFunctions creates a new VtctlReparentFunctions which is used in ERS ans PRS -func NewVtctlReparentFunctions(newPrimaryAlias *topodatapb.TabletAlias, ignoreReplicas sets.String, waitReplicasTimeout time.Duration, keyspace string, shard string, ts *topo.Server) *VtctlReparentFunctions { - return &VtctlReparentFunctions{ - NewPrimaryAlias: newPrimaryAlias, - IgnoreReplicas: ignoreReplicas, - WaitReplicasTimeout: waitReplicasTimeout, - keyspace: keyspace, - shard: shard, - ts: ts, - } -} - -// LockShard implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) LockShard(ctx context.Context) (context.Context, func(*error), error) { - vtctlReparent.lockAction = vtctlReparent.getLockAction(vtctlReparent.NewPrimaryAlias) - - return vtctlReparent.ts.LockShard(ctx, vtctlReparent.keyspace, vtctlReparent.shard, vtctlReparent.lockAction) -} - -// GetTopoServer implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) GetTopoServer() *topo.Server { - return vtctlReparent.ts -} - -// GetKeyspace implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) GetKeyspace() string { - return vtctlReparent.keyspace -} - -// GetShard implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) GetShard() string { - return vtctlReparent.shard -} - -// CheckIfFixed implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) CheckIfFixed() bool { - return false -} - -// PreRecoveryProcesses implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) PreRecoveryProcesses(ctx context.Context) error { - return nil -} - -func (vtctlReparentFunctions *VtctlReparentFunctions) GetWaitReplicasTimeout() time.Duration { - return vtctlReparentFunctions.WaitReplicasTimeout -} - -func (vtctlReparentFunctions *VtctlReparentFunctions) GetIgnoreReplicas() sets.String { - return vtctlReparentFunctions.IgnoreReplicas -} - -// CheckPrimaryRecoveryType implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) CheckPrimaryRecoveryType() error { - return nil -} - -// RestrictValidCandidates implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) RestrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { - restrictedValidCandidates := make(map[string]mysql.Position) - - for candidate, position := range validCandidates { - candidateInfo, ok := tabletMap[candidate] - if !ok { - return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) - } - - if candidateInfo.Type != topodatapb.TabletType_PRIMARY && candidateInfo.Type != topodatapb.TabletType_REPLICA { - continue - } - - restrictedValidCandidates[candidate] = position - } - - return restrictedValidCandidates, nil -} - -// FindPrimaryCandidates implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, error) { - // Elect the candidate with the most up-to-date position. - for alias, position := range validCandidates { - if vtctlReparent.winningPosition.IsZero() || position.AtLeast(vtctlReparent.winningPosition) { - vtctlReparent.winningPosition = position - vtctlReparent.winningPrimaryTabletAliasStr = alias - } - } - - vtctlReparent.validCandidates = validCandidates - - // If we were requested to elect a particular primary, verify it's a valid - // candidate (non-zero position, no errant GTIDs) and is at least as - // advanced as the winning position. - if vtctlReparent.NewPrimaryAlias != nil { - vtctlReparent.winningPrimaryTabletAliasStr = topoproto.TabletAliasString(vtctlReparent.NewPrimaryAlias) - pos, ok := vtctlReparent.validCandidates[vtctlReparent.winningPrimaryTabletAliasStr] - switch { - case !ok: - return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v has errant GTIDs", vtctlReparent.winningPrimaryTabletAliasStr) - case !pos.AtLeast(vtctlReparent.winningPosition): - return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v at position %v is not fully caught up. Winning position: %v", vtctlReparent.winningPrimaryTabletAliasStr, pos, vtctlReparent.winningPosition) - } - } - - // TODO:= handle not found error - newPrimaryAlias := tabletMap[vtctlReparent.winningPrimaryTabletAliasStr] - return newPrimaryAlias.Tablet, nil -} - -// PostReplicationChangeHook implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) PostReplicationChangeHook(*topodatapb.Tablet) { - return -} - -// PromotedReplicaIsIdeal implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) PromotedReplicaIsIdeal(newPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, primaryStatus map[string]*replicationdatapb.PrimaryStatus, validCandidates map[string]mysql.Position) bool { - if vtctlReparent.NewPrimaryAlias != nil { - //explicit request to promote a specific tablet - return true - } - if len(primaryStatus) == 1 { - var prevPrimary *topo.TabletInfo - for tablet := range primaryStatus { - prevPrimary = tabletMap[tablet] - } - if (newPrimary.Type == topodatapb.TabletType_PRIMARY || newPrimary.Type == topodatapb.TabletType_REPLICA) && newPrimary.Alias.Cell == prevPrimary.Alias.Cell { - return true - } - return false - } - - if newPrimary.Type == topodatapb.TabletType_PRIMARY || newPrimary.Type == topodatapb.TabletType_REPLICA { - return true - } - return false -} - -// GetBetterCandidate implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) GetBetterCandidate(newPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, primaryStatus map[string]*replicationdatapb.PrimaryStatus, tabletMap map[string]*topo.TabletInfo) *topodatapb.Tablet { - if len(primaryStatus) == 1 { - var prevPrimary *topo.TabletInfo - for tablet := range primaryStatus { - prevPrimary = tabletMap[tablet] - } - // find one which is of the correct type and matches the cell of the previous primary - for _, candidate := range validCandidates { - if (candidate.Type == topodatapb.TabletType_PRIMARY || candidate.Type == topodatapb.TabletType_REPLICA) && prevPrimary.Alias.Cell == candidate.Alias.Cell { - return candidate - } - } - } - for _, candidate := range validCandidates { - if candidate.Type == topodatapb.TabletType_PRIMARY || candidate.Type == topodatapb.TabletType_REPLICA { - return candidate - } - } - return newPrimary -} - -// CheckIfNeedToOverridePrimary implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePrimary() error { - return nil -} - -// StartReplication implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) StartReplication(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) error { - // Do the promotion. - //return vtctlReparent.promoteNewPrimary(ctx, ev, logger, tmc) - return nil -} - -// GetNewPrimary implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) GetNewPrimary() *topodatapb.Tablet { - return vtctlReparent.tabletMap[vtctlReparent.winningPrimaryTabletAliasStr].Tablet -} - -func (vtctlReparent *VtctlReparentFunctions) SetMaps(tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, primaryStatusMap map[string]*replicationdatapb.PrimaryStatus) { - vtctlReparent.tabletMap = tabletMap - vtctlReparent.statusMap = statusMap - vtctlReparent.primaryStatusMap = primaryStatusMap -} - -func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { - action := "EmergencyReparentShard" - - if newPrimaryAlias != nil { - action += fmt.Sprintf("(%v)", topoproto.TabletAliasString(newPrimaryAlias)) - } - - return action -} - func waitForAllRelayLogsToApply(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, waitReplicasTimeout time.Duration) error { errCh := make(chan error) defer close(errCh) @@ -341,28 +100,6 @@ func waitForAllRelayLogsToApply(ctx context.Context, logger logutil.Logger, tmc return nil } -func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) error { - logger.Infof("promoting tablet %v to master", vtctlReparent.winningPrimaryTabletAliasStr) - event.DispatchUpdate(ev, "promoting replica") - - newPrimaryTabletInfo, ok := vtctlReparent.tabletMap[vtctlReparent.winningPrimaryTabletAliasStr] - if !ok { - return vterrors.Errorf(vtrpc.Code_INTERNAL, "attempted to promote master-elect %v that was not in the tablet map; this an impossible situation", vtctlReparent.winningPrimaryTabletAliasStr) - } - - rp, err := tmc.PromoteReplica(ctx, newPrimaryTabletInfo.Tablet) - if err != nil { - return vterrors.Wrapf(err, "master-elect tablet %v failed to be upgraded to master: %v", vtctlReparent.winningPrimaryTabletAliasStr, err) - } - - if err := topo.CheckShardLocked(ctx, vtctlReparent.keyspace, vtctlReparent.shard); err != nil { - return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) - } - - _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.lockAction, rp, vtctlReparent.tabletMap, vtctlReparent.statusMap, vtctlReparent, false) - return err -} - func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, newPrimaryTablet *topodatapb.Tablet, lockAction, rp string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions, waitForAllReplicas bool) ([]*topodatapb.Tablet, error) { @@ -552,8 +289,8 @@ func ChooseNewPrimary( return nil, nil } -// PromotePrimaryCandidateAndStartReplication promotes the primary candidate that we have, but it does not set to start accepting writes -func PromotePrimaryCandidateAndStartReplication(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, +// promotePrimaryCandidateAndStartReplication promotes the primary candidate that we have, but it does not set to start accepting writes +func promotePrimaryCandidateAndStartReplication(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, lockAction, rp string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions, isIdeal bool) ([]*topodatapb.Tablet, error) { if err := promotePrimary(ctx, tmc, ts, logger, newPrimary); err != nil { return nil, err From a4881e67dd6fe63dc15611f9e794ac3dbc72bf52 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 26 Aug 2021 10:28:20 +0530 Subject: [PATCH 037/176] added function to replace primary with a better candidate Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter.go | 5 ++- go/vt/vtctl/reparentutil/util.go | 36 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 2b6b71f32ce..3d2b390a4c6 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -169,7 +169,10 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } if !topoproto.TabletAliasEqual(betterCandidate.Alias, newPrimary.Alias) { - + err = replaceWithBetterCandidate(ctx, erp.tmc, ts, ev, erp.logger, newPrimary, betterCandidate, "", "", tabletMap, statusMap, reparentFunctions) + if err != nil { + return err + } } if err := reparentFunctions.CheckIfNeedToOverridePrimary(); err != nil { diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 30792417454..344a0b92a5e 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -408,3 +408,39 @@ func FindCurrentPrimary(tabletMap map[string]*topo.TabletInfo, logger logutil.Lo return currentPrimary } + +// replaceWithBetterCandidate promotes the newer candidate over the primary candidate that we have, but it does not set to start accepting writes +func replaceWithBetterCandidate(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, prevPrimary, newPrimary *topodatapb.Tablet, + lockAction, rp string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions) error { + + pos, err := tmc.PrimaryPosition(ctx, prevPrimary) + if err != nil { + return err + } + + _, err = tmc.StopReplicationMinimum(ctx, newPrimary, pos, reparentFunctions.GetWaitReplicasTimeout()) + if err != nil { + return err + } + + if err := promotePrimary(ctx, tmc, ts, logger, newPrimary); err != nil { + return err + } + + // if the promoted primary is not ideal then we wait for all the replicas so that we choose a better candidate from them later + _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, rp, tabletMap, statusMap, reparentFunctions, false) + if err != nil { + return err + } + + // TODO := add as a postponed function + //if postponedFunctionsContainer != nil && postponeAllMatchOperations != nil && postponeAllMatchOperations(candidateReplica, hasBestPromotionRule) { + // postponedFunctionsContainer.AddPostponedFunction(moveGTIDFunc, fmt.Sprintf("regroup-replicas-gtid %+v", candidateReplica.Key)) + //} else { + // err = moveGTIDFunc() + //} + + //log.Debugf("RegroupReplicasGTID: done") + //inst.AuditOperation("regroup-replicas-gtid", masterKey, fmt.Sprintf("regrouped replicas of %+v via GTID; promoted %+v", *masterKey, candidateReplica.Key)) + return nil //unmovedReplicas, candidateReplica, err +} From d7d52f0696420fe65a5c650e1bf7de04731dc4ba Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 26 Aug 2021 15:14:08 +0530 Subject: [PATCH 038/176] fix test timeouts Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/main_test.go | 38 ++++++++++--------- .../vtorc/test_config_crosscenter_prefer.json | 1 + ...est_config_crosscenter_prefer_prevent.json | 1 + 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/go/test/endtoend/vtorc/main_test.go b/go/test/endtoend/vtorc/main_test.go index 63f7f73f2c1..e4b4d6c9025 100644 --- a/go/test/endtoend/vtorc/main_test.go +++ b/go/test/endtoend/vtorc/main_test.go @@ -405,24 +405,28 @@ func checkPrimaryTablet(t *testing.T, cluster *cluster.LocalProcessCluster, tabl log.Warningf("Tablet %v is not primary yet, sleep for 1 second\n", tablet.Alias) time.Sleep(time.Second) continue - } else { - // allow time for tablet state to be updated after topo is updated - time.Sleep(2 * time.Second) - // make sure the health stream is updated - result, err = cluster.VtctlclientProcess.ExecuteCommandWithOutput("VtTabletStreamHealth", "-count", "1", tablet.Alias) - require.NoError(t, err) - var streamHealthResponse querypb.StreamHealthResponse - - err = json2.Unmarshal([]byte(result), &streamHealthResponse) - require.NoError(t, err) - //if !streamHealthResponse.GetServing() { - // log.Exitf("stream health not updated") - //} - assert.True(t, streamHealthResponse.GetServing(), "stream health: %v", &streamHealthResponse) - tabletType := streamHealthResponse.GetTarget().GetTabletType() - require.Equal(t, topodatapb.TabletType_PRIMARY, tabletType) - break + } // make sure the health stream is updated + result, err = cluster.VtctlclientProcess.ExecuteCommandWithOutput("VtTabletStreamHealth", "-count", "1", tablet.Alias) + require.NoError(t, err) + var streamHealthResponse querypb.StreamHealthResponse + + err = json2.Unmarshal([]byte(result), &streamHealthResponse) + require.NoError(t, err) + //if !streamHealthResponse.GetServing() { + // log.Exitf("stream health not updated") + //} + if !streamHealthResponse.GetServing() { + log.Warningf("Tablet %v is not serving in health stream yet, sleep for 1 second\n", tablet.Alias) + time.Sleep(time.Second) + continue + } + tabletType := streamHealthResponse.GetTarget().GetTabletType() + if tabletType != topodatapb.TabletType_PRIMARY { + log.Warningf("Tablet %v is not primary in health stream yet, sleep for 1 second\n", tablet.Alias) + time.Sleep(time.Second) + continue } + break } } diff --git a/go/test/endtoend/vtorc/test_config_crosscenter_prefer.json b/go/test/endtoend/vtorc/test_config_crosscenter_prefer.json index 7afd417ffcb..0e4e86e7922 100644 --- a/go/test/endtoend/vtorc/test_config_crosscenter_prefer.json +++ b/go/test/endtoend/vtorc/test_config_crosscenter_prefer.json @@ -6,6 +6,7 @@ "MySQLReplicaPassword": "", "RecoveryPeriodBlockSeconds": 1, "InstancePollSeconds": 1, + "LockShardTimeoutSeconds": 45, "Durability": "specified", "DurabilityParams": { "zone2-0000000200": "prefer" diff --git a/go/test/endtoend/vtorc/test_config_crosscenter_prefer_prevent.json b/go/test/endtoend/vtorc/test_config_crosscenter_prefer_prevent.json index 794a36f8a22..2263dabe8f2 100644 --- a/go/test/endtoend/vtorc/test_config_crosscenter_prefer_prevent.json +++ b/go/test/endtoend/vtorc/test_config_crosscenter_prefer_prevent.json @@ -6,6 +6,7 @@ "MySQLReplicaPassword": "", "RecoveryPeriodBlockSeconds": 1, "InstancePollSeconds": 1, + "LockShardTimeoutSeconds": 45, "Durability": "specified", "DurabilityParams": { "zone2-0000000200": "prefer" From eecc87e15d9e6d9da0137dc6e983b5ab263a8ae3 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 26 Aug 2021 15:31:35 +0530 Subject: [PATCH 039/176] potential bug fix Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/reparent_functions.go | 2 +- go/vt/vtctl/reparentutil/reparent_functions.go | 4 ++-- go/vt/vtctl/reparentutil/util.go | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 758c371e5a1..5cdd1981077 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -378,7 +378,7 @@ func (vtOrcReparent *VtOrcReparentFunctions) PromotedReplicaIsIdeal(newPrimary * } // PostReplicationChangeHook implements the ReparentFunctions interface -func (vtOrcReparent *VtOrcReparentFunctions) PostReplicationChangeHook(tablet *topodatapb.Tablet) { +func (vtOrcReparent *VtOrcReparentFunctions) PostTabletChangeHook(tablet *topodatapb.Tablet) { inst.ReadTopologyInstance(&inst.InstanceKey{ Hostname: tablet.MysqlHostname, Port: int(tablet.MysqlPort), diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index e8cab458638..e2e01eaf41c 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -53,7 +53,7 @@ type ( RestrictValidCandidates(map[string]mysql.Position, map[string]*topo.TabletInfo) (map[string]mysql.Position, error) FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) (*topodatapb.Tablet, error) PromotedReplicaIsIdeal(*topodatapb.Tablet, map[string]*topo.TabletInfo, map[string]*replicationdatapb.PrimaryStatus, map[string]mysql.Position) bool - PostReplicationChangeHook(*topodatapb.Tablet) + PostTabletChangeHook(*topodatapb.Tablet) GetBetterCandidate(*topodatapb.Tablet, []*topodatapb.Tablet, map[string]*replicationdatapb.PrimaryStatus, map[string]*topo.TabletInfo) *topodatapb.Tablet CheckIfNeedToOverridePrimary() error StartReplication(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) error @@ -194,7 +194,7 @@ func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.C } // PostReplicationChangeHook implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) PostReplicationChangeHook(*topodatapb.Tablet) { +func (vtctlReparent *VtctlReparentFunctions) PostTabletChangeHook(*topodatapb.Tablet) { return } diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 344a0b92a5e..1e562b0b789 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -159,7 +159,7 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent } replicasStartedReplication = append(replicasStartedReplication, ti.Tablet) - reparentFunctions.PostReplicationChangeHook(ti.Tablet) + reparentFunctions.PostTabletChangeHook(ti.Tablet) // Signal that at least one goroutine succeeded to SetReplicationSource. // We do this only when we do not want to wair for all the replicas @@ -296,6 +296,8 @@ func promotePrimaryCandidateAndStartReplication(ctx context.Context, tmc tmclien return nil, err } + reparentFunctions.PostTabletChangeHook(newPrimary) + // if the promoted primary is not ideal then we wait for all the replicas so that we choose a better candidate from them later replicasStartedReplication, err := reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, rp, tabletMap, statusMap, reparentFunctions, !isIdeal) if err != nil { @@ -427,6 +429,8 @@ func replaceWithBetterCandidate(ctx context.Context, tmc tmclient.TabletManagerC return err } + reparentFunctions.PostTabletChangeHook(newPrimary) + // if the promoted primary is not ideal then we wait for all the replicas so that we choose a better candidate from them later _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, rp, tabletMap, statusMap, reparentFunctions, false) if err != nil { From cc99d26f1041eef34c51178e592537e882224386 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 26 Aug 2021 15:46:15 +0530 Subject: [PATCH 040/176] refactor override promotion method Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 58 +++++++++---------- .../reparentutil/emergency_reparenter.go | 5 +- .../vtctl/reparentutil/reparent_functions.go | 4 +- 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 5cdd1981077..d192a54eecc 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -223,7 +223,7 @@ func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.C AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: %d postponed functions", vtorcReparent.topologyRecovery.PostponedFunctionsContainer.Len())) vtorcReparent.promotedReplica = promotedReplica - vtorcReparent.lostReplicas = lostReplicas + vtorcReparent.topologyRecovery.LostReplicas.AddInstances(lostReplicas) vtorcReparent.recoveryAttempted = true return newPrimary, nil } @@ -554,51 +554,49 @@ func getInstanceFromTablet(tablet *topodatapb.Tablet) *inst.Instance { } // CheckIfNeedToOverridePrimary implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) CheckIfNeedToOverridePrimary() error { - if vtorcReparent.promotedReplica == nil { - err := TabletUndoDemoteMaster(vtorcReparent.analysisEntry.AnalyzedInstanceKey) - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: TabletUndoDemoteMaster: %v", err)) - message := "Failure: no replica promoted." - AuditTopologyRecovery(vtorcReparent.topologyRecovery, message) - inst.AuditOperation("recover-dead-master", &vtorcReparent.analysisEntry.AnalyzedInstanceKey, message) - return err - } +func (vtorcReparent *VtOrcReparentFunctions) CheckIfNeedToOverridePromotion(newPrimary *topodatapb.Tablet) error { + // TODO : use fixing code outside + //if vtorcReparent.promotedReplica == nil { + // err := TabletUndoDemoteMaster(vtorcReparent.analysisEntry.AnalyzedInstanceKey) + // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: TabletUndoDemoteMaster: %v", err)) + // message := "Failure: no replica promoted." + // AuditTopologyRecovery(vtorcReparent.topologyRecovery, message) + // inst.AuditOperation("recover-dead-master", &vtorcReparent.analysisEntry.AnalyzedInstanceKey, message) + // return err + //} + // TODO: Move out to post-change code message := fmt.Sprintf("promoted replica: %+v", vtorcReparent.promotedReplica.Key) AuditTopologyRecovery(vtorcReparent.topologyRecovery, message) inst.AuditOperation("recover-dead-master", &vtorcReparent.analysisEntry.AnalyzedInstanceKey, message) - vtorcReparent.topologyRecovery.LostReplicas.AddInstances(vtorcReparent.lostReplicas) - var err error - overrideMasterPromotion := func() (*inst.Instance, error) { - if vtorcReparent.promotedReplica == nil { - // No promotion; nothing to override. - return vtorcReparent.promotedReplica, err - } + newPrimaryInstance := getInstanceFromTablet(newPrimary) + overrideMasterPromotion := func() error { // Scenarios where we might cancel the promotion. - if satisfied, reason := MasterFailoverGeographicConstraintSatisfied(&vtorcReparent.analysisEntry, vtorcReparent.promotedReplica); !satisfied { - return nil, fmt.Errorf("RecoverDeadMaster: failed %+v promotion; %s", vtorcReparent.promotedReplica.Key, reason) + if satisfied, reason := MasterFailoverGeographicConstraintSatisfied(&vtorcReparent.analysisEntry, newPrimaryInstance); !satisfied { + return fmt.Errorf("RecoverDeadMaster: failed %+v promotion; %s", newPrimaryInstance.Key, reason) } if config.Config.FailMasterPromotionOnLagMinutes > 0 && - time.Duration(vtorcReparent.promotedReplica.ReplicationLagSeconds.Int64)*time.Second >= time.Duration(config.Config.FailMasterPromotionOnLagMinutes)*time.Minute { + time.Duration(newPrimaryInstance.ReplicationLagSeconds.Int64)*time.Second >= time.Duration(config.Config.FailMasterPromotionOnLagMinutes)*time.Minute { // candidate replica lags too much - return nil, fmt.Errorf("RecoverDeadMaster: failed promotion. FailMasterPromotionOnLagMinutes is set to %d (minutes) and promoted replica %+v 's lag is %d (seconds)", config.Config.FailMasterPromotionOnLagMinutes, vtorcReparent.promotedReplica.Key, vtorcReparent.promotedReplica.ReplicationLagSeconds.Int64) + return fmt.Errorf("RecoverDeadMaster: failed promotion. FailMasterPromotionOnLagMinutes is set to %d (minutes) and promoted replica %+v 's lag is %d (seconds)", config.Config.FailMasterPromotionOnLagMinutes, newPrimaryInstance.Key, newPrimaryInstance.ReplicationLagSeconds.Int64) } - if config.Config.FailMasterPromotionIfSQLThreadNotUpToDate && !vtorcReparent.promotedReplica.SQLThreadUpToDate() { - return nil, fmt.Errorf("RecoverDeadMaster: failed promotion. FailMasterPromotionIfSQLThreadNotUpToDate is set and promoted replica %+v 's sql thread is not up to date (relay logs still unapplied). Aborting promotion", vtorcReparent.promotedReplica.Key) + if config.Config.FailMasterPromotionIfSQLThreadNotUpToDate && !newPrimaryInstance.SQLThreadUpToDate() { + return fmt.Errorf("RecoverDeadMaster: failed promotion. FailMasterPromotionIfSQLThreadNotUpToDate is set and promoted replica %+v 's sql thread is not up to date (relay logs still unapplied). Aborting promotion", newPrimaryInstance.Key) } - if config.Config.DelayMasterPromotionIfSQLThreadNotUpToDate && !vtorcReparent.promotedReplica.SQLThreadUpToDate() { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("DelayMasterPromotionIfSQLThreadNotUpToDate: waiting for SQL thread on %+v", vtorcReparent.promotedReplica.Key)) - if _, err := inst.WaitForSQLThreadUpToDate(&vtorcReparent.promotedReplica.Key, 0, 0); err != nil { - return nil, fmt.Errorf("DelayMasterPromotionIfSQLThreadNotUpToDate error: %+v", err) + if config.Config.DelayMasterPromotionIfSQLThreadNotUpToDate && !newPrimaryInstance.SQLThreadUpToDate() { + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("DelayMasterPromotionIfSQLThreadNotUpToDate: waiting for SQL thread on %+v", newPrimaryInstance.Key)) + if _, err := inst.WaitForSQLThreadUpToDate(&newPrimaryInstance.Key, 0, 0); err != nil { + return fmt.Errorf("DelayMasterPromotionIfSQLThreadNotUpToDate error: %+v", err) } - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("DelayMasterPromotionIfSQLThreadNotUpToDate: SQL thread caught up on %+v", vtorcReparent.promotedReplica.Key)) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("DelayMasterPromotionIfSQLThreadNotUpToDate: SQL thread caught up on %+v", newPrimaryInstance.Key)) } // All seems well. No override done. - return vtorcReparent.promotedReplica, err + return nil } - if vtorcReparent.promotedReplica, err = overrideMasterPromotion(); err != nil { + if err := overrideMasterPromotion(); err != nil { AuditTopologyRecovery(vtorcReparent.topologyRecovery, err.Error()) + return err } return nil } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 3d2b390a4c6..09d48e93a1d 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -173,9 +173,12 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve if err != nil { return err } + newPrimary = betterCandidate } - if err := reparentFunctions.CheckIfNeedToOverridePrimary(); err != nil { + if err := reparentFunctions.CheckIfNeedToOverridePromotion(newPrimary); err != nil { + erp.logger.Errorf("have to override promotion because of constraint failure - %v", err) + // TODO: override promotion here return err } diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index e2e01eaf41c..1bd36dd7e16 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -55,7 +55,7 @@ type ( PromotedReplicaIsIdeal(*topodatapb.Tablet, map[string]*topo.TabletInfo, map[string]*replicationdatapb.PrimaryStatus, map[string]mysql.Position) bool PostTabletChangeHook(*topodatapb.Tablet) GetBetterCandidate(*topodatapb.Tablet, []*topodatapb.Tablet, map[string]*replicationdatapb.PrimaryStatus, map[string]*topo.TabletInfo) *topodatapb.Tablet - CheckIfNeedToOverridePrimary() error + CheckIfNeedToOverridePromotion(newPrimary *topodatapb.Tablet) error StartReplication(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) error GetNewPrimary() *topodatapb.Tablet @@ -244,7 +244,7 @@ func (vtctlReparent *VtctlReparentFunctions) GetBetterCandidate(newPrimary *topo } // CheckIfNeedToOverridePrimary implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePrimary() error { +func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePromotion(newPrimary *topodatapb.Tablet) error { return nil } From 186857f30c48e2bb31782fdc18f12bfac16aa3ba Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 26 Aug 2021 16:37:03 +0530 Subject: [PATCH 041/176] add undo demotion code Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 1 + .../reparentutil/emergency_reparenter.go | 45 ++++++++++++++++--- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index d192a54eecc..4300613f0b8 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -596,6 +596,7 @@ func (vtorcReparent *VtOrcReparentFunctions) CheckIfNeedToOverridePromotion(newP } if err := overrideMasterPromotion(); err != nil { AuditTopologyRecovery(vtorcReparent.topologyRecovery, err.Error()) + vtorcReparent.promotedReplica = nil return err } return nil diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 09d48e93a1d..0c99268a7e3 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -18,6 +18,9 @@ package reparentutil import ( "context" + "time" + + replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" "vitess.io/vitess/go/vt/topo/topoproto" @@ -176,10 +179,17 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve newPrimary = betterCandidate } - if err := reparentFunctions.CheckIfNeedToOverridePromotion(newPrimary); err != nil { - erp.logger.Errorf("have to override promotion because of constraint failure - %v", err) - // TODO: override promotion here - return err + var errInPromotion error + errInPromotion = reparentFunctions.CheckIfNeedToOverridePromotion(newPrimary) + if errInPromotion != nil { + erp.logger.Errorf("have to override promotion because of constraint failure - %v", errInPromotion) + newPrimary, err = erp.undoPromotion(ctx, ts, ev, keyspace, shard, primaryStatusMap, "", "", tabletMap, statusMap, reparentFunctions) + if err != nil { + return err + } + if newPrimary == nil { + return vterrors.Errorf(vtrpc.Code_ABORTED, "more than 1 tablets thought they were primary, cannot undo promotion") + } } if err := reparentFunctions.StartReplication(ctx, ev, erp.logger, erp.tmc); err != nil { @@ -188,5 +198,30 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve ev.NewPrimary = proto.Clone(reparentFunctions.GetNewPrimary()).(*topodatapb.Tablet) - return nil + return errInPromotion +} + +func (erp *EmergencyReparenter) undoPromotion(ctx context.Context, ts *topo.Server, ev *events.Reparent, keyspace, shard string, primaryStatusMap map[string]*replicationdatapb.PrimaryStatus, + lockAction, rp string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions) (*topodatapb.Tablet, error) { + if len(primaryStatusMap) == 1 { + var prevPrimary *topodatapb.Tablet + for tablet := range primaryStatusMap { + prevPrimary = tabletMap[tablet].Tablet + } + if prevPrimary != nil { + _, err := promotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, prevPrimary, lockAction, rp, tabletMap, statusMap, reparentFunctions, true) + if err != nil { + return prevPrimary, err + } + } + return prevPrimary, nil + } + + newTerm := time.Now() + _, err := ts.UpdateShardFields(ctx, keyspace, shard, func(si *topo.ShardInfo) error { + si.PrimaryAlias = nil + si.PrimaryTermStartTime = logutil.TimeToProto(newTerm) + return nil + }) + return nil, err } From ae47acdb956271ec947963a8bd9da1da1236a183 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 27 Aug 2021 13:59:56 +0530 Subject: [PATCH 042/176] promotePrimary to fix replication and semi-sync Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 56 +++++++++---------- .../reparentutil/emergency_reparenter.go | 7 ++- go/vt/vtctl/reparentutil/util.go | 2 +- 3 files changed, 35 insertions(+), 30 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 4300613f0b8..b2ca129dc1d 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -613,34 +613,34 @@ func (vtorcReparent *VtOrcReparentFunctions) StartReplication(ctx context.Contex AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: successfully promoted %+v", vtorcReparent.promotedReplica.Key)) AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: promoted server coordinates: %+v", vtorcReparent.promotedReplica.SelfBinlogCoordinates)) - AuditTopologyRecovery(vtorcReparent.topologyRecovery, "- RecoverDeadMaster: will apply MySQL changes to promoted master") - { - _, err := inst.ResetReplicationOperation(&vtorcReparent.promotedReplica.Key) - if err != nil { - // Ugly, but this is important. Let's give it another try - _, err = inst.ResetReplicationOperation(&vtorcReparent.promotedReplica.Key) - } - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying RESET SLAVE ALL on promoted master: success=%t", (err == nil))) - if err != nil { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: NOTE that %+v is promoted even though SHOW SLAVE STATUS may still show it has a master", vtorcReparent.promotedReplica.Key)) - } - } - { - count := inst.MasterSemiSync(vtorcReparent.promotedReplica.Key) - err := inst.SetSemiSyncMaster(&vtorcReparent.promotedReplica.Key, count > 0) - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying semi-sync %v: success=%t", count > 0, (err == nil))) - - // Dont' allow writes if semi-sync settings fail. - if err == nil { - _, err := inst.SetReadOnly(&vtorcReparent.promotedReplica.Key, false) - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying read-only=0 on promoted master: success=%t", (err == nil))) - } - } - // Let's attempt, though we won't necessarily succeed, to set old master as read-only - go func() { - _, err := inst.SetReadOnly(&vtorcReparent.analysisEntry.AnalyzedInstanceKey, true) - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying read-only=1 on demoted master: success=%t", (err == nil))) - }() + //AuditTopologyRecovery(vtorcReparent.topologyRecovery, "- RecoverDeadMaster: will apply MySQL changes to promoted master") + //{ + // _, err := inst.ResetReplicationOperation(&vtorcReparent.promotedReplica.Key) + // if err != nil { + // // Ugly, but this is important. Let's give it another try + // _, err = inst.ResetReplicationOperation(&vtorcReparent.promotedReplica.Key) + // } + // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying RESET SLAVE ALL on promoted master: success=%t", (err == nil))) + // if err != nil { + // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: NOTE that %+v is promoted even though SHOW SLAVE STATUS may still show it has a master", vtorcReparent.promotedReplica.Key)) + // } + //} + //{ + // count := inst.MasterSemiSync(vtorcReparent.promotedReplica.Key) + // err := inst.SetSemiSyncMaster(&vtorcReparent.promotedReplica.Key, count > 0) + // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying semi-sync %v: success=%t", count > 0, (err == nil))) + // + // // Dont' allow writes if semi-sync settings fail. + // if err == nil { + // _, err := inst.SetReadOnly(&vtorcReparent.promotedReplica.Key, false) + // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying read-only=0 on promoted master: success=%t", (err == nil))) + // } + //} + //// Let's attempt, though we won't necessarily succeed, to set old master as read-only + //go func() { + // _, err := inst.SetReadOnly(&vtorcReparent.analysisEntry.AnalyzedInstanceKey, true) + // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying read-only=1 on demoted master: success=%t", (err == nil))) + //}() kvPairs := inst.GetClusterMasterKVPairs(vtorcReparent.analysisEntry.ClusterDetails.ClusterAlias, &vtorcReparent.promotedReplica.Key) AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("Writing KV %+v", kvPairs)) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 0c99268a7e3..c664bc405ae 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -192,11 +192,16 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } } + _, err = erp.tmc.PromoteReplica(ctx, newPrimary) + if err != nil { + return vterrors.Wrapf(err, "primary-elect tablet %v failed to be upgraded to primary: %v", newPrimary.Alias, err) + } + if err := reparentFunctions.StartReplication(ctx, ev, erp.logger, erp.tmc); err != nil { return err } - ev.NewPrimary = proto.Clone(reparentFunctions.GetNewPrimary()).(*topodatapb.Tablet) + ev.NewPrimary = proto.Clone(newPrimary).(*topodatapb.Tablet) return errInPromotion } diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 1e562b0b789..445ea36ff52 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -420,7 +420,7 @@ func replaceWithBetterCandidate(ctx context.Context, tmc tmclient.TabletManagerC return err } - _, err = tmc.StopReplicationMinimum(ctx, newPrimary, pos, reparentFunctions.GetWaitReplicasTimeout()) + err = tmc.WaitForPosition(ctx, newPrimary, pos) if err != nil { return err } From 1f74e4619ed75a135f037d2094628e90e580862c Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 27 Aug 2021 14:29:55 +0530 Subject: [PATCH 043/176] fix tests to make the logs less polluted Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/main_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/go/test/endtoend/vtorc/main_test.go b/go/test/endtoend/vtorc/main_test.go index e4b4d6c9025..419972af111 100644 --- a/go/test/endtoend/vtorc/main_test.go +++ b/go/test/endtoend/vtorc/main_test.go @@ -345,6 +345,13 @@ func TestMain(m *testing.M) { }() cluster.PanicHandler(nil) + + // stop vtorc first otherwise its logs get polluted + // with instances being unreachable triggering unnecessary operations + if clusterInstance.VtorcProcess != nil { + _ = clusterInstance.VtorcProcess.TearDown() + } + for _, cellInfo := range cellInfos { killTablets(cellInfo.replicaTablets) killTablets(cellInfo.rdonlyTablets) From be0180d881b24e1763cbe4c1a068d1f7993497f1 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 27 Aug 2021 14:30:26 +0530 Subject: [PATCH 044/176] improve test to check replication statuses after failover Signed-off-by: Manan Gupta --- .../endtoend/vtorc/primary_failure_test.go | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index 3eb08d67309..97901d1afdc 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -33,13 +33,30 @@ import ( // covers the test case master-failover from orchestrator func TestDownPrimary(t *testing.T) { defer cluster.PanicHandler(t) - setupVttabletsAndVtorc(t, 2, 0, nil, "test_config.json") + setupVttabletsAndVtorc(t, 2, 1, nil, "test_config.json") keyspace := &clusterInstance.Keyspaces[0] shard0 := &keyspace.Shards[0] // find primary from topo curPrimary := shardPrimaryTablet(t, clusterInstance, keyspace, shard0) assert.NotNil(t, curPrimary, "should have elected a primary") + // find the replica and rdonly tablets + var replica, rdonly *cluster.Vttablet + for _, tablet := range shard0.Vttablets { + // we know we have only two replcia tablets, so the one not the primary must be the other replica + if tablet.Alias != curPrimary.Alias && tablet.Type == "replica" { + replica = tablet + } + if tablet.Type == "rdonly" { + rdonly = tablet + } + } + assert.NotNil(t, replica, "could not find replica tablet") + assert.NotNil(t, rdonly, "could not find rdonly tablet") + + // check that the replication is setup correctly before we failover + checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{rdonly, replica}, 10*time.Second) + // Make the current primary database unavailable. err := curPrimary.MysqlctlProcess.Stop() require.NoError(t, err) @@ -48,13 +65,10 @@ func TestDownPrimary(t *testing.T) { permanentlyRemoveVttablet(curPrimary) }() - for _, tablet := range shard0.Vttablets { - // we know we have only two tablets, so the "other" one must be the new primary - if tablet.Alias != curPrimary.Alias { - checkPrimaryTablet(t, clusterInstance, tablet) - break - } - } + // check that the replica gets promoted + checkPrimaryTablet(t, clusterInstance, replica) + // also check that the replication is working correctly after failover + runAdditionalCommands(t, replica, []*cluster.Vttablet{replica}, 10*time.Second) } // Failover should not be cross data centers, according to the configuration file From 984e0dd4e69e07305f3f76408bc4f69f798d2a39 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 27 Aug 2021 15:22:41 +0530 Subject: [PATCH 045/176] improved tests and a bug fix Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/main_test.go | 23 ++++++++++ .../endtoend/vtorc/primary_failure_test.go | 42 +++++++++++-------- .../inst/instance_topology_dao.go | 4 +- 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/go/test/endtoend/vtorc/main_test.go b/go/test/endtoend/vtorc/main_test.go index 419972af111..c8afb3ed782 100644 --- a/go/test/endtoend/vtorc/main_test.go +++ b/go/test/endtoend/vtorc/main_test.go @@ -393,6 +393,29 @@ func shardPrimaryTablet(t *testing.T, cluster *cluster.LocalProcessCluster, keys } } +// checkShardNoPrimaryTablet waits till the given shard has no primary tablet. It times out after 1 minute +func checkShardNoPrimaryTablet(t *testing.T, cluster *cluster.LocalProcessCluster, keyspace *cluster.Keyspace, shard *cluster.Shard) { + start := time.Now() + for { + now := time.Now() + if now.Sub(start) > time.Second*60 { + assert.FailNow(t, "failed to find a point in time when shard had no primary before timeout") + } + result, err := cluster.VtctlclientProcess.ExecuteCommandWithOutput("GetShard", fmt.Sprintf("%s/%s", keyspace.Name, shard.Name)) + assert.Nil(t, err) + + var shardInfo topodatapb.Shard + err = json2.Unmarshal([]byte(result), &shardInfo) + assert.Nil(t, err) + if shardInfo.PrimaryAlias == nil { + return + } + log.Warningf("Shard %v/%v has a primary yet, sleep for 1 second\n", keyspace.Name, shard.Name) + time.Sleep(time.Second) + continue + } +} + // Makes sure the tablet type is primary, and its health check agrees. func checkPrimaryTablet(t *testing.T, cluster *cluster.LocalProcessCluster, tablet *cluster.Vttablet) { start := time.Now() diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index 97901d1afdc..31c8aae9f1c 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -20,9 +20,6 @@ import ( "testing" "time" - "vitess.io/vitess/go/json2" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -68,7 +65,7 @@ func TestDownPrimary(t *testing.T) { // check that the replica gets promoted checkPrimaryTablet(t, clusterInstance, replica) // also check that the replication is working correctly after failover - runAdditionalCommands(t, replica, []*cluster.Vttablet{replica}, 10*time.Second) + runAdditionalCommands(t, replica, []*cluster.Vttablet{rdonly}, 10*time.Second) } // Failover should not be cross data centers, according to the configuration file @@ -82,18 +79,23 @@ func TestCrossDataCenterFailure(t *testing.T) { curPrimary := shardPrimaryTablet(t, clusterInstance, keyspace, shard0) assert.NotNil(t, curPrimary, "should have elected a primary") - var replicaInSameCell *cluster.Vttablet + // find the replica and rdonly tablets + var replicaInSameCell, rdonly *cluster.Vttablet for _, tablet := range shard0.Vttablets { // we know we have only two replcia tablets, so the one not the primary must be the other replica if tablet.Alias != curPrimary.Alias && tablet.Type == "replica" { replicaInSameCell = tablet - break + } + if tablet.Type == "rdonly" { + rdonly = tablet } } + assert.NotNil(t, replicaInSameCell, "could not find replica tablet") + assert.NotNil(t, rdonly, "could not find rdonly tablet") crossCellReplica := startVttablet(t, cell2, false) // newly started tablet does not replicate from anyone yet, we will allow orchestrator to fix this too - checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{crossCellReplica, replicaInSameCell}, 25*time.Second) + checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{crossCellReplica, replicaInSameCell, rdonly}, 25*time.Second) // Make the current primary database unavailable. err := curPrimary.MysqlctlProcess.Stop() @@ -105,6 +107,8 @@ func TestCrossDataCenterFailure(t *testing.T) { // we have a replica in the same cell, so that is the one which should be promoted and not the one from another cell checkPrimaryTablet(t, clusterInstance, replicaInSameCell) + // also check that the replication is working correctly after failover + runAdditionalCommands(t, replicaInSameCell, []*cluster.Vttablet{crossCellReplica, rdonly}, 10*time.Second) } // Failover should not be cross data centers, according to the configuration file @@ -118,9 +122,18 @@ func TestCrossDataCenterFailureError(t *testing.T) { curPrimary := shardPrimaryTablet(t, clusterInstance, keyspace, shard0) assert.NotNil(t, curPrimary, "should have elected a primary") + // find the rdonly tablet + var rdonly *cluster.Vttablet + for _, tablet := range shard0.Vttablets { + if tablet.Type == "rdonly" { + rdonly = tablet + } + } + assert.NotNil(t, rdonly, "could not find rdonly tablet") + crossCellReplica := startVttablet(t, cell2, false) // newly started tablet does not replicate from anyone yet, we will allow orchestrator to fix this too - checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{crossCellReplica}, 25*time.Second) + checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{crossCellReplica, rdonly}, 25*time.Second) // Make the current primary database unavailable. err := curPrimary.MysqlctlProcess.Stop() @@ -130,16 +143,9 @@ func TestCrossDataCenterFailureError(t *testing.T) { permanentlyRemoveVttablet(curPrimary) }() - // Give vtorc time to repair - time.Sleep(15 * time.Second) - - // we should not have promoted the crossCellReplica - result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", crossCellReplica.Alias) - require.NoError(t, err) - var tabletInfo topodatapb.Tablet - err = json2.Unmarshal([]byte(result), &tabletInfo) - require.NoError(t, err) - require.NotEqual(t, topodatapb.TabletType_PRIMARY, tabletInfo.GetType()) + // vtorc would run a deadPrimary failure but should not be able to elect any new primary + // it will try to undo the changes and should set the shard to have no primary in that case + checkShardNoPrimaryTablet(t, clusterInstance, keyspace, shard0) } // Failover will sometimes lead to a rdonly which can no longer replicate. diff --git a/go/vt/orchestrator/inst/instance_topology_dao.go b/go/vt/orchestrator/inst/instance_topology_dao.go index 03a1880b18f..51b65f2adb9 100644 --- a/go/vt/orchestrator/inst/instance_topology_dao.go +++ b/go/vt/orchestrator/inst/instance_topology_dao.go @@ -360,7 +360,9 @@ func StopReplicas(replicas [](*Instance), stopReplicationMethod StopReplicationM // StopReplicasNicely will attemt to stop all given replicas nicely, up to timeout func StopReplicasNicely(replicas [](*Instance), timeout time.Duration) [](*Instance) { - return StopReplicas(replicas, StopReplicationNice, timeout) + stoppedReplicas := StopReplicas(replicas, StopReplicationNice, timeout) + stoppedReplicas = RemoveNilInstances(stoppedReplicas) + return stoppedReplicas } // StopReplication stops replication on a given instance From 8a803ae694851f213a7d0ec2f362aed7502be9d2 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 27 Aug 2021 15:56:55 +0530 Subject: [PATCH 046/176] handle error and waiting time for relay logs differently in vtorc Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/reparent_functions.go | 11 +++++++++++ go/vt/vtctl/reparentutil/emergency_reparenter.go | 7 +++++-- go/vt/vtctl/reparentutil/reparent_functions.go | 10 ++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index b2ca129dc1d..7c00ae29af9 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -136,6 +136,17 @@ func (vtorcReparent *VtOrcReparentFunctions) GetWaitReplicasTimeout() time.Durat return time.Duration(config.Config.LockShardTimeoutSeconds) * time.Second } +// TODO : Discuss correct way +func (vtorcReparent *VtOrcReparentFunctions) GetWaitForRelayLogsTimeout() time.Duration { + return 1 * time.Second +} + +// TODO : Discuss correct way +func (vtorcReparent *VtOrcReparentFunctions) HandleRelayLogFailure(err error) error { + log.Infof("failed to apply all relay logs - %v", err) + return nil +} + func (vtorcReparent *VtOrcReparentFunctions) GetIgnoreReplicas() sets.String { return nil } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index c664bc405ae..9d5192aa8fa 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -144,8 +144,11 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // Wait for all candidates to apply relay logs - if err := waitForAllRelayLogsToApply(ctx, erp.logger, erp.tmc, validCandidates, tabletMap, statusMap, reparentFunctions.GetWaitReplicasTimeout()); err != nil { - return err + if err := waitForAllRelayLogsToApply(ctx, erp.logger, erp.tmc, validCandidates, tabletMap, statusMap, reparentFunctions.GetWaitForRelayLogsTimeout()); err != nil { + err = reparentFunctions.HandleRelayLogFailure(err) + if err != nil { + return err + } } newPrimary, err := reparentFunctions.FindPrimaryCandidates(ctx, erp.logger, erp.tmc, validCandidates, tabletMap) diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 1bd36dd7e16..83fc6e7dc83 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -48,6 +48,8 @@ type ( CheckIfFixed() bool PreRecoveryProcesses(context.Context) error GetWaitReplicasTimeout() time.Duration + GetWaitForRelayLogsTimeout() time.Duration + HandleRelayLogFailure(err error) error GetIgnoreReplicas() sets.String CheckPrimaryRecoveryType() error RestrictValidCandidates(map[string]mysql.Position, map[string]*topo.TabletInfo) (map[string]mysql.Position, error) @@ -133,6 +135,14 @@ func (vtctlReparentFunctions *VtctlReparentFunctions) GetWaitReplicasTimeout() t return vtctlReparentFunctions.WaitReplicasTimeout } +func (vtctlReparentFunctions *VtctlReparentFunctions) GetWaitForRelayLogsTimeout() time.Duration { + return vtctlReparentFunctions.WaitReplicasTimeout +} + +func (vtctlReparentFunctions *VtctlReparentFunctions) HandleRelayLogFailure(err error) error { + return err +} + func (vtctlReparentFunctions *VtctlReparentFunctions) GetIgnoreReplicas() sets.String { return vtctlReparentFunctions.IgnoreReplicas } From cf49355655f456a0edef3f380a2d9ef5248d14a4 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 27 Aug 2021 16:54:55 +0530 Subject: [PATCH 047/176] fix lost rdonly test and the bug found Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/main_test.go | 18 ++++++- .../endtoend/vtorc/primary_failure_test.go | 53 ++++++++++++------- .../orchestrator/logic/reparent_functions.go | 25 +++++++-- .../reparentutil/emergency_reparenter.go | 3 +- .../vtctl/reparentutil/reparent_functions.go | 10 ++-- 5 files changed, 77 insertions(+), 32 deletions(-) diff --git a/go/test/endtoend/vtorc/main_test.go b/go/test/endtoend/vtorc/main_test.go index c8afb3ed782..6ead4878e19 100644 --- a/go/test/endtoend/vtorc/main_test.go +++ b/go/test/endtoend/vtorc/main_test.go @@ -324,8 +324,8 @@ func TestMain(m *testing.M) { // setup cellInfos before creating the cluster cellInfos = append(cellInfos, &cellInfo{ cellName: cell1, - numReplicas: 11, - numRdonly: 1, + numReplicas: 12, + numRdonly: 2, uidBase: 100, }) cellInfos = append(cellInfos, &cellInfo{ @@ -696,3 +696,17 @@ func permanentlyRemoveVttablet(tablet *cluster.Vttablet) { } } } + +func changePrivileges(t *testing.T, sql string, tablet *cluster.Vttablet, user string) { + _, err := runSQL(t, "SET sql_log_bin = OFF;"+sql+";SET sql_log_bin = ON;", tablet, "") + require.NoError(t, err) + + res, err := runSQL(t, fmt.Sprintf("SELECT id FROM INFORMATION_SCHEMA.PROCESSLIST WHERE user = '%s'", user), tablet, "") + require.NoError(t, err) + for _, row := range res.Rows { + id, err := row[0].ToInt64() + require.NoError(t, err) + _, err = runSQL(t, fmt.Sprintf("kill %d", id), tablet, "") + require.NoError(t, err) + } +} diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index 31c8aae9f1c..64117c6c5e4 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -152,50 +152,56 @@ func TestCrossDataCenterFailureError(t *testing.T) { // covers part of the test case master-failover-lost-replicas from orchestrator func TestLostRdonlyOnPrimaryFailure(t *testing.T) { defer cluster.PanicHandler(t) - setupVttabletsAndVtorc(t, 2, 1, nil, "test_config.json") + setupVttabletsAndVtorc(t, 2, 2, nil, "test_config.json") keyspace := &clusterInstance.Keyspaces[0] shard0 := &keyspace.Shards[0] // find primary from topo curPrimary := shardPrimaryTablet(t, clusterInstance, keyspace, shard0) assert.NotNil(t, curPrimary, "should have elected a primary") - // get the replicas - var replica, rdonly *cluster.Vttablet + // get the tablets + var replica, rdonly, aheadRdonly *cluster.Vttablet for _, tablet := range shard0.Vttablets { // find tablets which are not the primary if tablet.Alias != curPrimary.Alias { if tablet.Type == "replica" { replica = tablet } else { - rdonly = tablet + if rdonly == nil { + rdonly = tablet + } else { + aheadRdonly = tablet + } } } } assert.NotNil(t, replica, "could not find replica tablet") - assert.NotNil(t, rdonly, "could not find rdonly tablet") + assert.NotNil(t, rdonly, "could not find any rdonly tablet") + assert.NotNil(t, aheadRdonly, "could not find both rdonly tablet") // check that replication is setup correctly - checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{rdonly, replica}, 15*time.Second) + checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{rdonly, aheadRdonly, replica}, 15*time.Second) - // make the replica lag by setting the source_delay to 20 seconds - runSQL(t, "STOP SLAVE", replica, "") - runSQL(t, "CHANGE MASTER TO MASTER_DELAY = 20", replica, "") - runSQL(t, "START SLAVE", replica, "") + // revoke super privileges from vtorc on replica and rdonly so that it is unable to repair the replication + changePrivileges(t, `REVOKE SUPER ON *.* FROM 'orc_client_user'@'%'`, replica, "orc_client_user") + changePrivileges(t, `REVOKE SUPER ON *.* FROM 'orc_client_user'@'%'`, rdonly, "orc_client_user") - defer func() { - // fix the crossCell replica back so that no other tests see this as a side effect - runSQL(t, "STOP SLAVE", replica, "") - runSQL(t, "CHANGE MASTER TO MASTER_DELAY = 0", replica, "") - runSQL(t, "START SLAVE", replica, "") - }() + // stop replication on the replica and rdonly. + err := clusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", replica.Alias) + require.NoError(t, err) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", rdonly.Alias) + require.NoError(t, err) - // check that rdonly is able to replicate. We also want to add some queries to rdonly which will not be there in replica - runAdditionalCommands(t, curPrimary, []*cluster.Vttablet{rdonly}, 15*time.Second) + // check that aheadRdonly is able to replicate. We also want to add some queries to aheadRdonly which will not be there in replica and rdonly + runAdditionalCommands(t, curPrimary, []*cluster.Vttablet{aheadRdonly}, 15*time.Second) - // assert that the replica is indeed lagging and does not have the new insertion by checking the count of rows in the table + // assert that the replica and rdonly are indeed lagging and do not have the new insertion by checking the count of rows in the tables out, err := runSQL(t, "SELECT * FROM vt_insert_test", replica, "vt_ks") require.NoError(t, err) require.Equal(t, 1, len(out.Rows)) + out, err = runSQL(t, "SELECT * FROM vt_insert_test", rdonly, "vt_ks") + require.NoError(t, err) + require.Equal(t, 1, len(out.Rows)) // Make the current primary database unavailable. err = curPrimary.MysqlctlProcess.Stop() @@ -205,11 +211,18 @@ func TestLostRdonlyOnPrimaryFailure(t *testing.T) { permanentlyRemoveVttablet(curPrimary) }() + // grant super privileges back to vtorc on replica and rdonly so that it can repair + changePrivileges(t, `GRANT SUPER ON *.* TO 'orc_client_user'@'%'`, replica, "orc_client_user") + changePrivileges(t, `GRANT SUPER ON *.* TO 'orc_client_user'@'%'`, rdonly, "orc_client_user") + // vtorc must promote the lagging replica and not the rdonly, since it has a MustNotPromoteRule promotion rule checkPrimaryTablet(t, clusterInstance, replica) + // also check that the replication is setup correctly + runAdditionalCommands(t, replica, []*cluster.Vttablet{rdonly}, 15*time.Second) + // check that the rdonly is lost. The lost replica has is detached and its host is prepended with `//` - out, err = runSQL(t, "SELECT HOST FROM performance_schema.replication_connection_configuration", rdonly, "") + out, err = runSQL(t, "SELECT HOST FROM performance_schema.replication_connection_configuration", aheadRdonly, "") require.NoError(t, err) require.Equal(t, "//localhost", out.Rows[0][0].ToString()) } diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 7c00ae29af9..39f5e6a97ca 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -170,7 +170,7 @@ func (vtorcReparent *VtOrcReparentFunctions) RestrictValidCandidates(validCandid } // FindPrimaryCandidates implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, error) { +func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) { vtorcReparent.postponedAll = false //promotedReplicaIsIdeal := func(promoted *inst.Instance, hasBestPromotionRule bool) bool { // if promoted == nil { @@ -197,11 +197,11 @@ func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.C vtorcReparent.topologyRecovery.AddError(err) vtorcReparent.hasBestPromotionRule = hasBestPromotionRule if err != nil { - return nil, err + return nil, nil, err } newPrimary, err := inst.ReadTablet(promotedReplica.Key) if err != nil { - return nil, err + return nil, nil, err } for _, replica := range lostReplicas { @@ -236,7 +236,24 @@ func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.C vtorcReparent.promotedReplica = promotedReplica vtorcReparent.topologyRecovery.LostReplicas.AddInstances(lostReplicas) vtorcReparent.recoveryAttempted = true - return newPrimary, nil + + tabletMapWithoutLostReplicas := map[string]*topo.TabletInfo{} + + for alias, info := range tabletMap { + instance := getInstanceFromTablet(info.Tablet) + isLost := false + for _, replica := range lostReplicas { + if instance.Key.Equals(&replica.Key) { + isLost = true + break + } + } + if !isLost { + tabletMapWithoutLostReplicas[alias] = info + } + } + + return newPrimary, tabletMapWithoutLostReplicas, nil } // ChooseCandidate will choose a candidate replica of a given instance, and take its siblings using GTID diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 9d5192aa8fa..0a45a20cf85 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -151,7 +151,8 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } } - newPrimary, err := reparentFunctions.FindPrimaryCandidates(ctx, erp.logger, erp.tmc, validCandidates, tabletMap) + var newPrimary *topodatapb.Tablet + newPrimary, tabletMap, err = reparentFunctions.FindPrimaryCandidates(ctx, erp.logger, erp.tmc, validCandidates, tabletMap) if err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 83fc6e7dc83..59a64a7c041 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -53,7 +53,7 @@ type ( GetIgnoreReplicas() sets.String CheckPrimaryRecoveryType() error RestrictValidCandidates(map[string]mysql.Position, map[string]*topo.TabletInfo) (map[string]mysql.Position, error) - FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) (*topodatapb.Tablet, error) + FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) PromotedReplicaIsIdeal(*topodatapb.Tablet, map[string]*topo.TabletInfo, map[string]*replicationdatapb.PrimaryStatus, map[string]mysql.Position) bool PostTabletChangeHook(*topodatapb.Tablet) GetBetterCandidate(*topodatapb.Tablet, []*topodatapb.Tablet, map[string]*replicationdatapb.PrimaryStatus, map[string]*topo.TabletInfo) *topodatapb.Tablet @@ -173,7 +173,7 @@ func (vtctlReparent *VtctlReparentFunctions) RestrictValidCandidates(validCandid } // FindPrimaryCandidates implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, error) { +func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) { // Elect the candidate with the most up-to-date position. for alias, position := range validCandidates { if vtctlReparent.winningPosition.IsZero() || position.AtLeast(vtctlReparent.winningPosition) { @@ -192,15 +192,15 @@ func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.C pos, ok := vtctlReparent.validCandidates[vtctlReparent.winningPrimaryTabletAliasStr] switch { case !ok: - return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v has errant GTIDs", vtctlReparent.winningPrimaryTabletAliasStr) + return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v has errant GTIDs", vtctlReparent.winningPrimaryTabletAliasStr) case !pos.AtLeast(vtctlReparent.winningPosition): - return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v at position %v is not fully caught up. Winning position: %v", vtctlReparent.winningPrimaryTabletAliasStr, pos, vtctlReparent.winningPosition) + return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v at position %v is not fully caught up. Winning position: %v", vtctlReparent.winningPrimaryTabletAliasStr, pos, vtctlReparent.winningPosition) } } // TODO:= handle not found error newPrimaryAlias := tabletMap[vtctlReparent.winningPrimaryTabletAliasStr] - return newPrimaryAlias.Tablet, nil + return newPrimaryAlias.Tablet, tabletMap, nil } // PostReplicationChangeHook implements the ReparentFunctions interface From 239e05e451bda4605700dfc6dabf2830230548e7 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 27 Aug 2021 17:01:23 +0530 Subject: [PATCH 048/176] fix promotion lag success test Signed-off-by: Manan Gupta --- .../endtoend/vtorc/primary_failure_test.go | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index 64117c6c5e4..fd2fe2d8edd 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -231,13 +231,30 @@ func TestLostRdonlyOnPrimaryFailure(t *testing.T) { // covers the test case master-failover-fail-promotion-lag-minutes-success from orchestrator func TestPromotionLagSuccess(t *testing.T) { defer cluster.PanicHandler(t) - setupVttabletsAndVtorc(t, 2, 0, nil, "test_config_promotion_success.json") + setupVttabletsAndVtorc(t, 2, 1, nil, "test_config_promotion_success.json") keyspace := &clusterInstance.Keyspaces[0] shard0 := &keyspace.Shards[0] // find primary from topo curPrimary := shardPrimaryTablet(t, clusterInstance, keyspace, shard0) assert.NotNil(t, curPrimary, "should have elected a primary") + // find the replica and rdonly tablets + var replica, rdonly *cluster.Vttablet + for _, tablet := range shard0.Vttablets { + // we know we have only two replcia tablets, so the one not the primary must be the other replica + if tablet.Alias != curPrimary.Alias && tablet.Type == "replica" { + replica = tablet + } + if tablet.Type == "rdonly" { + rdonly = tablet + } + } + assert.NotNil(t, replica, "could not find replica tablet") + assert.NotNil(t, rdonly, "could not find rdonly tablet") + + // check that the replication is setup correctly before we failover + checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{rdonly, replica}, 10*time.Second) + // Make the current primary database unavailable. err := curPrimary.MysqlctlProcess.Stop() require.NoError(t, err) @@ -246,13 +263,10 @@ func TestPromotionLagSuccess(t *testing.T) { permanentlyRemoveVttablet(curPrimary) }() - for _, tablet := range shard0.Vttablets { - // we know we have only two tablets, so the "other" one must be the new primary - if tablet.Alias != curPrimary.Alias { - checkPrimaryTablet(t, clusterInstance, tablet) - break - } - } + // check that the replica gets promoted + checkPrimaryTablet(t, clusterInstance, replica) + // also check that the replication is working correctly after failover + runAdditionalCommands(t, replica, []*cluster.Vttablet{rdonly}, 10*time.Second) } // This test checks that the promotion of a tablet succeeds if it passes the promotion lag test From dcd22082dfd89cd05c7cbdf7f2d6fe1ee2f37d02 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 27 Aug 2021 17:04:31 +0530 Subject: [PATCH 049/176] fix promotion lag failure test and uncomment it Signed-off-by: Manan Gupta --- .../endtoend/vtorc/primary_failure_test.go | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index fd2fe2d8edd..250d3e5a974 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -272,16 +272,31 @@ func TestPromotionLagSuccess(t *testing.T) { // This test checks that the promotion of a tablet succeeds if it passes the promotion lag test // covers the test case master-failover-fail-promotion-lag-minutes-failure from orchestrator func TestPromotionLagFailure(t *testing.T) { - // skip the test since it fails now - t.Skip() defer cluster.PanicHandler(t) - setupVttabletsAndVtorc(t, 2, 0, nil, "test_config_promotion_failure.json") + setupVttabletsAndVtorc(t, 2, 1, nil, "test_config_promotion_failure.json") keyspace := &clusterInstance.Keyspaces[0] shard0 := &keyspace.Shards[0] // find primary from topo curPrimary := shardPrimaryTablet(t, clusterInstance, keyspace, shard0) assert.NotNil(t, curPrimary, "should have elected a primary") + // find the replica and rdonly tablets + var replica, rdonly *cluster.Vttablet + for _, tablet := range shard0.Vttablets { + // we know we have only two replcia tablets, so the one not the primary must be the other replica + if tablet.Alias != curPrimary.Alias && tablet.Type == "replica" { + replica = tablet + } + if tablet.Type == "rdonly" { + rdonly = tablet + } + } + assert.NotNil(t, replica, "could not find replica tablet") + assert.NotNil(t, rdonly, "could not find rdonly tablet") + + // check that the replication is setup correctly before we failover + checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{rdonly, replica}, 10*time.Second) + // Make the current primary database unavailable. err := curPrimary.MysqlctlProcess.Stop() require.NoError(t, err) @@ -290,11 +305,9 @@ func TestPromotionLagFailure(t *testing.T) { permanentlyRemoveVttablet(curPrimary) }() - // wait for 20 seconds - time.Sleep(20 * time.Second) - - // the previous primary should still be the primary since recovery of dead primary should fail - checkPrimaryTablet(t, clusterInstance, curPrimary) + // vtorc would run a deadPrimary failure but should not be able to elect any new primary + // it will try to undo the changes and should set the shard to have no primary in that case + checkShardNoPrimaryTablet(t, clusterInstance, keyspace, shard0) } // covers the test case master-failover-candidate from orchestrator From 899fd0ec9925a75b88e26a86bc28d7468ec8c227 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 27 Aug 2021 17:21:28 +0530 Subject: [PATCH 050/176] improve down primary promotion rule test Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/primary_failure_test.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index 250d3e5a974..96861f95d2d 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -322,9 +322,23 @@ func TestDownPrimaryPromotionRule(t *testing.T) { curPrimary := shardPrimaryTablet(t, clusterInstance, keyspace, shard0) assert.NotNil(t, curPrimary, "should have elected a primary") + // find the replica and rdonly tablets + var replica, rdonly *cluster.Vttablet + for _, tablet := range shard0.Vttablets { + // we know we have only two replcia tablets, so the one not the primary must be the other replica + if tablet.Alias != curPrimary.Alias && tablet.Type == "replica" { + replica = tablet + } + if tablet.Type == "rdonly" { + rdonly = tablet + } + } + assert.NotNil(t, replica, "could not find replica tablet") + assert.NotNil(t, rdonly, "could not find rdonly tablet") + crossCellReplica := startVttablet(t, cell2, false) // newly started tablet does not replicate from anyone yet, we will allow orchestrator to fix this too - checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{crossCellReplica}, 25*time.Second) + checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{crossCellReplica, rdonly, replica}, 25*time.Second) // Make the current primary database unavailable. err := curPrimary.MysqlctlProcess.Stop() @@ -336,6 +350,8 @@ func TestDownPrimaryPromotionRule(t *testing.T) { // we have a replica in the same cell, so that is the one which should be promoted and not the one from another cell checkPrimaryTablet(t, clusterInstance, crossCellReplica) + // also check that the replication is working correctly after failover + runAdditionalCommands(t, crossCellReplica, []*cluster.Vttablet{rdonly, replica}, 10*time.Second) } // covers the test case master-failover-candidate-lag from orchestrator From f825979b4d5658b6456452e2afbc43a9c006cb75 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 27 Aug 2021 21:37:47 +0530 Subject: [PATCH 051/176] improved down primary promotion rule with lag test Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/main_test.go | 15 ++++++++++ .../endtoend/vtorc/primary_failure_test.go | 30 ++++++++++++------- .../vtorc/test_config_crosscenter_prefer.json | 2 +- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/go/test/endtoend/vtorc/main_test.go b/go/test/endtoend/vtorc/main_test.go index 6ead4878e19..1e638a303f7 100644 --- a/go/test/endtoend/vtorc/main_test.go +++ b/go/test/endtoend/vtorc/main_test.go @@ -710,3 +710,18 @@ func changePrivileges(t *testing.T, sql string, tablet *cluster.Vttablet, user s require.NoError(t, err) } } + +func resetPrimaryLogs(t *testing.T, curPrimary *cluster.Vttablet) { + _, err := runSQL(t, "FLUSH BINARY LOGS", curPrimary, "") + require.NoError(t, err) + + binLogsOutput, err := runSQL(t, "SHOW BINARY LOGS", curPrimary, "") + require.NoError(t, err) + require.True(t, len(binLogsOutput.Rows) >= 2, "there should be atlease 2 binlog files") + + lastLogFile := binLogsOutput.Rows[len(binLogsOutput.Rows)-1][0].ToString() + + // purge binary logs of the primary so that crossCellReplica cannot catch up + _, err = runSQL(t, "PURGE BINARY LOGS TO '"+lastLogFile+"'", curPrimary, "") + require.NoError(t, err) +} diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index 96861f95d2d..4179e5e82c0 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -386,21 +386,26 @@ func TestDownPrimaryPromotionRuleWithLag(t *testing.T) { // newly started tablet does not replicate from anyone yet, we will allow orchestrator to fix this too checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{crossCellReplica, replica, rdonly}, 25*time.Second) - // make the crossCellReplica lag by setting the source_delay to 20 seconds - runSQL(t, "STOP SLAVE", crossCellReplica, "") - runSQL(t, "CHANGE MASTER TO MASTER_DELAY = 20", crossCellReplica, "") - runSQL(t, "START SLAVE", crossCellReplica, "") + // revoke super privileges from vtorc on crossCellReplica so that it is unable to repair the replication + changePrivileges(t, `REVOKE SUPER ON *.* FROM 'orc_client_user'@'%'`, crossCellReplica, "orc_client_user") - defer func() { - // fix the crossCell replica back so that no other tests see this as a side effect - runSQL(t, "STOP SLAVE", crossCellReplica, "") - runSQL(t, "CHANGE MASTER TO MASTER_DELAY = 0", crossCellReplica, "") - runSQL(t, "START SLAVE", crossCellReplica, "") - }() + // stop replication on the crossCellReplica. + err := clusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", crossCellReplica.Alias) + require.NoError(t, err) // check that rdonly and replica are able to replicate. We also want to add some queries to replica which will not be there in crossCellReplica runAdditionalCommands(t, curPrimary, []*cluster.Vttablet{replica, rdonly}, 15*time.Second) + // reset the primary logs so that crossCellReplica can never catch up + resetPrimaryLogs(t, curPrimary) + + // start replication back on the crossCellReplica. + err = clusterInstance.VtctlclientProcess.ExecuteCommand("StartReplication", crossCellReplica.Alias) + require.NoError(t, err) + + // grant super privileges back to vtorc on crossCellReplica so that it can repair + changePrivileges(t, `GRANT SUPER ON *.* TO 'orc_client_user'@'%'`, crossCellReplica, "orc_client_user") + // assert that the crossCellReplica is indeed lagging and does not have the new insertion by checking the count of rows in the table out, err := runSQL(t, "SELECT * FROM vt_insert_test", crossCellReplica, "vt_ks") require.NoError(t, err) @@ -417,6 +422,11 @@ func TestDownPrimaryPromotionRuleWithLag(t *testing.T) { // the crossCellReplica is set to be preferred according to the durability requirements. So it must be promoted checkPrimaryTablet(t, clusterInstance, crossCellReplica) + // assert that the crossCellReplica has indeed caught up + out, err = runSQL(t, "SELECT * FROM vt_insert_test", crossCellReplica, "vt_ks") + require.NoError(t, err) + require.Equal(t, 2, len(out.Rows)) + // check that rdonly and replica are able to replicate from the crossCellReplica runAdditionalCommands(t, crossCellReplica, []*cluster.Vttablet{replica, rdonly}, 15*time.Second) } diff --git a/go/test/endtoend/vtorc/test_config_crosscenter_prefer.json b/go/test/endtoend/vtorc/test_config_crosscenter_prefer.json index 0e4e86e7922..35ddf1b2184 100644 --- a/go/test/endtoend/vtorc/test_config_crosscenter_prefer.json +++ b/go/test/endtoend/vtorc/test_config_crosscenter_prefer.json @@ -6,7 +6,7 @@ "MySQLReplicaPassword": "", "RecoveryPeriodBlockSeconds": 1, "InstancePollSeconds": 1, - "LockShardTimeoutSeconds": 45, + "LockShardTimeoutSeconds": 5, "Durability": "specified", "DurabilityParams": { "zone2-0000000200": "prefer" From ad31a29609918a1e0f5feb357000f9a93475ca82 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 27 Aug 2021 21:51:23 +0530 Subject: [PATCH 052/176] improved down primary promotion rule with lag cross center test Signed-off-by: Manan Gupta --- .../endtoend/vtorc/primary_failure_test.go | 30 ++++++++++++------- ...est_config_crosscenter_prefer_prevent.json | 2 +- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index 4179e5e82c0..04c9954a240 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -463,21 +463,26 @@ func TestDownPrimaryPromotionRuleWithLagCrossCenter(t *testing.T) { // newly started tablet does not replicate from anyone yet, we will allow orchestrator to fix this too checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{crossCellReplica, replica, rdonly}, 25*time.Second) - // make the replica lag by setting the source_delay to 20 seconds - runSQL(t, "STOP SLAVE", replica, "") - runSQL(t, "CHANGE MASTER TO MASTER_DELAY = 20", replica, "") - runSQL(t, "START SLAVE", replica, "") + // revoke super privileges from vtorc on replica so that it is unable to repair the replication + changePrivileges(t, `REVOKE SUPER ON *.* FROM 'orc_client_user'@'%'`, replica, "orc_client_user") - defer func() { - // fix the replica back so that no other tests see this as a side effect - runSQL(t, "STOP SLAVE", replica, "") - runSQL(t, "CHANGE MASTER TO MASTER_DELAY = 0", replica, "") - runSQL(t, "START SLAVE", replica, "") - }() + // stop replication on the replica. + err := clusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", replica.Alias) + require.NoError(t, err) // check that rdonly and crossCellReplica are able to replicate. We also want to add some queries to crossCenterReplica which will not be there in replica runAdditionalCommands(t, curPrimary, []*cluster.Vttablet{rdonly, crossCellReplica}, 15*time.Second) + // reset the primary logs so that crossCellReplica can never catch up + resetPrimaryLogs(t, curPrimary) + + // start replication back on the replica. + err = clusterInstance.VtctlclientProcess.ExecuteCommand("StartReplication", replica.Alias) + require.NoError(t, err) + + // grant super privileges back to vtorc on replica so that it can repair + changePrivileges(t, `GRANT SUPER ON *.* TO 'orc_client_user'@'%'`, replica, "orc_client_user") + // assert that the replica is indeed lagging and does not have the new insertion by checking the count of rows in the table out, err := runSQL(t, "SELECT * FROM vt_insert_test", replica, "vt_ks") require.NoError(t, err) @@ -494,6 +499,11 @@ func TestDownPrimaryPromotionRuleWithLagCrossCenter(t *testing.T) { // the replica should be promoted since we have prevented cross cell promotions checkPrimaryTablet(t, clusterInstance, replica) + // assert that the replica has indeed caught up + out, err = runSQL(t, "SELECT * FROM vt_insert_test", replica, "vt_ks") + require.NoError(t, err) + require.Equal(t, 2, len(out.Rows)) + // check that rdonly and crossCellReplica are able to replicate from the replica runAdditionalCommands(t, replica, []*cluster.Vttablet{crossCellReplica, rdonly}, 15*time.Second) } diff --git a/go/test/endtoend/vtorc/test_config_crosscenter_prefer_prevent.json b/go/test/endtoend/vtorc/test_config_crosscenter_prefer_prevent.json index 2263dabe8f2..9da44b0fb88 100644 --- a/go/test/endtoend/vtorc/test_config_crosscenter_prefer_prevent.json +++ b/go/test/endtoend/vtorc/test_config_crosscenter_prefer_prevent.json @@ -6,7 +6,7 @@ "MySQLReplicaPassword": "", "RecoveryPeriodBlockSeconds": 1, "InstancePollSeconds": 1, - "LockShardTimeoutSeconds": 45, + "LockShardTimeoutSeconds": 5, "Durability": "specified", "DurabilityParams": { "zone2-0000000200": "prefer" From 6f3c1250e8ae1fddde764e0cf5f89bbbf2f0a7af Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 27 Aug 2021 22:32:34 +0530 Subject: [PATCH 053/176] bug fixes introduced via merging Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 39f5e6a97ca..80f3a43a8c1 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -115,7 +115,7 @@ func (vtorcReparent *VtOrcReparentFunctions) CheckIfFixed() bool { return true } AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("will handle DeadMaster event on %+v", vtorcReparent.analysisEntry.ClusterDetails.ClusterName)) - recoverDeadMasterCounter.Inc(1) + recoverDeadPrimaryCounter.Inc(1) return false } @@ -153,9 +153,9 @@ func (vtorcReparent *VtOrcReparentFunctions) GetIgnoreReplicas() sets.String { // CheckPrimaryRecoveryType implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) CheckPrimaryRecoveryType() error { - vtorcReparent.topologyRecovery.RecoveryType = GetMasterRecoveryType(&vtorcReparent.topologyRecovery.AnalysisEntry) + vtorcReparent.topologyRecovery.RecoveryType = GetPrimaryRecoveryType(&vtorcReparent.topologyRecovery.AnalysisEntry) AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: masterRecoveryType=%+v", vtorcReparent.topologyRecovery.RecoveryType)) - if vtorcReparent.topologyRecovery.RecoveryType != MasterRecoveryGTID { + if vtorcReparent.topologyRecovery.RecoveryType != PrimaryRecoveryGTID { return vtorcReparent.topologyRecovery.AddError(log.Errorf("RecoveryType unknown/unsupported")) } return nil @@ -208,12 +208,12 @@ func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.C AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: - lost replica: %+v", replica.Key)) } - if promotedReplica != nil && len(lostReplicas) > 0 && config.Config.DetachLostReplicasAfterMasterFailover { + if promotedReplica != nil && len(lostReplicas) > 0 && config.Config.DetachLostReplicasAfterPrimaryFailover { postponedFunction := func() error { AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: lost %+v replicas during recovery process; detaching them", len(lostReplicas))) for _, replica := range lostReplicas { replica := replica - inst.DetachReplicaMasterHost(&replica.Key) + inst.DetachReplicaPrimaryHost(&replica.Key) } return nil } @@ -474,7 +474,7 @@ func getReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, newPri AuditTopologyRecovery(topologyRecovery, "+ searching for an ideal candidate") if oldPrimary != nil { for _, candidateReplica := range preferredCandidates { - if canTakeOverPromotedServerAsMaster(getInstanceFromTablet(candidateReplica), getInstanceFromTablet(newPrimary)) && + if canTakeOverPromotedServerAsPrimary(getInstanceFromTablet(candidateReplica), getInstanceFromTablet(newPrimary)) && candidateReplica.Alias.Cell == oldPrimary.Alias.Cell { // This would make a great candidate AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("orchestrator picks %+v as candidate replacement, based on being in same cell as failed instance", candidateReplica.Alias)) @@ -488,7 +488,7 @@ func getReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, newPri for _, candidateReplica := range preferredCandidates { if topoproto.TabletAliasEqual(newPrimary.Alias, candidateReplica.Alias) { // Seems like we promoted a candidate replica (though not in same DC and ENV as dead primary) - if satisfied, reason := MasterFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(candidateReplica)); satisfied { + if satisfied, reason := PrimaryFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(candidateReplica)); satisfied { // Good enough. No further action required. AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("promoted replica %+v is a good candidate", newPrimary.Alias)) return newPrimary @@ -502,7 +502,7 @@ func getReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, newPri // Try a candidate replica that is in same DC & env as the promoted replica (our promoted replica is not an "is_candidate") AuditTopologyRecovery(topologyRecovery, "+ searching for a candidate") for _, candidateReplica := range preferredCandidates { - if canTakeOverPromotedServerAsMaster(getInstanceFromTablet(candidateReplica), getInstanceFromTablet(newPrimary)) && + if canTakeOverPromotedServerAsPrimary(getInstanceFromTablet(candidateReplica), getInstanceFromTablet(newPrimary)) && newPrimary.Alias.Cell == candidateReplica.Alias.Cell { // OK, better than nothing AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("no candidate was offered for %+v but orchestrator picks %+v as candidate replacement, based on being in same DC & env as promoted instance", newPrimary.Alias, candidateReplica.Alias)) @@ -514,8 +514,8 @@ func getReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, newPri // Try a candidate replica (our promoted replica is not an "is_candidate") AuditTopologyRecovery(topologyRecovery, "+ searching for a candidate") for _, candidateReplica := range preferredCandidates { - if canTakeOverPromotedServerAsMaster(getInstanceFromTablet(candidateReplica), getInstanceFromTablet(newPrimary)) { - if satisfied, reason := MasterFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(candidateReplica)); satisfied { + if canTakeOverPromotedServerAsPrimary(getInstanceFromTablet(candidateReplica), getInstanceFromTablet(newPrimary)) { + if satisfied, reason := PrimaryFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(candidateReplica)); satisfied { // OK, better than nothing AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("no candidate was offered for %+v but orchestrator picks %+v as candidate replacement", newPrimary.Alias, candidateReplica.Alias)) return candidateReplica @@ -526,7 +526,7 @@ func getReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, newPri } keepSearchingHint := "" - if satisfied, reason := MasterFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(newPrimary)); !satisfied { + if satisfied, reason := PrimaryFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(newPrimary)); !satisfied { keepSearchingHint = fmt.Sprintf("Will keep searching; %s", reason) } else if inst.PromotionRule(newPrimary) == inst.PreferNotPromoteRule { keepSearchingHint = fmt.Sprintf("Will keep searching because we have promoted a server with prefer_not rule: %+v", newPrimary.Alias) @@ -538,7 +538,7 @@ func getReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, newPri if oldPrimary != nil { AuditTopologyRecovery(topologyRecovery, "+ searching for a neutral server to replace promoted server, in same DC and env as dead master") for _, neutralReplica := range neutralReplicas { - if canTakeOverPromotedServerAsMaster(getInstanceFromTablet(neutralReplica), getInstanceFromTablet(newPrimary)) && + if canTakeOverPromotedServerAsPrimary(getInstanceFromTablet(neutralReplica), getInstanceFromTablet(newPrimary)) && oldPrimary.Alias.Cell == neutralReplica.Alias.Cell { AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("no candidate was offered for %+v but orchestrator picks %+v as candidate replacement, based on being in same DC & env as dead master", newPrimary.Alias, neutralReplica.Alias)) return neutralReplica @@ -549,7 +549,7 @@ func getReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, newPri // find neutral instance in same dv&env as promoted replica AuditTopologyRecovery(topologyRecovery, "+ searching for a neutral server to replace promoted server, in same DC and env as promoted replica") for _, neutralReplica := range neutralReplicas { - if canTakeOverPromotedServerAsMaster(getInstanceFromTablet(neutralReplica), getInstanceFromTablet(newPrimary)) && + if canTakeOverPromotedServerAsPrimary(getInstanceFromTablet(neutralReplica), getInstanceFromTablet(newPrimary)) && newPrimary.Alias.Cell == neutralReplica.Alias.Cell { AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("no candidate was offered for %+v but orchestrator picks %+v as candidate replacement, based on being in same DC & env as promoted instance", newPrimary.Alias, neutralReplica.Alias)) return neutralReplica @@ -558,8 +558,8 @@ func getReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, newPri AuditTopologyRecovery(topologyRecovery, "+ searching for a neutral server to replace a prefer_not") for _, neutralReplica := range neutralReplicas { - if canTakeOverPromotedServerAsMaster(getInstanceFromTablet(neutralReplica), getInstanceFromTablet(newPrimary)) { - if satisfied, reason := MasterFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(neutralReplica)); satisfied { + if canTakeOverPromotedServerAsPrimary(getInstanceFromTablet(neutralReplica), getInstanceFromTablet(newPrimary)) { + if satisfied, reason := PrimaryFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(neutralReplica)); satisfied { // OK, better than nothing AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("no candidate was offered for %+v but orchestrator picks %+v as candidate replacement, based on promoted instance having prefer_not promotion rule", newPrimary.Alias, neutralReplica.Alias)) return neutralReplica @@ -601,18 +601,18 @@ func (vtorcReparent *VtOrcReparentFunctions) CheckIfNeedToOverridePromotion(newP newPrimaryInstance := getInstanceFromTablet(newPrimary) overrideMasterPromotion := func() error { // Scenarios where we might cancel the promotion. - if satisfied, reason := MasterFailoverGeographicConstraintSatisfied(&vtorcReparent.analysisEntry, newPrimaryInstance); !satisfied { + if satisfied, reason := PrimaryFailoverGeographicConstraintSatisfied(&vtorcReparent.analysisEntry, newPrimaryInstance); !satisfied { return fmt.Errorf("RecoverDeadMaster: failed %+v promotion; %s", newPrimaryInstance.Key, reason) } - if config.Config.FailMasterPromotionOnLagMinutes > 0 && - time.Duration(newPrimaryInstance.ReplicationLagSeconds.Int64)*time.Second >= time.Duration(config.Config.FailMasterPromotionOnLagMinutes)*time.Minute { + if config.Config.FailPrimaryPromotionOnLagMinutes > 0 && + time.Duration(newPrimaryInstance.ReplicationLagSeconds.Int64)*time.Second >= time.Duration(config.Config.FailPrimaryPromotionOnLagMinutes)*time.Minute { // candidate replica lags too much - return fmt.Errorf("RecoverDeadMaster: failed promotion. FailMasterPromotionOnLagMinutes is set to %d (minutes) and promoted replica %+v 's lag is %d (seconds)", config.Config.FailMasterPromotionOnLagMinutes, newPrimaryInstance.Key, newPrimaryInstance.ReplicationLagSeconds.Int64) + return fmt.Errorf("RecoverDeadMaster: failed promotion. FailPrimaryPromotionOnLagMinutes is set to %d (minutes) and promoted replica %+v 's lag is %d (seconds)", config.Config.FailPrimaryPromotionOnLagMinutes, newPrimaryInstance.Key, newPrimaryInstance.ReplicationLagSeconds.Int64) } - if config.Config.FailMasterPromotionIfSQLThreadNotUpToDate && !newPrimaryInstance.SQLThreadUpToDate() { - return fmt.Errorf("RecoverDeadMaster: failed promotion. FailMasterPromotionIfSQLThreadNotUpToDate is set and promoted replica %+v 's sql thread is not up to date (relay logs still unapplied). Aborting promotion", newPrimaryInstance.Key) + if config.Config.FailPrimaryPromotionIfSQLThreadNotUpToDate && !newPrimaryInstance.SQLThreadUpToDate() { + return fmt.Errorf("RecoverDeadMaster: failed promotion. FailPrimaryPromotionIfSQLThreadNotUpToDate is set and promoted replica %+v 's sql thread is not up to date (relay logs still unapplied). Aborting promotion", newPrimaryInstance.Key) } - if config.Config.DelayMasterPromotionIfSQLThreadNotUpToDate && !newPrimaryInstance.SQLThreadUpToDate() { + if config.Config.DelayPrimaryPromotionIfSQLThreadNotUpToDate && !newPrimaryInstance.SQLThreadUpToDate() { AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("DelayMasterPromotionIfSQLThreadNotUpToDate: waiting for SQL thread on %+v", newPrimaryInstance.Key)) if _, err := inst.WaitForSQLThreadUpToDate(&newPrimaryInstance.Key, 0, 0); err != nil { return fmt.Errorf("DelayMasterPromotionIfSQLThreadNotUpToDate error: %+v", err) @@ -637,7 +637,7 @@ func (vtorcReparent *VtOrcReparentFunctions) StartReplication(ctx context.Contex // Now, see whether we are successful or not. From this point there's no going back. if vtorcReparent.promotedReplica != nil { // Success! - recoverDeadMasterSuccessCounter.Inc(1) + recoverDeadPrimarySuccessCounter.Inc(1) AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: successfully promoted %+v", vtorcReparent.promotedReplica.Key)) AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: promoted server coordinates: %+v", vtorcReparent.promotedReplica.SelfBinlogCoordinates)) @@ -670,7 +670,7 @@ func (vtorcReparent *VtOrcReparentFunctions) StartReplication(ctx context.Contex // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying read-only=1 on demoted master: success=%t", (err == nil))) //}() - kvPairs := inst.GetClusterMasterKVPairs(vtorcReparent.analysisEntry.ClusterDetails.ClusterAlias, &vtorcReparent.promotedReplica.Key) + kvPairs := inst.GetClusterPrimaryKVPairs(vtorcReparent.analysisEntry.ClusterDetails.ClusterAlias, &vtorcReparent.promotedReplica.Key) AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("Writing KV %+v", kvPairs)) for _, kvPair := range kvPairs { err := kv.PutKVPair(kvPair) @@ -681,10 +681,10 @@ func (vtorcReparent *VtOrcReparentFunctions) StartReplication(ctx context.Contex err := kv.DistributePairs(kvPairs) log.Errore(err) } - if config.Config.MasterFailoverDetachReplicaMasterHost { + if config.Config.PrimaryFailoverDetachReplicaPrimaryHost { postponedFunction := func() error { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, "- RecoverDeadMaster: detaching master host on promoted master") - inst.DetachReplicaMasterHost(&vtorcReparent.promotedReplica.Key) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, "- RecoverDeadPrimary: detaching master host on promoted master") + inst.DetachReplicaPrimaryHost(&vtorcReparent.promotedReplica.Key) return nil } vtorcReparent.topologyRecovery.AddPostponedFunction(postponedFunction, fmt.Sprintf("RecoverDeadMaster, detaching promoted master host %+v", vtorcReparent.promotedReplica.Key)) @@ -706,10 +706,10 @@ func (vtorcReparent *VtOrcReparentFunctions) StartReplication(ctx context.Contex if !vtorcReparent.skipProcesses { // Execute post master-failover processes - executeProcesses(config.Config.PostMasterFailoverProcesses, "PostMasterFailoverProcesses", vtorcReparent.topologyRecovery, false) + executeProcesses(config.Config.PostPrimaryFailoverProcesses, "PostPrimaryFailoverProcesses", vtorcReparent.topologyRecovery, false) } } else { - recoverDeadMasterFailureCounter.Inc(1) + recoverDeadPrimaryFailureCounter.Inc(1) } return nil } From 78c59c223b0874e9a68bff7283669df09962edd5 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 1 Sep 2021 16:15:59 +0530 Subject: [PATCH 054/176] update promotePrimary rpc Signed-off-by: Manan Gupta --- go/vt/vttablet/tabletmanager/rpc_replication.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index ec16d8003b8..dfad7dc45f0 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -795,7 +795,7 @@ func (tm *TabletManager) PromoteReplica(ctx context.Context) (string, error) { return "", err } - if err := tm.changeTypeLocked(ctx, topodatapb.TabletType_PRIMARY, DBActionSetReadWrite); err != nil { + if err := tm.MysqlDaemon.SetReadOnly(false); err != nil { return "", err } From 2ccf8281c26e8e0126f970102ef436faf9e5e890 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Sat, 4 Sep 2021 18:45:15 +0530 Subject: [PATCH 055/176] use shardInfo instead of primaryStatusMap and bug fixes Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/main_test.go | 7 +-- .../endtoend/vtorc/primary_failure_test.go | 30 +++++++------ go/test/endtoend/vtorc/vtorc_test.go | 6 +-- .../orchestrator/logic/reparent_functions.go | 22 ++++------ .../reparentutil/emergency_reparenter.go | 44 +++++++++++-------- .../vtctl/reparentutil/reparent_functions.go | 21 +++------ go/vt/vtctl/reparentutil/util.go | 10 +++-- 7 files changed, 71 insertions(+), 69 deletions(-) diff --git a/go/test/endtoend/vtorc/main_test.go b/go/test/endtoend/vtorc/main_test.go index 1e638a303f7..d205c5f554a 100644 --- a/go/test/endtoend/vtorc/main_test.go +++ b/go/test/endtoend/vtorc/main_test.go @@ -417,7 +417,7 @@ func checkShardNoPrimaryTablet(t *testing.T, cluster *cluster.LocalProcessCluste } // Makes sure the tablet type is primary, and its health check agrees. -func checkPrimaryTablet(t *testing.T, cluster *cluster.LocalProcessCluster, tablet *cluster.Vttablet) { +func checkPrimaryTablet(t *testing.T, cluster *cluster.LocalProcessCluster, tablet *cluster.Vttablet, checkServing bool) { start := time.Now() for { now := time.Now() @@ -435,7 +435,8 @@ func checkPrimaryTablet(t *testing.T, cluster *cluster.LocalProcessCluster, tabl log.Warningf("Tablet %v is not primary yet, sleep for 1 second\n", tablet.Alias) time.Sleep(time.Second) continue - } // make sure the health stream is updated + } + // make sure the health stream is updated result, err = cluster.VtctlclientProcess.ExecuteCommandWithOutput("VtTabletStreamHealth", "-count", "1", tablet.Alias) require.NoError(t, err) var streamHealthResponse querypb.StreamHealthResponse @@ -445,7 +446,7 @@ func checkPrimaryTablet(t *testing.T, cluster *cluster.LocalProcessCluster, tabl //if !streamHealthResponse.GetServing() { // log.Exitf("stream health not updated") //} - if !streamHealthResponse.GetServing() { + if checkServing && !streamHealthResponse.GetServing() { log.Warningf("Tablet %v is not serving in health stream yet, sleep for 1 second\n", tablet.Alias) time.Sleep(time.Second) continue diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index 04c9954a240..f4295dd069d 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -63,7 +63,7 @@ func TestDownPrimary(t *testing.T) { }() // check that the replica gets promoted - checkPrimaryTablet(t, clusterInstance, replica) + checkPrimaryTablet(t, clusterInstance, replica, true) // also check that the replication is working correctly after failover runAdditionalCommands(t, replica, []*cluster.Vttablet{rdonly}, 10*time.Second) } @@ -106,7 +106,7 @@ func TestCrossDataCenterFailure(t *testing.T) { }() // we have a replica in the same cell, so that is the one which should be promoted and not the one from another cell - checkPrimaryTablet(t, clusterInstance, replicaInSameCell) + checkPrimaryTablet(t, clusterInstance, replicaInSameCell, true) // also check that the replication is working correctly after failover runAdditionalCommands(t, replicaInSameCell, []*cluster.Vttablet{crossCellReplica, rdonly}, 10*time.Second) } @@ -143,9 +143,11 @@ func TestCrossDataCenterFailureError(t *testing.T) { permanentlyRemoveVttablet(curPrimary) }() - // vtorc would run a deadPrimary failure but should not be able to elect any new primary - // it will try to undo the changes and should set the shard to have no primary in that case - checkShardNoPrimaryTablet(t, clusterInstance, keyspace, shard0) + // wait for 20 seconds + time.Sleep(20 * time.Second) + + // the previous primary should still be the primary since recovery of dead primary should fail + checkPrimaryTablet(t, clusterInstance, curPrimary, false) } // Failover will sometimes lead to a rdonly which can no longer replicate. @@ -216,7 +218,7 @@ func TestLostRdonlyOnPrimaryFailure(t *testing.T) { changePrivileges(t, `GRANT SUPER ON *.* TO 'orc_client_user'@'%'`, rdonly, "orc_client_user") // vtorc must promote the lagging replica and not the rdonly, since it has a MustNotPromoteRule promotion rule - checkPrimaryTablet(t, clusterInstance, replica) + checkPrimaryTablet(t, clusterInstance, replica, true) // also check that the replication is setup correctly runAdditionalCommands(t, replica, []*cluster.Vttablet{rdonly}, 15*time.Second) @@ -264,7 +266,7 @@ func TestPromotionLagSuccess(t *testing.T) { }() // check that the replica gets promoted - checkPrimaryTablet(t, clusterInstance, replica) + checkPrimaryTablet(t, clusterInstance, replica, true) // also check that the replication is working correctly after failover runAdditionalCommands(t, replica, []*cluster.Vttablet{rdonly}, 10*time.Second) } @@ -305,9 +307,11 @@ func TestPromotionLagFailure(t *testing.T) { permanentlyRemoveVttablet(curPrimary) }() - // vtorc would run a deadPrimary failure but should not be able to elect any new primary - // it will try to undo the changes and should set the shard to have no primary in that case - checkShardNoPrimaryTablet(t, clusterInstance, keyspace, shard0) + // wait for 20 seconds + time.Sleep(20 * time.Second) + + // the previous primary should still be the primary since recovery of dead primary should fail + checkPrimaryTablet(t, clusterInstance, curPrimary, false) } // covers the test case master-failover-candidate from orchestrator @@ -349,7 +353,7 @@ func TestDownPrimaryPromotionRule(t *testing.T) { }() // we have a replica in the same cell, so that is the one which should be promoted and not the one from another cell - checkPrimaryTablet(t, clusterInstance, crossCellReplica) + checkPrimaryTablet(t, clusterInstance, crossCellReplica, true) // also check that the replication is working correctly after failover runAdditionalCommands(t, crossCellReplica, []*cluster.Vttablet{rdonly, replica}, 10*time.Second) } @@ -420,7 +424,7 @@ func TestDownPrimaryPromotionRuleWithLag(t *testing.T) { }() // the crossCellReplica is set to be preferred according to the durability requirements. So it must be promoted - checkPrimaryTablet(t, clusterInstance, crossCellReplica) + checkPrimaryTablet(t, clusterInstance, crossCellReplica, true) // assert that the crossCellReplica has indeed caught up out, err = runSQL(t, "SELECT * FROM vt_insert_test", crossCellReplica, "vt_ks") @@ -497,7 +501,7 @@ func TestDownPrimaryPromotionRuleWithLagCrossCenter(t *testing.T) { }() // the replica should be promoted since we have prevented cross cell promotions - checkPrimaryTablet(t, clusterInstance, replica) + checkPrimaryTablet(t, clusterInstance, replica, true) // assert that the replica has indeed caught up out, err = runSQL(t, "SELECT * FROM vt_insert_test", replica, "vt_ks") diff --git a/go/test/endtoend/vtorc/vtorc_test.go b/go/test/endtoend/vtorc/vtorc_test.go index 01d4cea5764..e3009f67e61 100644 --- a/go/test/endtoend/vtorc/vtorc_test.go +++ b/go/test/endtoend/vtorc/vtorc_test.go @@ -42,7 +42,7 @@ func TestPrimaryElection(t *testing.T) { keyspace := &clusterInstance.Keyspaces[0] shard0 := &keyspace.Shards[0] - checkPrimaryTablet(t, clusterInstance, shard0.Vttablets[0]) + checkPrimaryTablet(t, clusterInstance, shard0.Vttablets[0], true) checkReplication(t, clusterInstance, shard0.Vttablets[0], shard0.Vttablets[1:], 10*time.Second) } @@ -56,7 +56,7 @@ func TestSingleKeyspace(t *testing.T) { keyspace := &clusterInstance.Keyspaces[0] shard0 := &keyspace.Shards[0] - checkPrimaryTablet(t, clusterInstance, shard0.Vttablets[0]) + checkPrimaryTablet(t, clusterInstance, shard0.Vttablets[0], true) checkReplication(t, clusterInstance, shard0.Vttablets[0], shard0.Vttablets[1:], 10*time.Second) } @@ -70,7 +70,7 @@ func TestKeyspaceShard(t *testing.T) { keyspace := &clusterInstance.Keyspaces[0] shard0 := &keyspace.Shards[0] - checkPrimaryTablet(t, clusterInstance, shard0.Vttablets[0]) + checkPrimaryTablet(t, clusterInstance, shard0.Vttablets[0], true) checkReplication(t, clusterInstance, shard0.Vttablets[0], shard0.Vttablets[1:], 10*time.Second) } diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 80f3a43a8c1..ff7f50b372d 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -132,8 +132,9 @@ func (vtorcReparent *VtOrcReparentFunctions) PreRecoveryProcesses(ctx context.Co return nil } +// TODO : Discuss correct way func (vtorcReparent *VtOrcReparentFunctions) GetWaitReplicasTimeout() time.Duration { - return time.Duration(config.Config.LockShardTimeoutSeconds) * time.Second + return 1 * time.Second } // TODO : Discuss correct way @@ -383,7 +384,7 @@ func ChooseCandidate( } // PromotedReplicaIsIdeal implements the ReparentFunctions interface -func (vtOrcReparent *VtOrcReparentFunctions) PromotedReplicaIsIdeal(newPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, primaryStatus map[string]*replicationdatapb.PrimaryStatus, validCandidates map[string]mysql.Position) bool { +func (vtOrcReparent *VtOrcReparentFunctions) PromotedReplicaIsIdeal(newPrimary, oldPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, validCandidates map[string]mysql.Position) bool { AuditTopologyRecovery(vtOrcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: promotedReplicaIsIdeal(%+v)", newPrimary.Alias)) newPrimaryKey := &inst.InstanceKey{ Hostname: newPrimary.MysqlHostname, @@ -407,13 +408,15 @@ func (vtOrcReparent *VtOrcReparentFunctions) PromotedReplicaIsIdeal(newPrimary * // PostReplicationChangeHook implements the ReparentFunctions interface func (vtOrcReparent *VtOrcReparentFunctions) PostTabletChangeHook(tablet *topodatapb.Tablet) { - inst.ReadTopologyInstance(&inst.InstanceKey{ + instanceKey := &inst.InstanceKey{ Hostname: tablet.MysqlHostname, Port: int(tablet.MysqlPort), - }) + } + inst.ReadTopologyInstance(instanceKey) + TabletRefresh(*instanceKey) } -func (vtorcReparent *VtOrcReparentFunctions) GetBetterCandidate(newPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, primaryStatus map[string]*replicationdatapb.PrimaryStatus, tabletMap map[string]*topo.TabletInfo) *topodatapb.Tablet { +func (vtorcReparent *VtOrcReparentFunctions) GetBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo) *topodatapb.Tablet { if vtorcReparent.candidateInstanceKey != nil { candidateTablet, _ := inst.ReadTablet(*vtorcReparent.candidateInstanceKey) // return the requested candidate as long as it is valid @@ -423,14 +426,7 @@ func (vtorcReparent *VtOrcReparentFunctions) GetBetterCandidate(newPrimary *topo } } } - var oldPrimary *topodatapb.Tablet - if len(primaryStatus) == 1 { - for tablet := range primaryStatus { - ti := tabletMap[tablet] - oldPrimary = ti.Tablet - } - } - replacementCandidate := getReplacementForPromotedReplica(vtorcReparent.topologyRecovery, newPrimary, oldPrimary, validCandidates) + replacementCandidate := getReplacementForPromotedReplica(vtorcReparent.topologyRecovery, newPrimary, prevPrimary, validCandidates) vtorcReparent.promotedReplica = getInstanceFromTablet(replacementCandidate) return replacementCandidate diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 0a45a20cf85..ca87b4bcf53 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -107,6 +107,11 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve ev.ShardInfo = *shardInfo event.DispatchUpdate(ev, "reading all tablets") + prevPrimary, err := ts.GetTablet(ctx, shardInfo.PrimaryAlias) + if err != nil { + return err + } + if err := reparentFunctions.PreRecoveryProcesses(ctx); err != nil { return err } @@ -157,10 +162,10 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } - isIdeal := reparentFunctions.PromotedReplicaIsIdeal(newPrimary, tabletMap, primaryStatusMap, validCandidates) + isIdeal := reparentFunctions.PromotedReplicaIsIdeal(newPrimary, prevPrimary.Tablet, tabletMap, validCandidates) // TODO := LockAction and RP - validReplacementCandidates, err := promotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, newPrimary, "", "", tabletMap, statusMap, reparentFunctions, isIdeal) + validReplacementCandidates, err := promotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, newPrimary, "", "", tabletMap, statusMap, reparentFunctions, isIdeal, true) if err != nil { return err } @@ -172,7 +177,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve betterCandidate := newPrimary if !isIdeal { - betterCandidate = reparentFunctions.GetBetterCandidate(newPrimary, validReplacementCandidates, primaryStatusMap, tabletMap) + betterCandidate = reparentFunctions.GetBetterCandidate(newPrimary, prevPrimary.Tablet, validReplacementCandidates, tabletMap) } if !topoproto.TabletAliasEqual(betterCandidate.Alias, newPrimary.Alias) { @@ -187,12 +192,12 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve errInPromotion = reparentFunctions.CheckIfNeedToOverridePromotion(newPrimary) if errInPromotion != nil { erp.logger.Errorf("have to override promotion because of constraint failure - %v", errInPromotion) - newPrimary, err = erp.undoPromotion(ctx, ts, ev, keyspace, shard, primaryStatusMap, "", "", tabletMap, statusMap, reparentFunctions) + newPrimary, err = erp.undoPromotion(ctx, ts, ev, keyspace, shard, prevPrimary.Tablet, "", "", tabletMap, statusMap, reparentFunctions) if err != nil { return err } if newPrimary == nil { - return vterrors.Errorf(vtrpc.Code_ABORTED, "more than 1 tablets thought they were primary, cannot undo promotion") + return vterrors.Errorf(vtrpc.Code_ABORTED, "could not undo promotion") } } @@ -200,6 +205,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve if err != nil { return vterrors.Wrapf(err, "primary-elect tablet %v failed to be upgraded to primary: %v", newPrimary.Alias, err) } + reparentFunctions.PostTabletChangeHook(newPrimary) if err := reparentFunctions.StartReplication(ctx, ev, erp.logger, erp.tmc); err != nil { return err @@ -210,27 +216,27 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return errInPromotion } -func (erp *EmergencyReparenter) undoPromotion(ctx context.Context, ts *topo.Server, ev *events.Reparent, keyspace, shard string, primaryStatusMap map[string]*replicationdatapb.PrimaryStatus, +func (erp *EmergencyReparenter) undoPromotion(ctx context.Context, ts *topo.Server, ev *events.Reparent, keyspace, shard string, prevPrimary *topodatapb.Tablet, lockAction, rp string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions) (*topodatapb.Tablet, error) { - if len(primaryStatusMap) == 1 { - var prevPrimary *topodatapb.Tablet - for tablet := range primaryStatusMap { - prevPrimary = tabletMap[tablet].Tablet - } - if prevPrimary != nil { - _, err := promotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, prevPrimary, lockAction, rp, tabletMap, statusMap, reparentFunctions, true) - if err != nil { - return prevPrimary, err - } + var primaryAlias *topodatapb.TabletAlias + + if prevPrimary != nil { + _, err := promotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, prevPrimary, lockAction, rp, tabletMap, statusMap, reparentFunctions, true, false) + if err == nil { + return prevPrimary, nil } - return prevPrimary, nil + erp.logger.Errorf("error in undoing promotion - %v", err) + primaryAlias = prevPrimary.Alias } newTerm := time.Now() _, err := ts.UpdateShardFields(ctx, keyspace, shard, func(si *topo.ShardInfo) error { - si.PrimaryAlias = nil + if proto.Equal(si.PrimaryAlias, primaryAlias) { + return nil + } + si.PrimaryAlias = primaryAlias si.PrimaryTermStartTime = logutil.TimeToProto(newTerm) return nil }) - return nil, err + return prevPrimary, err } diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 59a64a7c041..087844faffa 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -54,9 +54,9 @@ type ( CheckPrimaryRecoveryType() error RestrictValidCandidates(map[string]mysql.Position, map[string]*topo.TabletInfo) (map[string]mysql.Position, error) FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) - PromotedReplicaIsIdeal(*topodatapb.Tablet, map[string]*topo.TabletInfo, map[string]*replicationdatapb.PrimaryStatus, map[string]mysql.Position) bool + PromotedReplicaIsIdeal(*topodatapb.Tablet, *topodatapb.Tablet, map[string]*topo.TabletInfo, map[string]mysql.Position) bool PostTabletChangeHook(*topodatapb.Tablet) - GetBetterCandidate(*topodatapb.Tablet, []*topodatapb.Tablet, map[string]*replicationdatapb.PrimaryStatus, map[string]*topo.TabletInfo) *topodatapb.Tablet + GetBetterCandidate(*topodatapb.Tablet, *topodatapb.Tablet, []*topodatapb.Tablet, map[string]*topo.TabletInfo) *topodatapb.Tablet CheckIfNeedToOverridePromotion(newPrimary *topodatapb.Tablet) error StartReplication(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) error GetNewPrimary() *topodatapb.Tablet @@ -209,16 +209,12 @@ func (vtctlReparent *VtctlReparentFunctions) PostTabletChangeHook(*topodatapb.Ta } // PromotedReplicaIsIdeal implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) PromotedReplicaIsIdeal(newPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, primaryStatus map[string]*replicationdatapb.PrimaryStatus, validCandidates map[string]mysql.Position) bool { +func (vtctlReparent *VtctlReparentFunctions) PromotedReplicaIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, validCandidates map[string]mysql.Position) bool { if vtctlReparent.NewPrimaryAlias != nil { //explicit request to promote a specific tablet return true } - if len(primaryStatus) == 1 { - var prevPrimary *topo.TabletInfo - for tablet := range primaryStatus { - prevPrimary = tabletMap[tablet] - } + if prevPrimary != nil { if (newPrimary.Type == topodatapb.TabletType_PRIMARY || newPrimary.Type == topodatapb.TabletType_REPLICA) && newPrimary.Alias.Cell == prevPrimary.Alias.Cell { return true } @@ -232,12 +228,9 @@ func (vtctlReparent *VtctlReparentFunctions) PromotedReplicaIsIdeal(newPrimary * } // GetBetterCandidate implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) GetBetterCandidate(newPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, primaryStatus map[string]*replicationdatapb.PrimaryStatus, tabletMap map[string]*topo.TabletInfo) *topodatapb.Tablet { - if len(primaryStatus) == 1 { - var prevPrimary *topo.TabletInfo - for tablet := range primaryStatus { - prevPrimary = tabletMap[tablet] - } +func (vtctlReparent *VtctlReparentFunctions) GetBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo) *topodatapb.Tablet { + + if prevPrimary != nil { // find one which is of the correct type and matches the cell of the previous primary for _, candidate := range validCandidates { if (candidate.Type == topodatapb.TabletType_PRIMARY || candidate.Type == topodatapb.TabletType_REPLICA) && prevPrimary.Alias.Cell == candidate.Alias.Cell { diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 445ea36ff52..e3db302adbc 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -291,7 +291,7 @@ func ChooseNewPrimary( // promotePrimaryCandidateAndStartReplication promotes the primary candidate that we have, but it does not set to start accepting writes func promotePrimaryCandidateAndStartReplication(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, - lockAction, rp string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions, isIdeal bool) ([]*topodatapb.Tablet, error) { + lockAction, rp string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions, isIdeal bool, startReplication bool) ([]*topodatapb.Tablet, error) { if err := promotePrimary(ctx, tmc, ts, logger, newPrimary); err != nil { return nil, err } @@ -311,9 +311,11 @@ func promotePrimaryCandidateAndStartReplication(ctx context.Context, tmc tmclien // err = moveGTIDFunc() //} - err = tmc.StartReplication(ctx, newPrimary) - if err != nil { - return nil, err + if startReplication { + err = tmc.StartReplication(ctx, newPrimary) + if err != nil { + return nil, err + } } //log.Debugf("RegroupReplicasGTID: done") From 826781cedb63a5b19ff63446966a42757f183f2c Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 6 Sep 2021 15:37:54 +0530 Subject: [PATCH 056/176] remove uncalled GetNewPrimary function Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/reparent_functions.go | 6 ------ go/vt/vtctl/reparentutil/reparent_functions.go | 6 ------ 2 files changed, 12 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index ff7f50b372d..8bde3845a96 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -710,11 +710,5 @@ func (vtorcReparent *VtOrcReparentFunctions) StartReplication(ctx context.Contex return nil } -// GetNewPrimary implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) GetNewPrimary() *topodatapb.Tablet { - tablet, _ := inst.ReadTablet(vtorcReparent.promotedReplica.Key) - return tablet -} - func (vtorcReparent *VtOrcReparentFunctions) SetMaps(tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, primaryStatusMap map[string]*replicationdatapb.PrimaryStatus) { } diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 087844faffa..c5d3dd5a8a3 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -59,7 +59,6 @@ type ( GetBetterCandidate(*topodatapb.Tablet, *topodatapb.Tablet, []*topodatapb.Tablet, map[string]*topo.TabletInfo) *topodatapb.Tablet CheckIfNeedToOverridePromotion(newPrimary *topodatapb.Tablet) error StartReplication(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) error - GetNewPrimary() *topodatapb.Tablet // TODO: remove this SetMaps(map[string]*topo.TabletInfo, map[string]*replicationdatapb.StopReplicationStatus, map[string]*replicationdatapb.PrimaryStatus) @@ -258,11 +257,6 @@ func (vtctlReparent *VtctlReparentFunctions) StartReplication(ctx context.Contex return nil } -// GetNewPrimary implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) GetNewPrimary() *topodatapb.Tablet { - return vtctlReparent.tabletMap[vtctlReparent.winningPrimaryTabletAliasStr].Tablet -} - func (vtctlReparent *VtctlReparentFunctions) SetMaps(tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, primaryStatusMap map[string]*replicationdatapb.PrimaryStatus) { vtctlReparent.tabletMap = tabletMap vtctlReparent.statusMap = statusMap From 5d147cd24b985fefe2e161ec48de7cc9ec2c3a8f Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 6 Sep 2021 15:52:49 +0530 Subject: [PATCH 057/176] remove winningPrimaryTabletAliasString from the vtctlreparentFunctions struct Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter_test.go | 3 +- .../vtctl/reparentutil/reparent_functions.go | 48 +++++++++---------- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 17868b471ee..8cc36bac323 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -1418,11 +1418,10 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { } var err error - tt.vtctlReparentFunctions.winningPrimaryTabletAliasStr = tt.newPrimaryTabletAlias tt.vtctlReparentFunctions.tabletMap = tt.tabletMap tt.vtctlReparentFunctions.statusMap = tt.statusMap - err = tt.vtctlReparentFunctions.promoteNewPrimary(ctx, ev, logger, tt.tmc) + err = tt.vtctlReparentFunctions.promoteNewPrimary(ctx, ev, logger, tt.tmc, tt.newPrimaryTabletAlias) if tt.shouldErr { assert.Error(t, err) return diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index c5d3dd5a8a3..4a7d3a21340 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -66,19 +66,18 @@ type ( // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions VtctlReparentFunctions struct { - NewPrimaryAlias *topodatapb.TabletAlias - IgnoreReplicas sets.String - WaitReplicasTimeout time.Duration - keyspace string - shard string - ts *topo.Server - lockAction string - tabletMap map[string]*topo.TabletInfo - statusMap map[string]*replicationdatapb.StopReplicationStatus - primaryStatusMap map[string]*replicationdatapb.PrimaryStatus - validCandidates map[string]mysql.Position - winningPosition mysql.Position - winningPrimaryTabletAliasStr string + NewPrimaryAlias *topodatapb.TabletAlias + IgnoreReplicas sets.String + WaitReplicasTimeout time.Duration + keyspace string + shard string + ts *topo.Server + lockAction string + tabletMap map[string]*topo.TabletInfo + statusMap map[string]*replicationdatapb.StopReplicationStatus + primaryStatusMap map[string]*replicationdatapb.PrimaryStatus + validCandidates map[string]mysql.Position + winningPosition mysql.Position } ) @@ -174,10 +173,11 @@ func (vtctlReparent *VtctlReparentFunctions) RestrictValidCandidates(validCandid // FindPrimaryCandidates implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) { // Elect the candidate with the most up-to-date position. + var winningPrimaryTabletAliasStr string for alias, position := range validCandidates { if vtctlReparent.winningPosition.IsZero() || position.AtLeast(vtctlReparent.winningPosition) { vtctlReparent.winningPosition = position - vtctlReparent.winningPrimaryTabletAliasStr = alias + winningPrimaryTabletAliasStr = alias } } @@ -187,18 +187,18 @@ func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.C // candidate (non-zero position, no errant GTIDs) and is at least as // advanced as the winning position. if vtctlReparent.NewPrimaryAlias != nil { - vtctlReparent.winningPrimaryTabletAliasStr = topoproto.TabletAliasString(vtctlReparent.NewPrimaryAlias) - pos, ok := vtctlReparent.validCandidates[vtctlReparent.winningPrimaryTabletAliasStr] + winningPrimaryTabletAliasStr = topoproto.TabletAliasString(vtctlReparent.NewPrimaryAlias) + pos, ok := vtctlReparent.validCandidates[winningPrimaryTabletAliasStr] switch { case !ok: - return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v has errant GTIDs", vtctlReparent.winningPrimaryTabletAliasStr) + return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v has errant GTIDs", winningPrimaryTabletAliasStr) case !pos.AtLeast(vtctlReparent.winningPosition): - return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v at position %v is not fully caught up. Winning position: %v", vtctlReparent.winningPrimaryTabletAliasStr, pos, vtctlReparent.winningPosition) + return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v at position %v is not fully caught up. Winning position: %v", winningPrimaryTabletAliasStr, pos, vtctlReparent.winningPosition) } } // TODO:= handle not found error - newPrimaryAlias := tabletMap[vtctlReparent.winningPrimaryTabletAliasStr] + newPrimaryAlias := tabletMap[winningPrimaryTabletAliasStr] return newPrimaryAlias.Tablet, tabletMap, nil } @@ -273,18 +273,18 @@ func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topo return action } -func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) error { - logger.Infof("promoting tablet %v to master", vtctlReparent.winningPrimaryTabletAliasStr) +func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, winningPrimaryTabletAliasStr string) error { + logger.Infof("promoting tablet %v to master", winningPrimaryTabletAliasStr) event.DispatchUpdate(ev, "promoting replica") - newPrimaryTabletInfo, ok := vtctlReparent.tabletMap[vtctlReparent.winningPrimaryTabletAliasStr] + newPrimaryTabletInfo, ok := vtctlReparent.tabletMap[winningPrimaryTabletAliasStr] if !ok { - return vterrors.Errorf(vtrpc.Code_INTERNAL, "attempted to promote master-elect %v that was not in the tablet map; this an impossible situation", vtctlReparent.winningPrimaryTabletAliasStr) + return vterrors.Errorf(vtrpc.Code_INTERNAL, "attempted to promote master-elect %v that was not in the tablet map; this an impossible situation", winningPrimaryTabletAliasStr) } rp, err := tmc.PromoteReplica(ctx, newPrimaryTabletInfo.Tablet) if err != nil { - return vterrors.Wrapf(err, "master-elect tablet %v failed to be upgraded to master: %v", vtctlReparent.winningPrimaryTabletAliasStr, err) + return vterrors.Wrapf(err, "master-elect tablet %v failed to be upgraded to master: %v", winningPrimaryTabletAliasStr, err) } if err := topo.CheckShardLocked(ctx, vtctlReparent.keyspace, vtctlReparent.shard); err != nil { From 56cf2d64fb67e3cfa9e7b2797a1969f74b3f0085 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 6 Sep 2021 15:54:07 +0530 Subject: [PATCH 058/176] remove winningPosition from the vtctlreparentFunctions struct Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/reparent_functions.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 4a7d3a21340..63ee7870b00 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -77,7 +77,6 @@ type ( statusMap map[string]*replicationdatapb.StopReplicationStatus primaryStatusMap map[string]*replicationdatapb.PrimaryStatus validCandidates map[string]mysql.Position - winningPosition mysql.Position } ) @@ -174,9 +173,10 @@ func (vtctlReparent *VtctlReparentFunctions) RestrictValidCandidates(validCandid func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) { // Elect the candidate with the most up-to-date position. var winningPrimaryTabletAliasStr string + var winningPosition mysql.Position for alias, position := range validCandidates { - if vtctlReparent.winningPosition.IsZero() || position.AtLeast(vtctlReparent.winningPosition) { - vtctlReparent.winningPosition = position + if winningPosition.IsZero() || position.AtLeast(winningPosition) { + winningPosition = position winningPrimaryTabletAliasStr = alias } } @@ -192,8 +192,8 @@ func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.C switch { case !ok: return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v has errant GTIDs", winningPrimaryTabletAliasStr) - case !pos.AtLeast(vtctlReparent.winningPosition): - return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v at position %v is not fully caught up. Winning position: %v", winningPrimaryTabletAliasStr, pos, vtctlReparent.winningPosition) + case !pos.AtLeast(winningPosition): + return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v at position %v is not fully caught up. Winning position: %v", winningPrimaryTabletAliasStr, pos, winningPosition) } } From ab25e7557e56bb54cbef916a6f4fc2646d7ac21b Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 6 Sep 2021 15:54:47 +0530 Subject: [PATCH 059/176] remove validCandidates from the vtctlreparentFunctions struct Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/reparent_functions.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 63ee7870b00..5eb679511ea 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -76,7 +76,6 @@ type ( tabletMap map[string]*topo.TabletInfo statusMap map[string]*replicationdatapb.StopReplicationStatus primaryStatusMap map[string]*replicationdatapb.PrimaryStatus - validCandidates map[string]mysql.Position } ) @@ -181,14 +180,12 @@ func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.C } } - vtctlReparent.validCandidates = validCandidates - // If we were requested to elect a particular primary, verify it's a valid // candidate (non-zero position, no errant GTIDs) and is at least as // advanced as the winning position. if vtctlReparent.NewPrimaryAlias != nil { winningPrimaryTabletAliasStr = topoproto.TabletAliasString(vtctlReparent.NewPrimaryAlias) - pos, ok := vtctlReparent.validCandidates[winningPrimaryTabletAliasStr] + pos, ok := validCandidates[winningPrimaryTabletAliasStr] switch { case !ok: return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v has errant GTIDs", winningPrimaryTabletAliasStr) From ab90712787c223edf0f654501fd3795dfffb1993 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 6 Sep 2021 15:56:08 +0530 Subject: [PATCH 060/176] remove statusMaps from the vtctlreparentFunctions struct Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/emergency_reparenter_test.go | 3 +-- go/vt/vtctl/reparentutil/reparent_functions.go | 8 ++------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 8cc36bac323..a85d4369e0a 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -1419,9 +1419,8 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { var err error tt.vtctlReparentFunctions.tabletMap = tt.tabletMap - tt.vtctlReparentFunctions.statusMap = tt.statusMap - err = tt.vtctlReparentFunctions.promoteNewPrimary(ctx, ev, logger, tt.tmc, tt.newPrimaryTabletAlias) + err = tt.vtctlReparentFunctions.promoteNewPrimary(ctx, ev, logger, tt.tmc, tt.newPrimaryTabletAlias, tt.statusMap) if tt.shouldErr { assert.Error(t, err) return diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 5eb679511ea..0d0e9ef4a13 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -74,8 +74,6 @@ type ( ts *topo.Server lockAction string tabletMap map[string]*topo.TabletInfo - statusMap map[string]*replicationdatapb.StopReplicationStatus - primaryStatusMap map[string]*replicationdatapb.PrimaryStatus } ) @@ -256,8 +254,6 @@ func (vtctlReparent *VtctlReparentFunctions) StartReplication(ctx context.Contex func (vtctlReparent *VtctlReparentFunctions) SetMaps(tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, primaryStatusMap map[string]*replicationdatapb.PrimaryStatus) { vtctlReparent.tabletMap = tabletMap - vtctlReparent.statusMap = statusMap - vtctlReparent.primaryStatusMap = primaryStatusMap } func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { @@ -270,7 +266,7 @@ func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topo return action } -func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, winningPrimaryTabletAliasStr string) error { +func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, winningPrimaryTabletAliasStr string, statusMap map[string]*replicationdatapb.StopReplicationStatus) error { logger.Infof("promoting tablet %v to master", winningPrimaryTabletAliasStr) event.DispatchUpdate(ev, "promoting replica") @@ -288,6 +284,6 @@ func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Conte return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } - _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.lockAction, rp, vtctlReparent.tabletMap, vtctlReparent.statusMap, vtctlReparent, false) + _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.lockAction, rp, vtctlReparent.tabletMap, statusMap, vtctlReparent, false) return err } From ae4911957e25e6bc791cbf5213b5c762294bf7df Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 6 Sep 2021 15:57:15 +0530 Subject: [PATCH 061/176] remove tabletMap from the vtctlreparentFunctions struct Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/emergency_reparenter_test.go | 3 +-- go/vt/vtctl/reparentutil/reparent_functions.go | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index a85d4369e0a..8c095cfdf8c 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -1418,9 +1418,8 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { } var err error - tt.vtctlReparentFunctions.tabletMap = tt.tabletMap - err = tt.vtctlReparentFunctions.promoteNewPrimary(ctx, ev, logger, tt.tmc, tt.newPrimaryTabletAlias, tt.statusMap) + err = tt.vtctlReparentFunctions.promoteNewPrimary(ctx, ev, logger, tt.tmc, tt.newPrimaryTabletAlias, tt.statusMap, tt.tabletMap) if tt.shouldErr { assert.Error(t, err) return diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 0d0e9ef4a13..708c2dfab9e 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -73,7 +73,6 @@ type ( shard string ts *topo.Server lockAction string - tabletMap map[string]*topo.TabletInfo } ) @@ -253,7 +252,6 @@ func (vtctlReparent *VtctlReparentFunctions) StartReplication(ctx context.Contex } func (vtctlReparent *VtctlReparentFunctions) SetMaps(tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, primaryStatusMap map[string]*replicationdatapb.PrimaryStatus) { - vtctlReparent.tabletMap = tabletMap } func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { @@ -266,11 +264,11 @@ func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topo return action } -func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, winningPrimaryTabletAliasStr string, statusMap map[string]*replicationdatapb.StopReplicationStatus) error { +func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, winningPrimaryTabletAliasStr string, statusMap map[string]*replicationdatapb.StopReplicationStatus, tabletMap map[string]*topo.TabletInfo) error { logger.Infof("promoting tablet %v to master", winningPrimaryTabletAliasStr) event.DispatchUpdate(ev, "promoting replica") - newPrimaryTabletInfo, ok := vtctlReparent.tabletMap[winningPrimaryTabletAliasStr] + newPrimaryTabletInfo, ok := tabletMap[winningPrimaryTabletAliasStr] if !ok { return vterrors.Errorf(vtrpc.Code_INTERNAL, "attempted to promote master-elect %v that was not in the tablet map; this an impossible situation", winningPrimaryTabletAliasStr) } @@ -284,6 +282,6 @@ func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Conte return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } - _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.lockAction, rp, vtctlReparent.tabletMap, statusMap, vtctlReparent, false) + _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.lockAction, rp, tabletMap, statusMap, vtctlReparent, false) return err } From f2bc87d37d5ddb672d5d2823fc47297c63f2398a Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 6 Sep 2021 15:58:10 +0530 Subject: [PATCH 062/176] removed setMaps entirely Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/reparent_functions.go | 5 ----- go/vt/vtctl/reparentutil/emergency_reparenter.go | 3 --- go/vt/vtctl/reparentutil/reparent_functions.go | 6 ------ 3 files changed, 14 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 8bde3845a96..c07a0dcc8d9 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -23,8 +23,6 @@ import ( "vitess.io/vitess/go/vt/topo/topoproto" - replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" - "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/vterrors" @@ -709,6 +707,3 @@ func (vtorcReparent *VtOrcReparentFunctions) StartReplication(ctx context.Contex } return nil } - -func (vtorcReparent *VtOrcReparentFunctions) SetMaps(tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, primaryStatusMap map[string]*replicationdatapb.PrimaryStatus) { -} diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index ca87b4bcf53..5d4a9b8df7e 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -134,9 +134,6 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } - // TODO: remove this entirely - reparentFunctions.SetMaps(tabletMap, statusMap, primaryStatusMap) - validCandidates, err := FindValidEmergencyReparentCandidates(statusMap, primaryStatusMap) if err != nil { return err diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 708c2dfab9e..6d632b4b67a 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -59,9 +59,6 @@ type ( GetBetterCandidate(*topodatapb.Tablet, *topodatapb.Tablet, []*topodatapb.Tablet, map[string]*topo.TabletInfo) *topodatapb.Tablet CheckIfNeedToOverridePromotion(newPrimary *topodatapb.Tablet) error StartReplication(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) error - - // TODO: remove this - SetMaps(map[string]*topo.TabletInfo, map[string]*replicationdatapb.StopReplicationStatus, map[string]*replicationdatapb.PrimaryStatus) } // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions @@ -251,9 +248,6 @@ func (vtctlReparent *VtctlReparentFunctions) StartReplication(ctx context.Contex return nil } -func (vtctlReparent *VtctlReparentFunctions) SetMaps(tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, primaryStatusMap map[string]*replicationdatapb.PrimaryStatus) { -} - func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { action := "EmergencyReparentShard" From c3eab1bcba48b0ee26e29872a704ef32ac026acb Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 6 Sep 2021 16:09:02 +0530 Subject: [PATCH 063/176] remove lockString from the vtctlreparentFunctions struct Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/reparent_functions.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 6d632b4b67a..e08ca02633b 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -69,7 +69,6 @@ type ( keyspace string shard string ts *topo.Server - lockAction string } ) @@ -91,9 +90,7 @@ func NewVtctlReparentFunctions(newPrimaryAlias *topodatapb.TabletAlias, ignoreRe // LockShard implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) LockShard(ctx context.Context) (context.Context, func(*error), error) { - vtctlReparent.lockAction = vtctlReparent.getLockAction(vtctlReparent.NewPrimaryAlias) - - return vtctlReparent.ts.LockShard(ctx, vtctlReparent.keyspace, vtctlReparent.shard, vtctlReparent.lockAction) + return vtctlReparent.ts.LockShard(ctx, vtctlReparent.keyspace, vtctlReparent.shard, vtctlReparent.getLockAction(vtctlReparent.NewPrimaryAlias)) } // GetTopoServer implements the ReparentFunctions interface @@ -276,6 +273,6 @@ func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Conte return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } - _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.lockAction, rp, tabletMap, statusMap, vtctlReparent, false) + _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.getLockAction(vtctlReparent.NewPrimaryAlias), rp, tabletMap, statusMap, vtctlReparent, false) return err } From b14ac051ec92d7f59425b5c751c78e4520dc725c Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 6 Sep 2021 16:10:24 +0530 Subject: [PATCH 064/176] unexport fields from the vtctlreparentFunctions struct Signed-off-by: Manan Gupta --- .../vtctl/reparentutil/reparent_functions.go | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index e08ca02633b..6c0faf98613 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -63,9 +63,9 @@ type ( // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions VtctlReparentFunctions struct { - NewPrimaryAlias *topodatapb.TabletAlias - IgnoreReplicas sets.String - WaitReplicasTimeout time.Duration + newPrimaryAlias *topodatapb.TabletAlias + ignoreReplicas sets.String + waitReplicasTimeout time.Duration keyspace string shard string ts *topo.Server @@ -79,9 +79,9 @@ var ( // NewVtctlReparentFunctions creates a new VtctlReparentFunctions which is used in ERS ans PRS func NewVtctlReparentFunctions(newPrimaryAlias *topodatapb.TabletAlias, ignoreReplicas sets.String, waitReplicasTimeout time.Duration, keyspace string, shard string, ts *topo.Server) *VtctlReparentFunctions { return &VtctlReparentFunctions{ - NewPrimaryAlias: newPrimaryAlias, - IgnoreReplicas: ignoreReplicas, - WaitReplicasTimeout: waitReplicasTimeout, + newPrimaryAlias: newPrimaryAlias, + ignoreReplicas: ignoreReplicas, + waitReplicasTimeout: waitReplicasTimeout, keyspace: keyspace, shard: shard, ts: ts, @@ -90,7 +90,7 @@ func NewVtctlReparentFunctions(newPrimaryAlias *topodatapb.TabletAlias, ignoreRe // LockShard implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) LockShard(ctx context.Context) (context.Context, func(*error), error) { - return vtctlReparent.ts.LockShard(ctx, vtctlReparent.keyspace, vtctlReparent.shard, vtctlReparent.getLockAction(vtctlReparent.NewPrimaryAlias)) + return vtctlReparent.ts.LockShard(ctx, vtctlReparent.keyspace, vtctlReparent.shard, vtctlReparent.getLockAction(vtctlReparent.newPrimaryAlias)) } // GetTopoServer implements the ReparentFunctions interface @@ -119,11 +119,11 @@ func (vtctlReparent *VtctlReparentFunctions) PreRecoveryProcesses(ctx context.Co } func (vtctlReparentFunctions *VtctlReparentFunctions) GetWaitReplicasTimeout() time.Duration { - return vtctlReparentFunctions.WaitReplicasTimeout + return vtctlReparentFunctions.waitReplicasTimeout } func (vtctlReparentFunctions *VtctlReparentFunctions) GetWaitForRelayLogsTimeout() time.Duration { - return vtctlReparentFunctions.WaitReplicasTimeout + return vtctlReparentFunctions.waitReplicasTimeout } func (vtctlReparentFunctions *VtctlReparentFunctions) HandleRelayLogFailure(err error) error { @@ -131,7 +131,7 @@ func (vtctlReparentFunctions *VtctlReparentFunctions) HandleRelayLogFailure(err } func (vtctlReparentFunctions *VtctlReparentFunctions) GetIgnoreReplicas() sets.String { - return vtctlReparentFunctions.IgnoreReplicas + return vtctlReparentFunctions.ignoreReplicas } // CheckPrimaryRecoveryType implements the ReparentFunctions interface @@ -174,8 +174,8 @@ func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.C // If we were requested to elect a particular primary, verify it's a valid // candidate (non-zero position, no errant GTIDs) and is at least as // advanced as the winning position. - if vtctlReparent.NewPrimaryAlias != nil { - winningPrimaryTabletAliasStr = topoproto.TabletAliasString(vtctlReparent.NewPrimaryAlias) + if vtctlReparent.newPrimaryAlias != nil { + winningPrimaryTabletAliasStr = topoproto.TabletAliasString(vtctlReparent.newPrimaryAlias) pos, ok := validCandidates[winningPrimaryTabletAliasStr] switch { case !ok: @@ -197,7 +197,7 @@ func (vtctlReparent *VtctlReparentFunctions) PostTabletChangeHook(*topodatapb.Ta // PromotedReplicaIsIdeal implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) PromotedReplicaIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, validCandidates map[string]mysql.Position) bool { - if vtctlReparent.NewPrimaryAlias != nil { + if vtctlReparent.newPrimaryAlias != nil { //explicit request to promote a specific tablet return true } @@ -273,6 +273,6 @@ func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Conte return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } - _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.getLockAction(vtctlReparent.NewPrimaryAlias), rp, tabletMap, statusMap, vtctlReparent, false) + _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.getLockAction(vtctlReparent.newPrimaryAlias), rp, tabletMap, statusMap, vtctlReparent, false) return err } From f333a781e7f0995f4dcd43b7bf9e932b95e221c9 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 6 Sep 2021 16:11:06 +0530 Subject: [PATCH 065/176] remove postponeAll from the vtorcreparentFunctions struct Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/reparent_functions.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index c07a0dcc8d9..7e4d6ee457c 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -60,7 +60,6 @@ type VtOrcReparentFunctions struct { lostReplicas [](*inst.Instance) recoveryAttempted bool hasBestPromotionRule bool - postponedAll bool } func NewVtorcReparentFunctions(analysisEntry inst.ReplicationAnalysis, candidateInstanceKey *inst.InstanceKey, skipProcesses bool, topologyRecovery *TopologyRecovery) *VtOrcReparentFunctions { @@ -170,7 +169,6 @@ func (vtorcReparent *VtOrcReparentFunctions) RestrictValidCandidates(validCandid // FindPrimaryCandidates implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) { - vtorcReparent.postponedAll = false //promotedReplicaIsIdeal := func(promoted *inst.Instance, hasBestPromotionRule bool) bool { // if promoted == nil { // return false @@ -397,7 +395,6 @@ func (vtOrcReparent *VtOrcReparentFunctions) PromotedReplicaIsIdeal(newPrimary, if newPrimaryInst.PromotionRule == inst.MustPromoteRule || newPrimaryInst.PromotionRule == inst.PreferPromoteRule || (vtOrcReparent.hasBestPromotionRule && newPrimaryInst.PromotionRule != inst.MustNotPromoteRule) { AuditTopologyRecovery(vtOrcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: found %+v to be ideal candidate; will optimize recovery", newPrimaryInst.Key)) - vtOrcReparent.postponedAll = true return true } } From d8dd849b516bbf9188d5c5f647f900e77767eacc Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 6 Sep 2021 16:26:08 +0530 Subject: [PATCH 066/176] rename function to PostERSCompletionHook and call it after ERS completes Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 34 ++----------------- .../reparentutil/emergency_reparenter.go | 7 ++-- .../vtctl/reparentutil/reparent_functions.go | 9 ++--- 3 files changed, 7 insertions(+), 43 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 7e4d6ee457c..e573739d694 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -621,8 +621,8 @@ func (vtorcReparent *VtOrcReparentFunctions) CheckIfNeedToOverridePromotion(newP return nil } -// StartReplication implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) StartReplication(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) error { +// PostERSCompletionHook implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) PostERSCompletionHook(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) { // And this is the end; whether successful or not, we're done. resolveRecovery(vtorcReparent.topologyRecovery, vtorcReparent.promotedReplica) // Now, see whether we are successful or not. From this point there's no going back. @@ -632,35 +632,6 @@ func (vtorcReparent *VtOrcReparentFunctions) StartReplication(ctx context.Contex AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: successfully promoted %+v", vtorcReparent.promotedReplica.Key)) AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: promoted server coordinates: %+v", vtorcReparent.promotedReplica.SelfBinlogCoordinates)) - //AuditTopologyRecovery(vtorcReparent.topologyRecovery, "- RecoverDeadMaster: will apply MySQL changes to promoted master") - //{ - // _, err := inst.ResetReplicationOperation(&vtorcReparent.promotedReplica.Key) - // if err != nil { - // // Ugly, but this is important. Let's give it another try - // _, err = inst.ResetReplicationOperation(&vtorcReparent.promotedReplica.Key) - // } - // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying RESET SLAVE ALL on promoted master: success=%t", (err == nil))) - // if err != nil { - // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: NOTE that %+v is promoted even though SHOW SLAVE STATUS may still show it has a master", vtorcReparent.promotedReplica.Key)) - // } - //} - //{ - // count := inst.MasterSemiSync(vtorcReparent.promotedReplica.Key) - // err := inst.SetSemiSyncMaster(&vtorcReparent.promotedReplica.Key, count > 0) - // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying semi-sync %v: success=%t", count > 0, (err == nil))) - // - // // Dont' allow writes if semi-sync settings fail. - // if err == nil { - // _, err := inst.SetReadOnly(&vtorcReparent.promotedReplica.Key, false) - // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying read-only=0 on promoted master: success=%t", (err == nil))) - // } - //} - //// Let's attempt, though we won't necessarily succeed, to set old master as read-only - //go func() { - // _, err := inst.SetReadOnly(&vtorcReparent.analysisEntry.AnalyzedInstanceKey, true) - // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: applying read-only=1 on demoted master: success=%t", (err == nil))) - //}() - kvPairs := inst.GetClusterPrimaryKVPairs(vtorcReparent.analysisEntry.ClusterDetails.ClusterAlias, &vtorcReparent.promotedReplica.Key) AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("Writing KV %+v", kvPairs)) for _, kvPair := range kvPairs { @@ -702,5 +673,4 @@ func (vtorcReparent *VtOrcReparentFunctions) StartReplication(ctx context.Contex } else { recoverDeadPrimaryFailureCounter.Inc(1) } - return nil } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 5d4a9b8df7e..5c23dd14884 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -88,6 +88,8 @@ func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, reparentFunct err = erp.reparentShardLocked(ctx, ev, reparentFunctions) + reparentFunctions.PostERSCompletionHook(ctx, ev, erp.logger, erp.tmc) + return ev, err } @@ -204,12 +206,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } reparentFunctions.PostTabletChangeHook(newPrimary) - if err := reparentFunctions.StartReplication(ctx, ev, erp.logger, erp.tmc); err != nil { - return err - } - ev.NewPrimary = proto.Clone(newPrimary).(*topodatapb.Tablet) - return errInPromotion } diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 6c0faf98613..200442f8216 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -58,7 +58,7 @@ type ( PostTabletChangeHook(*topodatapb.Tablet) GetBetterCandidate(*topodatapb.Tablet, *topodatapb.Tablet, []*topodatapb.Tablet, map[string]*topo.TabletInfo) *topodatapb.Tablet CheckIfNeedToOverridePromotion(newPrimary *topodatapb.Tablet) error - StartReplication(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) error + PostERSCompletionHook(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) } // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions @@ -238,11 +238,8 @@ func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePromotion(newP return nil } -// StartReplication implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) StartReplication(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) error { - // Do the promotion. - //return vtctlReparent.promoteNewPrimary(ctx, ev, logger, tmc) - return nil +// PostERSCompletionHook implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) PostERSCompletionHook(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) { } func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { From db587008562b6894e97f13cca95a157a763034b2 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 6 Sep 2021 16:42:39 +0530 Subject: [PATCH 067/176] handle lint errors Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 21 +++++++++---------- .../reparentutil/emergency_reparenter.go | 3 +-- .../reparentutil/emergency_reparenter_test.go | 4 +--- .../vtctl/reparentutil/reparent_functions.go | 1 - go/vt/vtctl/reparentutil/util.go | 3 +++ 5 files changed, 15 insertions(+), 17 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index e573739d694..6ff221ec1c5 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -57,7 +57,6 @@ type VtOrcReparentFunctions struct { skipProcesses bool topologyRecovery *TopologyRecovery promotedReplica *inst.Instance - lostReplicas [](*inst.Instance) recoveryAttempted bool hasBestPromotionRule bool } @@ -301,21 +300,21 @@ func ChooseCandidate( if err != nil { return emptyReplicas, candidateReplica, false, err } - if candidateReplica != nil { - mostUpToDateReplica := replicas[0] - if candidateReplica.ExecBinlogCoordinates.SmallerThan(&mostUpToDateReplica.ExecBinlogCoordinates) { - log.Warningf("GetCandidateReplica: chosen replica: %+v is behind most-up-to-date replica: %+v", candidateReplica.Key, mostUpToDateReplica.Key) - } + if candidateReplica == nil { + return emptyReplicas, candidateReplica, false, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "could not find a candidate replica for ERS") + } + mostUpToDateReplica := replicas[0] + if candidateReplica.ExecBinlogCoordinates.SmallerThan(&mostUpToDateReplica.ExecBinlogCoordinates) { + log.Warningf("GetCandidateReplica: chosen replica: %+v is behind most-up-to-date replica: %+v", candidateReplica.Key, mostUpToDateReplica.Key) } + log.Debugf("GetCandidateReplica: candidate: %+v, ahead: %d, equal: %d, late: %d, break: %d", candidateReplica.Key, len(aheadReplicas), len(equalReplicas), len(laterReplicas), len(cannotReplicateReplicas)) replicasToMove := append(equalReplicas, laterReplicas...) hasBestPromotionRule = true - if candidateReplica != nil { - for _, replica := range replicasToMove { - if replica.PromotionRule.BetterThan(candidateReplica.PromotionRule) { - hasBestPromotionRule = false - } + for _, replica := range replicasToMove { + if replica.PromotionRule.BetterThan(candidateReplica.PromotionRule) { + hasBestPromotionRule = false } } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 5c23dd14884..755420489db 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -187,8 +187,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve newPrimary = betterCandidate } - var errInPromotion error - errInPromotion = reparentFunctions.CheckIfNeedToOverridePromotion(newPrimary) + errInPromotion := reparentFunctions.CheckIfNeedToOverridePromotion(newPrimary) if errInPromotion != nil { erp.logger.Errorf("have to override promotion because of constraint failure - %v", errInPromotion) newPrimary, err = erp.undoPromotion(ctx, ts, ev, keyspace, shard, prevPrimary.Tablet, "", "", tabletMap, statusMap, reparentFunctions) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 8c095cfdf8c..73d4df507ce 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -1417,9 +1417,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }() } - var err error - - err = tt.vtctlReparentFunctions.promoteNewPrimary(ctx, ev, logger, tt.tmc, tt.newPrimaryTabletAlias, tt.statusMap, tt.tabletMap) + err := tt.vtctlReparentFunctions.promoteNewPrimary(ctx, ev, logger, tt.tmc, tt.newPrimaryTabletAlias, tt.statusMap, tt.tabletMap) if tt.shouldErr { assert.Error(t, err) return diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 200442f8216..7bfeb21f5f8 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -192,7 +192,6 @@ func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.C // PostReplicationChangeHook implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) PostTabletChangeHook(*topodatapb.Tablet) { - return } // PromotedReplicaIsIdeal implements the ReparentFunctions interface diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index e3db302adbc..1a3cb644d47 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -332,6 +332,9 @@ func promotePrimary(ctx context.Context, tmc tmclient.TabletManagerClient, ts *t return err } ti, err := ts.GetTablet(ctx, newPrimary.Alias) + if err != nil { + return err + } newPrimary = ti.Tablet ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) defer cancel() From d86f7f3ba5aab9f847b2159fd086f99a97e4fc3d Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 6 Sep 2021 19:01:41 +0530 Subject: [PATCH 068/176] add topoServer, keyspace and shard to common code Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 17 ------------ go/vt/orchestrator/logic/topology_recovery.go | 9 +++++-- go/vt/vtctl/grpcvtctldserver/server.go | 4 ++- .../reparentutil/emergency_reparenter.go | 27 +++++++++---------- .../reparentutil/emergency_reparenter_test.go | 6 ++--- .../vtctl/reparentutil/reparent_functions.go | 18 ------------- go/vt/wrangler/reparent.go | 4 ++- 7 files changed, 29 insertions(+), 56 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 6ff221ec1c5..c8427ac5a30 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -82,23 +82,6 @@ func (vtorcReparent *VtOrcReparentFunctions) LockShard(ctx context.Context) (con return ctx, unlock, nil } -// GetTopoServer implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) GetTopoServer() *topo.Server { - return ts -} - -// GetKeyspace implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) GetKeyspace() string { - tablet, _ := inst.ReadTablet(vtorcReparent.analysisEntry.AnalyzedInstanceKey) - return tablet.Keyspace -} - -// GetShard implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) GetShard() string { - tablet, _ := inst.ReadTablet(vtorcReparent.analysisEntry.AnalyzedInstanceKey) - return tablet.Shard -} - // CheckIfFixed implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) CheckIfFixed() bool { // Check if someone else fixed the problem. diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index aa97c69a917..2a924964ccb 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -616,6 +616,11 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat if !(forceInstanceRecovery || analysisEntry.ClusterDetails.HasAutomatedPrimaryRecovery) { return false, nil, nil } + tablet, err := inst.ReadTablet(analysisEntry.AnalyzedInstanceKey) + if err != nil { + return false, nil, err + } + topologyRecovery, err = AttemptRecoveryRegistration(&analysisEntry, !forceInstanceRecovery, !forceInstanceRecovery) if topologyRecovery == nil { AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("found an active or recent recovery on %+v. Will not issue another RecoverDeadPrimary.", analysisEntry.AnalyzedInstanceKey)) @@ -624,7 +629,7 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat log.Infof("Analysis: %v, deadprimary %+v", analysisEntry.Analysis, analysisEntry.AnalyzedInstanceKey) reparentFunctions := NewVtorcReparentFunctions(analysisEntry, candidateInstanceKey, skipProcesses, topologyRecovery) - _, err = reparentutil.NewEmergencyReparenter(tmclient.NewTabletManagerClient(), logutil.NewCallbackLogger(func(event *logutilpb.Event) { + _, err = reparentutil.NewEmergencyReparenter(ts, tmclient.NewTabletManagerClient(), logutil.NewCallbackLogger(func(event *logutilpb.Event) { level := event.GetLevel() value := event.GetValue() switch level { @@ -635,7 +640,7 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat default: log.Infof("ERP - %s", value) } - })).ReparentShard(context.Background(), reparentFunctions) + })).ReparentShard(context.Background(), tablet.Keyspace, tablet.Shard, reparentFunctions) return reparentFunctions.recoveryAttempted, topologyRecovery, err } diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 105b28350d6..fe9905b7d0a 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -604,7 +604,9 @@ func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldat logstream = append(logstream, e) }) - ev, err := reparentutil.NewEmergencyReparenter(s.tmc, logger).ReparentShard(ctx, + ev, err := reparentutil.NewEmergencyReparenter(s.ts, s.tmc, logger).ReparentShard(ctx, + req.Keyspace, + req.Shard, reparentutil.NewVtctlReparentFunctions(req.NewPrimary, sets.NewString(ignoreReplicaAliases...), waitReplicasTimeout, diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 755420489db..2ba1ed32b13 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -43,17 +43,19 @@ import ( // EmergencyReparenter performs EmergencyReparentShard operations. type EmergencyReparenter struct { + ts *topo.Server tmc tmclient.TabletManagerClient logger logutil.Logger } // NewEmergencyReparenter returns a new EmergencyReparenter object, ready to -// perform EmergencyReparentShard operations using the given +// perform EmergencyReparentShard operations using the given topo.Server, // TabletManagerClient, and logger. // // Providing a nil logger instance is allowed. -func NewEmergencyReparenter(tmc tmclient.TabletManagerClient, logger logutil.Logger) *EmergencyReparenter { +func NewEmergencyReparenter(ts *topo.Server, tmc tmclient.TabletManagerClient, logger logutil.Logger) *EmergencyReparenter { erp := EmergencyReparenter{ + ts: ts, tmc: tmc, logger: logger, } @@ -69,7 +71,7 @@ func NewEmergencyReparenter(tmc tmclient.TabletManagerClient, logger logutil.Log // ReparentShard performs the EmergencyReparentShard operation on the given // keyspace and shard. -func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, reparentFunctions ReparentFunctions) (*events.Reparent, error) { +func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, shard string, reparentFunctions ReparentFunctions) (*events.Reparent, error) { ctx, unlock, err := reparentFunctions.LockShard(ctx) if err != nil { return nil, err @@ -86,30 +88,27 @@ func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, reparentFunct } }() - err = erp.reparentShardLocked(ctx, ev, reparentFunctions) + err = erp.reparentShardLocked(ctx, ev, keyspace, shard, reparentFunctions) reparentFunctions.PostERSCompletionHook(ctx, ev, erp.logger, erp.tmc) return ev, err } -func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *events.Reparent, reparentFunctions ReparentFunctions) error { +func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *events.Reparent, keyspace, shard string, reparentFunctions ReparentFunctions) error { if reparentFunctions.CheckIfFixed() { return nil } - ts := reparentFunctions.GetTopoServer() - keyspace := reparentFunctions.GetKeyspace() - shard := reparentFunctions.GetShard() - shardInfo, err := ts.GetShard(ctx, keyspace, shard) + shardInfo, err := erp.ts.GetShard(ctx, keyspace, shard) if err != nil { return err } ev.ShardInfo = *shardInfo event.DispatchUpdate(ev, "reading all tablets") - prevPrimary, err := ts.GetTablet(ctx, shardInfo.PrimaryAlias) + prevPrimary, err := erp.ts.GetTablet(ctx, shardInfo.PrimaryAlias) if err != nil { return err } @@ -122,7 +121,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } - tabletMap, err := ts.GetTabletMapForShard(ctx, keyspace, shard) + tabletMap, err := erp.ts.GetTabletMapForShard(ctx, keyspace, shard) if err != nil { return vterrors.Wrapf(err, "failed to get tablet map for %v/%v: %v", keyspace, shard, err) } @@ -164,7 +163,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve isIdeal := reparentFunctions.PromotedReplicaIsIdeal(newPrimary, prevPrimary.Tablet, tabletMap, validCandidates) // TODO := LockAction and RP - validReplacementCandidates, err := promotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, newPrimary, "", "", tabletMap, statusMap, reparentFunctions, isIdeal, true) + validReplacementCandidates, err := promotePrimaryCandidateAndStartReplication(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, "", "", tabletMap, statusMap, reparentFunctions, isIdeal, true) if err != nil { return err } @@ -180,7 +179,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } if !topoproto.TabletAliasEqual(betterCandidate.Alias, newPrimary.Alias) { - err = replaceWithBetterCandidate(ctx, erp.tmc, ts, ev, erp.logger, newPrimary, betterCandidate, "", "", tabletMap, statusMap, reparentFunctions) + err = replaceWithBetterCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, betterCandidate, "", "", tabletMap, statusMap, reparentFunctions) if err != nil { return err } @@ -190,7 +189,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve errInPromotion := reparentFunctions.CheckIfNeedToOverridePromotion(newPrimary) if errInPromotion != nil { erp.logger.Errorf("have to override promotion because of constraint failure - %v", errInPromotion) - newPrimary, err = erp.undoPromotion(ctx, ts, ev, keyspace, shard, prevPrimary.Tablet, "", "", tabletMap, statusMap, reparentFunctions) + newPrimary, err = erp.undoPromotion(ctx, erp.ts, ev, keyspace, shard, prevPrimary.Tablet, "", "", tabletMap, statusMap, reparentFunctions) if err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 73d4df507ce..5aae2cc4c4e 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -60,7 +60,7 @@ func TestNewEmergencyReparenter(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - er := NewEmergencyReparenter(nil, tt.logger) + er := NewEmergencyReparenter(nil, nil, tt.logger) assert.NotNil(t, er.logger, "NewEmergencyReparenter should never result in a nil logger instance on the EmergencyReparenter") }) } @@ -1007,9 +1007,9 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { ctx = lctx // make the reparentShardLocked call use the lock ctx } - erp := NewEmergencyReparenter(tt.tmc, logger) + erp := NewEmergencyReparenter(tt.ts, tt.tmc, logger) - err := erp.reparentShardLocked(ctx, ev, tt.vtctlReparentFunctions) + err := erp.reparentShardLocked(ctx, ev, tt.vtctlReparentFunctions.keyspace, tt.vtctlReparentFunctions.shard, tt.vtctlReparentFunctions) if tt.shouldErr { assert.Error(t, err) return diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 7bfeb21f5f8..a630cda99f2 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -42,9 +42,6 @@ type ( // ReparentFunctions is an interface which has all the functions implementation required for re-parenting ReparentFunctions interface { LockShard(context.Context) (context.Context, func(*error), error) - GetTopoServer() *topo.Server - GetKeyspace() string - GetShard() string CheckIfFixed() bool PreRecoveryProcesses(context.Context) error GetWaitReplicasTimeout() time.Duration @@ -93,21 +90,6 @@ func (vtctlReparent *VtctlReparentFunctions) LockShard(ctx context.Context) (con return vtctlReparent.ts.LockShard(ctx, vtctlReparent.keyspace, vtctlReparent.shard, vtctlReparent.getLockAction(vtctlReparent.newPrimaryAlias)) } -// GetTopoServer implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) GetTopoServer() *topo.Server { - return vtctlReparent.ts -} - -// GetKeyspace implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) GetKeyspace() string { - return vtctlReparent.keyspace -} - -// GetShard implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) GetShard() string { - return vtctlReparent.shard -} - // CheckIfFixed implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) CheckIfFixed() bool { return false diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index 8ceb0c19cba..a2794dfabf4 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -146,8 +146,10 @@ func (wr *Wrangler) PlannedReparentShard(ctx context.Context, keyspace, shard st // EmergencyReparentShard will make the provided tablet the primary for // the shard, when the old primary is completely unreachable. func (wr *Wrangler) EmergencyReparentShard(ctx context.Context, keyspace, shard string, primaryElectTabletAlias *topodatapb.TabletAlias, waitReplicasTimeout time.Duration, ignoredTablets sets.String) (err error) { - _, err = reparentutil.NewEmergencyReparenter(wr.tmc, wr.logger).ReparentShard( + _, err = reparentutil.NewEmergencyReparenter(wr.ts, wr.tmc, wr.logger).ReparentShard( ctx, + keyspace, + shard, reparentutil.NewVtctlReparentFunctions(primaryElectTabletAlias, ignoredTablets, waitReplicasTimeout, From 47cae9df1860bcef955d77b3d74185cc2ca078d0 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 7 Sep 2021 10:08:16 +0530 Subject: [PATCH 069/176] removed keyspace, shard and ts from vtctlReparentFunctions struct Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 2 +- go/vt/vtctl/grpcvtctldserver/server.go | 7 +- .../reparentutil/emergency_reparenter.go | 18 ++- .../reparentutil/emergency_reparenter_test.go | 134 +++++++++++++----- .../vtctl/reparentutil/reparent_functions.go | 18 +-- go/vt/wrangler/reparent.go | 8 +- 6 files changed, 119 insertions(+), 68 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index c8427ac5a30..a4d14b27bd6 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -71,7 +71,7 @@ func NewVtorcReparentFunctions(analysisEntry inst.ReplicationAnalysis, candidate } // LockShard implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) LockShard(ctx context.Context) (context.Context, func(*error), error) { +func (vtorcReparent *VtOrcReparentFunctions) LockShard(ctx context.Context, ts *topo.Server, keyspace string, shard string) (context.Context, func(*error), error) { ctx, unlock, err := LockShard(ctx, vtorcReparent.analysisEntry.AnalyzedInstanceKey) if err != nil { log.Infof("CheckAndRecover: Analysis: %+v, InstanceKey: %+v, candidateInstanceKey: %+v, "+ diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index fe9905b7d0a..3e0b7e4e8da 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -607,12 +607,7 @@ func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldat ev, err := reparentutil.NewEmergencyReparenter(s.ts, s.tmc, logger).ReparentShard(ctx, req.Keyspace, req.Shard, - reparentutil.NewVtctlReparentFunctions(req.NewPrimary, - sets.NewString(ignoreReplicaAliases...), - waitReplicasTimeout, - req.Keyspace, - req.Shard, - s.ts)) + reparentutil.NewVtctlReparentFunctions(req.NewPrimary, sets.NewString(ignoreReplicaAliases...), waitReplicasTimeout)) resp := &vtctldatapb.EmergencyReparentShardResponse{ Keyspace: req.Keyspace, diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 2ba1ed32b13..64ce4131d72 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -72,7 +72,7 @@ func NewEmergencyReparenter(ts *topo.Server, tmc tmclient.TabletManagerClient, l // ReparentShard performs the EmergencyReparentShard operation on the given // keyspace and shard. func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, shard string, reparentFunctions ReparentFunctions) (*events.Reparent, error) { - ctx, unlock, err := reparentFunctions.LockShard(ctx) + ctx, unlock, err := reparentFunctions.LockShard(ctx, erp.ts, keyspace, shard) if err != nil { return nil, err } @@ -108,9 +108,13 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve ev.ShardInfo = *shardInfo event.DispatchUpdate(ev, "reading all tablets") - prevPrimary, err := erp.ts.GetTablet(ctx, shardInfo.PrimaryAlias) - if err != nil { - return err + var prevPrimary *topodatapb.Tablet + if shardInfo.PrimaryAlias != nil { + prevPrimaryInfo, err := erp.ts.GetTablet(ctx, shardInfo.PrimaryAlias) + if err != nil { + return err + } + prevPrimary = prevPrimaryInfo.Tablet } if err := reparentFunctions.PreRecoveryProcesses(ctx); err != nil { @@ -160,7 +164,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } - isIdeal := reparentFunctions.PromotedReplicaIsIdeal(newPrimary, prevPrimary.Tablet, tabletMap, validCandidates) + isIdeal := reparentFunctions.PromotedReplicaIsIdeal(newPrimary, prevPrimary, tabletMap, validCandidates) // TODO := LockAction and RP validReplacementCandidates, err := promotePrimaryCandidateAndStartReplication(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, "", "", tabletMap, statusMap, reparentFunctions, isIdeal, true) @@ -175,7 +179,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve betterCandidate := newPrimary if !isIdeal { - betterCandidate = reparentFunctions.GetBetterCandidate(newPrimary, prevPrimary.Tablet, validReplacementCandidates, tabletMap) + betterCandidate = reparentFunctions.GetBetterCandidate(newPrimary, prevPrimary, validReplacementCandidates, tabletMap) } if !topoproto.TabletAliasEqual(betterCandidate.Alias, newPrimary.Alias) { @@ -189,7 +193,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve errInPromotion := reparentFunctions.CheckIfNeedToOverridePromotion(newPrimary) if errInPromotion != nil { erp.logger.Errorf("have to override promotion because of constraint failure - %v", errInPromotion) - newPrimary, err = erp.undoPromotion(ctx, erp.ts, ev, keyspace, shard, prevPrimary.Tablet, "", "", tabletMap, statusMap, reparentFunctions) + newPrimary, err = erp.undoPromotion(ctx, erp.ts, ev, keyspace, shard, prevPrimary, "", "", tabletMap, statusMap, reparentFunctions) if err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 5aae2cc4c4e..df02c6ef474 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -115,6 +115,8 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { tmc *testutil.TabletManagerClient // setup ts *topo.Server + keyspace string + shard string unlockTopo bool shards []*vtctldatapb.Shard tablets []*topodatapb.Tablet @@ -123,7 +125,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }{ { name: "success", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000102": nil, @@ -219,6 +221,9 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Hostname: "most up-to-date position, wins election", }, }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), shouldErr: false, }, { @@ -228,7 +233,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { vtctlReparentFunctions: NewVtctlReparentFunctions(&topodatapb.TabletAlias{ Cell: "zone1", Uid: 101, - }, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + }, nil, 0), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000101": nil, @@ -323,11 +328,14 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Shard: "-", }, }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), shouldErr: false, }, { name: "success with existing primary", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.PrimaryStatus @@ -425,19 +433,25 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Hostname: "most up-to-date position, wins election", }, }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), shouldErr: false, }, { name: "shard not found", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), tmc: &testutil.TabletManagerClient{}, unlockTopo: true, // we shouldn't try to lock the nonexistent shard shards: nil, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), shouldErr: true, }, { name: "cannot stop replication", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -488,11 +502,14 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Shard: "-", }, }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), shouldErr: true, }, { name: "lost topo lock", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -543,11 +560,14 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Shard: "-", }, }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), shouldErr: true, }, { name: "cannot get reparent candidates", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -610,11 +630,14 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Hostname: "has a zero relay log position", }, }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), shouldErr: true, }, { name: "zero valid reparent candidates", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), tmc: &testutil.TabletManagerClient{}, shards: []*vtctldatapb.Shard{ { @@ -623,11 +646,14 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, }, shouldErr: true, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), }, { name: "error waiting for relay logs to apply", // one replica is going to take a minute to apply relay logs - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, time.Millisecond*50, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, time.Millisecond*50), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -709,13 +735,16 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, }, shouldErr: true, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), }, { name: "requested primary-elect is not in tablet map", vtctlReparentFunctions: NewVtctlReparentFunctions(&topodatapb.TabletAlias{ Cell: "zone1", Uid: 200, - }, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + }, nil, 0), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -791,6 +820,9 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Shard: "-", }, }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), shouldErr: true, }, { @@ -798,8 +830,10 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { vtctlReparentFunctions: NewVtctlReparentFunctions(&topodatapb.TabletAlias{ // we're requesting a tablet that's behind in replication Cell: "zone1", Uid: 102, - }, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), - ts: memorytopo.NewServer("zone1"), + }, nil, 0), + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -887,7 +921,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { &topodatapb.TabletAlias{ Cell: "zone1", Uid: 102, - }, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + }, nil, 0), tmc: &testutil.TabletManagerClient{ PromoteReplicaResults: map[string]struct { Result string @@ -972,6 +1006,9 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Hostname: "not most up-to-date position", }, }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), shouldErr: true, }, } @@ -990,18 +1027,18 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { tablet.Type = topodatapb.TabletType_REPLICA tt.tablets[i] = tablet } - tt.tmc.TopoServer = tt.vtctlReparentFunctions.ts + tt.tmc.TopoServer = tt.ts - testutil.AddShards(ctx, t, tt.vtctlReparentFunctions.ts, tt.shards...) - testutil.AddTablets(ctx, t, tt.vtctlReparentFunctions.ts, nil, tt.tablets...) + testutil.AddShards(ctx, t, tt.ts, tt.shards...) + testutil.AddTablets(ctx, t, tt.ts, nil, tt.tablets...) if !tt.unlockTopo { - lctx, unlock, lerr := tt.vtctlReparentFunctions.ts.LockShard(ctx, tt.vtctlReparentFunctions.keyspace, tt.vtctlReparentFunctions.shard, "test lock") - require.NoError(t, lerr, "could not lock %s/%s for testing", tt.vtctlReparentFunctions.keyspace, tt.vtctlReparentFunctions.shard) + lctx, unlock, lerr := tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "test lock") + require.NoError(t, lerr, "could not lock %s/%s for testing", tt.keyspace, tt.shard) defer func() { unlock(&lerr) - require.NoError(t, lerr, "could not unlock %s/%s after test", tt.vtctlReparentFunctions.keyspace, tt.vtctlReparentFunctions.shard) + require.NoError(t, lerr, "could not unlock %s/%s after test", tt.keyspace, tt.shard) }() ctx = lctx // make the reparentShardLocked call use the lock ctx @@ -1009,7 +1046,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { erp := NewEmergencyReparenter(tt.ts, tt.tmc, logger) - err := erp.reparentShardLocked(ctx, ev, tt.vtctlReparentFunctions.keyspace, tt.vtctlReparentFunctions.shard, tt.vtctlReparentFunctions) + err := erp.reparentShardLocked(ctx, ev, tt.keyspace, tt.shard, tt.vtctlReparentFunctions) if tt.shouldErr { assert.Error(t, err) return @@ -1029,13 +1066,16 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { tmc *testutil.TabletManagerClient unlockTopo bool newPrimaryTabletAlias string + ts *topo.Server + keyspace string + shard string tabletMap map[string]*topo.TabletInfo statusMap map[string]*replicationdatapb.StopReplicationStatus shouldErr bool }{ { name: "success", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, sets.NewString("zone1-0000000404"), 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, sets.NewString("zone1-0000000404"), 0), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1106,23 +1146,29 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), shouldErr: false, }, { name: "primary not in tablet map", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), tmc: &testutil.TabletManagerClient{}, newPrimaryTabletAlias: "zone2-0000000200", tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": {}, "zone1-0000000101": {}, }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, shouldErr: true, }, { name: "PromoteReplica error", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ PromoteReplicaResults: map[string]struct { Result string @@ -1153,11 +1199,14 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), shouldErr: true, }, { name: "lost topology lock", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ PromoteReplicaResults: map[string]struct { Result string @@ -1189,11 +1238,14 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), shouldErr: true, }, { name: "cannot repopulate reparent journal on new primary", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": assert.AnError, @@ -1227,11 +1279,14 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), shouldErr: true, }, { name: "all replicas failing to SetMaster does fail the promotion", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1278,11 +1333,14 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), shouldErr: true, }, { name: "all replicas slow to SetMaster does fail the promotion", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, time.Millisecond*10, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, time.Millisecond*10), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1334,10 +1392,13 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, shouldErr: true, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), }, { name: "one replica failing to SetMaster does not fail the promotion", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0, "testkeyspace", "-", memorytopo.NewServer("zone1")), + vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1383,6 +1444,9 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), shouldErr: false, }, } @@ -1397,9 +1461,9 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { logger := logutil.NewMemoryLogger() ev := &events.Reparent{} - testutil.AddShards(ctx, t, tt.vtctlReparentFunctions.ts, &vtctldatapb.Shard{ - Keyspace: tt.vtctlReparentFunctions.keyspace, - Name: tt.vtctlReparentFunctions.shard, + testutil.AddShards(ctx, t, tt.ts, &vtctldatapb.Shard{ + Keyspace: tt.keyspace, + Name: tt.shard, }) if !tt.unlockTopo { @@ -1408,16 +1472,16 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { lerr error ) - ctx, unlock, lerr = tt.vtctlReparentFunctions.ts.LockShard(ctx, tt.vtctlReparentFunctions.keyspace, tt.vtctlReparentFunctions.shard, "test lock") - require.NoError(t, lerr, "could not lock %s/%s for test", tt.vtctlReparentFunctions.keyspace, tt.vtctlReparentFunctions.shard) + ctx, unlock, lerr = tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "test lock") + require.NoError(t, lerr, "could not lock %s/%s for test", tt.keyspace, tt.shard) defer func() { unlock(&lerr) - require.NoError(t, lerr, "could not unlock %s/%s after test", tt.vtctlReparentFunctions.keyspace, tt.vtctlReparentFunctions.shard) + require.NoError(t, lerr, "could not unlock %s/%s after test", tt.keyspace, tt.shard) }() } - err := tt.vtctlReparentFunctions.promoteNewPrimary(ctx, ev, logger, tt.tmc, tt.newPrimaryTabletAlias, tt.statusMap, tt.tabletMap) + err := tt.vtctlReparentFunctions.promoteNewPrimary(ctx, ev, logger, tt.tmc, tt.newPrimaryTabletAlias, tt.statusMap, tt.tabletMap, tt.keyspace, tt.shard) if tt.shouldErr { assert.Error(t, err) return diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index a630cda99f2..4052b1fc3a0 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -41,7 +41,7 @@ import ( type ( // ReparentFunctions is an interface which has all the functions implementation required for re-parenting ReparentFunctions interface { - LockShard(context.Context) (context.Context, func(*error), error) + LockShard(context.Context, *topo.Server, string, string) (context.Context, func(*error), error) CheckIfFixed() bool PreRecoveryProcesses(context.Context) error GetWaitReplicasTimeout() time.Duration @@ -63,9 +63,6 @@ type ( newPrimaryAlias *topodatapb.TabletAlias ignoreReplicas sets.String waitReplicasTimeout time.Duration - keyspace string - shard string - ts *topo.Server } ) @@ -74,20 +71,17 @@ var ( ) // NewVtctlReparentFunctions creates a new VtctlReparentFunctions which is used in ERS ans PRS -func NewVtctlReparentFunctions(newPrimaryAlias *topodatapb.TabletAlias, ignoreReplicas sets.String, waitReplicasTimeout time.Duration, keyspace string, shard string, ts *topo.Server) *VtctlReparentFunctions { +func NewVtctlReparentFunctions(newPrimaryAlias *topodatapb.TabletAlias, ignoreReplicas sets.String, waitReplicasTimeout time.Duration) *VtctlReparentFunctions { return &VtctlReparentFunctions{ newPrimaryAlias: newPrimaryAlias, ignoreReplicas: ignoreReplicas, waitReplicasTimeout: waitReplicasTimeout, - keyspace: keyspace, - shard: shard, - ts: ts, } } // LockShard implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) LockShard(ctx context.Context) (context.Context, func(*error), error) { - return vtctlReparent.ts.LockShard(ctx, vtctlReparent.keyspace, vtctlReparent.shard, vtctlReparent.getLockAction(vtctlReparent.newPrimaryAlias)) +func (vtctlReparent *VtctlReparentFunctions) LockShard(ctx context.Context, ts *topo.Server, keyspace string, shard string) (context.Context, func(*error), error) { + return ts.LockShard(ctx, keyspace, shard, vtctlReparent.getLockAction(vtctlReparent.newPrimaryAlias)) } // CheckIfFixed implements the ReparentFunctions interface @@ -233,7 +227,7 @@ func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topo return action } -func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, winningPrimaryTabletAliasStr string, statusMap map[string]*replicationdatapb.StopReplicationStatus, tabletMap map[string]*topo.TabletInfo) error { +func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, winningPrimaryTabletAliasStr string, statusMap map[string]*replicationdatapb.StopReplicationStatus, tabletMap map[string]*topo.TabletInfo, keyspace, shard string) error { logger.Infof("promoting tablet %v to master", winningPrimaryTabletAliasStr) event.DispatchUpdate(ev, "promoting replica") @@ -247,7 +241,7 @@ func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Conte return vterrors.Wrapf(err, "master-elect tablet %v failed to be upgraded to master: %v", winningPrimaryTabletAliasStr, err) } - if err := topo.CheckShardLocked(ctx, vtctlReparent.keyspace, vtctlReparent.shard); err != nil { + if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index a2794dfabf4..ade96c09c14 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -150,13 +150,7 @@ func (wr *Wrangler) EmergencyReparentShard(ctx context.Context, keyspace, shard ctx, keyspace, shard, - reparentutil.NewVtctlReparentFunctions(primaryElectTabletAlias, - ignoredTablets, - waitReplicasTimeout, - keyspace, - shard, - wr.ts, - )) + reparentutil.NewVtctlReparentFunctions(primaryElectTabletAlias, ignoredTablets, waitReplicasTimeout)) return err } From c347dc982e3e20401b577a7be7886bafb348d50e Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 7 Sep 2021 10:54:10 +0530 Subject: [PATCH 070/176] bug fix in test Signed-off-by: Manan Gupta --- go/vt/vtctl/grpcvtctldserver/server_slow_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server_slow_test.go b/go/vt/vtctl/grpcvtctldserver/server_slow_test.go index 17d4ec7d054..001daec8cf5 100644 --- a/go/vt/vtctl/grpcvtctldserver/server_slow_test.go +++ b/go/vt/vtctl/grpcvtctldserver/server_slow_test.go @@ -42,7 +42,7 @@ func TestEmergencyReparentShardSlow(t *testing.T) { tests := []struct { name string ts *topo.Server - tmc tmclient.TabletManagerClient + tmc *testutil.TabletManagerClient tablets []*topodatapb.Tablet req *vtctldatapb.EmergencyReparentShardRequest @@ -296,6 +296,8 @@ func TestEmergencyReparentShardSlow(t *testing.T) { SkipShardCreation: false, }, tt.tablets...) + tt.tmc.TopoServer = tt.ts + vtctld := testutil.NewVtctldServerWithTabletManagerClient(t, tt.ts, tt.tmc, func(ts *topo.Server) vtctlservicepb.VtctldServer { return NewVtctldServer(ts) }) From ed441e44df9bf30c3763bff5f46f37ee8e69cc5f Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 7 Sep 2021 10:55:00 +0530 Subject: [PATCH 071/176] added inline comments for starting functions of ERS Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/reparent_functions.go | 12 +++++------- go/vt/vtctl/reparentutil/emergency_reparenter.go | 14 +++++++++++--- go/vt/vtctl/reparentutil/reparent_functions.go | 9 ++++++--- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index a4d14b27bd6..3810c13a3db 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -71,12 +71,10 @@ func NewVtorcReparentFunctions(analysisEntry inst.ReplicationAnalysis, candidate } // LockShard implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) LockShard(ctx context.Context, ts *topo.Server, keyspace string, shard string) (context.Context, func(*error), error) { +func (vtorcReparent *VtOrcReparentFunctions) LockShard(ctx context.Context, logger logutil.Logger, ts *topo.Server, keyspace string, shard string) (context.Context, func(*error), error) { ctx, unlock, err := LockShard(ctx, vtorcReparent.analysisEntry.AnalyzedInstanceKey) if err != nil { - log.Infof("CheckAndRecover: Analysis: %+v, InstanceKey: %+v, candidateInstanceKey: %+v, "+ - "skipProcesses: %v: NOT detecting/recovering host, could not obtain shard lock (%v)", - vtorcReparent.analysisEntry.Analysis, vtorcReparent.analysisEntry.AnalyzedInstanceKey, vtorcReparent.candidateInstanceKey, vtorcReparent.skipProcesses, err) + logger.Infof("could not obtain shard lock (%v)", err) return nil, nil, err } return ctx, unlock, nil @@ -93,21 +91,21 @@ func (vtorcReparent *VtOrcReparentFunctions) CheckIfFixed() bool { // TODO(sougou): see if we have to reset the cluster as healthy. return true } - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("will handle DeadMaster event on %+v", vtorcReparent.analysisEntry.ClusterDetails.ClusterName)) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("will handle DeadPrimary event on %+v", vtorcReparent.analysisEntry.ClusterDetails.ClusterName)) recoverDeadPrimaryCounter.Inc(1) return false } // PreRecoveryProcesses implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) PreRecoveryProcesses(ctx context.Context) error { - inst.AuditOperation("recover-dead-master", &vtorcReparent.analysisEntry.AnalyzedInstanceKey, "problem found; will recover") + inst.AuditOperation("recover-dead-primary", &vtorcReparent.analysisEntry.AnalyzedInstanceKey, "problem found; will recover") if !vtorcReparent.skipProcesses { if err := executeProcesses(config.Config.PreFailoverProcesses, "PreFailoverProcesses", vtorcReparent.topologyRecovery, true); err != nil { return vtorcReparent.topologyRecovery.AddError(err) } } - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: will recover %+v", vtorcReparent.analysisEntry.AnalyzedInstanceKey)) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: will recover %+v", vtorcReparent.analysisEntry.AnalyzedInstanceKey)) return nil } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 64ce4131d72..50414267424 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -72,12 +72,15 @@ func NewEmergencyReparenter(ts *topo.Server, tmc tmclient.TabletManagerClient, l // ReparentShard performs the EmergencyReparentShard operation on the given // keyspace and shard. func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, shard string, reparentFunctions ReparentFunctions) (*events.Reparent, error) { - ctx, unlock, err := reparentFunctions.LockShard(ctx, erp.ts, keyspace, shard) + // First step is to lock the shard for the given operation + ctx, unlock, err := reparentFunctions.LockShard(ctx, erp.logger, erp.ts, keyspace, shard) if err != nil { return nil, err } + // defer the unlock-shard function defer unlock(&err) + // dispatch success or failure of ERS ev := &events.Reparent{} defer func() { switch err { @@ -88,6 +91,7 @@ func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, sha } }() + // run ERS with shard already locked err = erp.reparentShardLocked(ctx, ev, keyspace, shard, reparentFunctions) reparentFunctions.PostERSCompletionHook(ctx, ev, erp.logger, erp.tmc) @@ -95,19 +99,21 @@ func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, sha return ev, err } +// reparentShardLocked performs Emergency Reparent Shard operation assuming that the shard is already locked func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *events.Reparent, keyspace, shard string, reparentFunctions ReparentFunctions) error { - + // check whether ERS is required or it has been fixed via an external agent if reparentFunctions.CheckIfFixed() { return nil } + // get the shard information from the topology server shardInfo, err := erp.ts.GetShard(ctx, keyspace, shard) if err != nil { return err } ev.ShardInfo = *shardInfo - event.DispatchUpdate(ev, "reading all tablets") + // get the previous primary according to the topology server var prevPrimary *topodatapb.Tablet if shardInfo.PrimaryAlias != nil { prevPrimaryInfo, err := erp.ts.GetTablet(ctx, shardInfo.PrimaryAlias) @@ -117,6 +123,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve prevPrimary = prevPrimaryInfo.Tablet } + // run the pre recovery processes if err := reparentFunctions.PreRecoveryProcesses(ctx); err != nil { return err } @@ -125,6 +132,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } + event.DispatchUpdate(ev, "reading all tablets") tabletMap, err := erp.ts.GetTabletMapForShard(ctx, keyspace, shard) if err != nil { return vterrors.Wrapf(err, "failed to get tablet map for %v/%v: %v", keyspace, shard, err) diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 4052b1fc3a0..56bf5875a8c 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -41,14 +41,14 @@ import ( type ( // ReparentFunctions is an interface which has all the functions implementation required for re-parenting ReparentFunctions interface { - LockShard(context.Context, *topo.Server, string, string) (context.Context, func(*error), error) + LockShard(context.Context, logutil.Logger, *topo.Server, string, string) (context.Context, func(*error), error) CheckIfFixed() bool PreRecoveryProcesses(context.Context) error + CheckPrimaryRecoveryType() error GetWaitReplicasTimeout() time.Duration GetWaitForRelayLogsTimeout() time.Duration HandleRelayLogFailure(err error) error GetIgnoreReplicas() sets.String - CheckPrimaryRecoveryType() error RestrictValidCandidates(map[string]mysql.Position, map[string]*topo.TabletInfo) (map[string]mysql.Position, error) FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) PromotedReplicaIsIdeal(*topodatapb.Tablet, *topodatapb.Tablet, map[string]*topo.TabletInfo, map[string]mysql.Position) bool @@ -80,17 +80,20 @@ func NewVtctlReparentFunctions(newPrimaryAlias *topodatapb.TabletAlias, ignoreRe } // LockShard implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) LockShard(ctx context.Context, ts *topo.Server, keyspace string, shard string) (context.Context, func(*error), error) { +func (vtctlReparent *VtctlReparentFunctions) LockShard(ctx context.Context, logger logutil.Logger, ts *topo.Server, keyspace string, shard string) (context.Context, func(*error), error) { return ts.LockShard(ctx, keyspace, shard, vtctlReparent.getLockAction(vtctlReparent.newPrimaryAlias)) } // CheckIfFixed implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) CheckIfFixed() bool { + // For vtctl command, we know there is no other third party to fix this. + // If a user has called this command, then we should execute EmergencyReparentShard return false } // PreRecoveryProcesses implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) PreRecoveryProcesses(ctx context.Context) error { + // there are no pre-recovery processes to be run return nil } From de8150548e0dd7084585d802640530bf810bbbb9 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 7 Sep 2021 10:55:45 +0530 Subject: [PATCH 072/176] remove unused function Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/main_test.go | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/go/test/endtoend/vtorc/main_test.go b/go/test/endtoend/vtorc/main_test.go index d205c5f554a..1e540f15415 100644 --- a/go/test/endtoend/vtorc/main_test.go +++ b/go/test/endtoend/vtorc/main_test.go @@ -393,29 +393,6 @@ func shardPrimaryTablet(t *testing.T, cluster *cluster.LocalProcessCluster, keys } } -// checkShardNoPrimaryTablet waits till the given shard has no primary tablet. It times out after 1 minute -func checkShardNoPrimaryTablet(t *testing.T, cluster *cluster.LocalProcessCluster, keyspace *cluster.Keyspace, shard *cluster.Shard) { - start := time.Now() - for { - now := time.Now() - if now.Sub(start) > time.Second*60 { - assert.FailNow(t, "failed to find a point in time when shard had no primary before timeout") - } - result, err := cluster.VtctlclientProcess.ExecuteCommandWithOutput("GetShard", fmt.Sprintf("%s/%s", keyspace.Name, shard.Name)) - assert.Nil(t, err) - - var shardInfo topodatapb.Shard - err = json2.Unmarshal([]byte(result), &shardInfo) - assert.Nil(t, err) - if shardInfo.PrimaryAlias == nil { - return - } - log.Warningf("Shard %v/%v has a primary yet, sleep for 1 second\n", keyspace.Name, shard.Name) - time.Sleep(time.Second) - continue - } -} - // Makes sure the tablet type is primary, and its health check agrees. func checkPrimaryTablet(t *testing.T, cluster *cluster.LocalProcessCluster, tablet *cluster.Vttablet, checkServing bool) { start := time.Now() From 3fb0e600544acbf6c4138ad25c7d5e02d680bf19 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 7 Sep 2021 11:03:58 +0530 Subject: [PATCH 073/176] add change type before calling promote replica in prs Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/planned_reparenter.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/go/vt/vtctl/reparentutil/planned_reparenter.go b/go/vt/vtctl/reparentutil/planned_reparenter.go index 7c6f591900a..8cf73634914 100644 --- a/go/vt/vtctl/reparentutil/planned_reparenter.go +++ b/go/vt/vtctl/reparentutil/planned_reparenter.go @@ -286,6 +286,10 @@ func (pr *PlannedReparenter) performGracefulPromotion( promoteCtx, promoteCancel := context.WithTimeout(ctx, opts.WaitReplicasTimeout) defer promoteCancel() + err = pr.tmc.ChangeType(promoteCtx, primaryElect, topodatapb.TabletType_PRIMARY) + if err != nil { + return "", vterrors.Wrapf(err, "primary-elect tablet %v failed to change type to primary; please try again", primaryElectAliasStr) + } rp, err := pr.tmc.PromoteReplica(promoteCtx, primaryElect) if err != nil { return "", vterrors.Wrapf(err, "primary-elect tablet %v failed to be promoted to primary; please try again", primaryElectAliasStr) From d2fcb06a05c393072b7f43c07503ac7790aae04c Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 7 Sep 2021 12:30:07 +0530 Subject: [PATCH 074/176] added inline comments upto check for ideal candidate Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 158 ++++++------------ .../reparentutil/emergency_reparenter.go | 16 +- .../vtctl/reparentutil/reparent_functions.go | 50 +++--- 3 files changed, 88 insertions(+), 136 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 3810c13a3db..5b426c21dd8 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -109,32 +109,41 @@ func (vtorcReparent *VtOrcReparentFunctions) PreRecoveryProcesses(ctx context.Co return nil } +// GetWaitReplicasTimeout implements the ReparentFunctions interface // TODO : Discuss correct way func (vtorcReparent *VtOrcReparentFunctions) GetWaitReplicasTimeout() time.Duration { return 1 * time.Second } +// GetWaitForRelayLogsTimeout implements the ReparentFunctions interface // TODO : Discuss correct way func (vtorcReparent *VtOrcReparentFunctions) GetWaitForRelayLogsTimeout() time.Duration { return 1 * time.Second } +// HandleRelayLogFailure implements the ReparentFunctions interface // TODO : Discuss correct way -func (vtorcReparent *VtOrcReparentFunctions) HandleRelayLogFailure(err error) error { - log.Infof("failed to apply all relay logs - %v", err) +func (vtorcReparent *VtOrcReparentFunctions) HandleRelayLogFailure(logger logutil.Logger, err error) error { + // We do not want to throw an error from vtorc, since there could be replicas which are + // so far lagging that they may take days to apply all their relay logs + logger.Infof("failed to apply all relay logs - %v", err) return nil } +// GetIgnoreReplicas implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) GetIgnoreReplicas() sets.String { + // vtorc does not ignore any replicas return nil } // CheckPrimaryRecoveryType implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) CheckPrimaryRecoveryType() error { +func (vtorcReparent *VtOrcReparentFunctions) CheckPrimaryRecoveryType(logger logutil.Logger) error { vtorcReparent.topologyRecovery.RecoveryType = GetPrimaryRecoveryType(&vtorcReparent.topologyRecovery.AnalysisEntry) - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: masterRecoveryType=%+v", vtorcReparent.topologyRecovery.RecoveryType)) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: primaryRecoveryType=%+v", vtorcReparent.topologyRecovery.RecoveryType)) if vtorcReparent.topologyRecovery.RecoveryType != PrimaryRecoveryGTID { - return vtorcReparent.topologyRecovery.AddError(log.Errorf("RecoveryType unknown/unsupported")) + err := fmt.Errorf("RecoveryType unknown/unsupported") + logger.Error(err) + return vtorcReparent.topologyRecovery.AddError(err) } return nil } @@ -147,30 +156,10 @@ func (vtorcReparent *VtOrcReparentFunctions) RestrictValidCandidates(validCandid return validCandidates, nil } -// FindPrimaryCandidates implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) { - //promotedReplicaIsIdeal := func(promoted *inst.Instance, hasBestPromotionRule bool) bool { - // if promoted == nil { - // return false - // } - // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: promotedReplicaIsIdeal(%+v)", promoted.Key)) - // if vtorcReparent.candidateInstanceKey != nil { //explicit request to promote a specific server - // return promoted.Key.Equals(vtorcReparent.candidateInstanceKey) - // } - // if promoted.DataCenter == vtorcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstanceDataCenter && - // promoted.PhysicalEnvironment == vtorcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstancePhysicalEnvironment { - // if promoted.PromotionRule == inst.MustPromoteRule || promoted.PromotionRule == inst.PreferPromoteRule || - // (hasBestPromotionRule && promoted.PromotionRule != inst.MustNotPromoteRule) { - // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: found %+v to be ideal candidate; will optimize recovery", promoted.Key)) - // vtorcReparent.postponedAll = true - // return true - // } - // } - // return false - //} - - AuditTopologyRecovery(vtorcReparent.topologyRecovery, "RecoverDeadMaster: regrouping replicas via GTID") - lostReplicas, promotedReplica, hasBestPromotionRule, err := ChooseCandidate(tmc, &vtorcReparent.analysisEntry.AnalyzedInstanceKey, validCandidates, tabletMap) +// FindPrimaryCandidate implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidate(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) { + AuditTopologyRecovery(vtorcReparent.topologyRecovery, "RecoverDeadPrimary: regrouping replicas via GTID") + lostReplicas, promotedReplica, hasBestPromotionRule, err := ChooseCandidate(&vtorcReparent.analysisEntry.AnalyzedInstanceKey, validCandidates, tabletMap, logger) vtorcReparent.topologyRecovery.AddError(err) vtorcReparent.hasBestPromotionRule = hasBestPromotionRule if err != nil { @@ -182,19 +171,20 @@ func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.C } for _, replica := range lostReplicas { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: - lost replica: %+v", replica.Key)) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: - lost replica: %+v", replica.Key)) } + // detach lost replicas if the configuration specifies it if promotedReplica != nil && len(lostReplicas) > 0 && config.Config.DetachLostReplicasAfterPrimaryFailover { postponedFunction := func() error { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: lost %+v replicas during recovery process; detaching them", len(lostReplicas))) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: lost %+v replicas during recovery process; detaching them", len(lostReplicas))) for _, replica := range lostReplicas { replica := replica inst.DetachReplicaPrimaryHost(&replica.Key) } return nil } - vtorcReparent.topologyRecovery.AddPostponedFunction(postponedFunction, fmt.Sprintf("RecoverDeadMaster, detach %+v lost replicas", len(lostReplicas))) + vtorcReparent.topologyRecovery.AddPostponedFunction(postponedFunction, fmt.Sprintf("RecoverDeadPrimary, detach %+v lost replicas", len(lostReplicas))) } func() error { @@ -208,12 +198,13 @@ func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.C return nil }() - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: %d postponed functions", vtorcReparent.topologyRecovery.PostponedFunctionsContainer.Len())) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: %d postponed functions", vtorcReparent.topologyRecovery.PostponedFunctionsContainer.Len())) vtorcReparent.promotedReplica = promotedReplica vtorcReparent.topologyRecovery.LostReplicas.AddInstances(lostReplicas) vtorcReparent.recoveryAttempted = true + // we now remove the lost replicas from the tabletMap so that we do not try to move them below the newly elected candidate now tabletMapWithoutLostReplicas := map[string]*topo.TabletInfo{} for alias, info := range tabletMap { @@ -233,12 +224,12 @@ func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidates(ctx context.C return newPrimary, tabletMapWithoutLostReplicas, nil } -// ChooseCandidate will choose a candidate replica of a given instance, and take its siblings using GTID +// ChooseCandidate will choose a candidate replica of a given instance and also returns whether the chosen candidate has the best promotion rule or not func ChooseCandidate( - tmc tmclient.TabletManagerClient, - masterKey *inst.InstanceKey, + primaryKey *inst.InstanceKey, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, + logger logutil.Logger, ) ( lostReplicas [](*inst.Instance), candidateReplica *inst.Instance, @@ -246,12 +237,10 @@ func ChooseCandidate( err error, ) { var emptyReplicas [](*inst.Instance) - var unmovedReplicas [](*inst.Instance) - //var movedReplicas [](*inst.Instance) dataCenterHint := "" - if master, _, _ := inst.ReadInstance(masterKey); master != nil { - dataCenterHint = master.DataCenter + if primary, _, _ := inst.ReadInstance(primaryKey); primary != nil { + dataCenterHint = primary.DataCenter } var replicas [](*inst.Instance) @@ -266,7 +255,7 @@ func ChooseCandidate( Port: int(candidateInfo.MysqlPort), }) if err != nil { - log.Errorf("%v", err) + logger.Errorf("%v", err) return emptyReplicas, candidateReplica, false, err } replicas = append(replicas, candidateInstance) @@ -274,7 +263,7 @@ func ChooseCandidate( inst.SortInstancesDataCenterHint(replicas, dataCenterHint) for _, replica := range replicas { - log.Debugf("- sorted replica: %+v %+v", replica.Key, replica.ExecBinlogCoordinates) + logger.Infof("ChooseCandidate - sorted replica: %+v %+v", replica.Key, replica.ExecBinlogCoordinates) } candidateReplica, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := inst.ChooseCandidateReplica(replicas) @@ -286,10 +275,10 @@ func ChooseCandidate( } mostUpToDateReplica := replicas[0] if candidateReplica.ExecBinlogCoordinates.SmallerThan(&mostUpToDateReplica.ExecBinlogCoordinates) { - log.Warningf("GetCandidateReplica: chosen replica: %+v is behind most-up-to-date replica: %+v", candidateReplica.Key, mostUpToDateReplica.Key) + logger.Warningf("ChooseCandidate: chosen replica: %+v is behind most-up-to-date replica: %+v", candidateReplica.Key, mostUpToDateReplica.Key) } - log.Debugf("GetCandidateReplica: candidate: %+v, ahead: %d, equal: %d, late: %d, break: %d", candidateReplica.Key, len(aheadReplicas), len(equalReplicas), len(laterReplicas), len(cannotReplicateReplicas)) + logger.Infof("ChooseCandidate: candidate: %+v, ahead: %d, equal: %d, late: %d, break: %d", candidateReplica.Key, len(aheadReplicas), len(equalReplicas), len(laterReplicas), len(cannotReplicateReplicas)) replicasToMove := append(equalReplicas, laterReplicas...) hasBestPromotionRule = true @@ -299,82 +288,28 @@ func ChooseCandidate( } } - //now := time.Now().UnixNano() - //if err := inst.SwitchMaster(candidateReplica.Key, *masterKey); err != nil { - // return emptyReplicas, candidateReplica, err - //} - // - //candidateReplicaTablet, err := inst.ReadTablet(candidateReplica.Key) - //if err != nil { - // return emptyReplicas, candidateReplica, err - //} - // - //moveGTIDFunc := func() error { - // log.Debugf("RegroupReplicasGTID: working on %d replicas", len(replicasToMove)) - // - // for _, instance := range replicasToMove { - // tablet, err := inst.ReadTablet(instance.Key) - // if err != nil { - // goto Cleanup - // } - // - // if maintenanceToken, merr := inst.BeginMaintenance(&instance.Key, inst.GetMaintenanceOwner(), fmt.Sprintf("move below %+v", candidateReplica.Key)); merr != nil { - // err = fmt.Errorf("Cannot begin maintenance on %+v: %v", instance.Key, merr) - // goto Cleanup - // } else { - // defer inst.EndMaintenance(maintenanceToken) - // } - // - // err = tmc.SetReplicationSource(context.Background(), tablet, candidateReplicaTablet.Alias, now, "", false) - // if err != nil { - // unmovedReplicas = append(unmovedReplicas, instance) - // err = vterrors.Wrapf(err, "tablet %v SetMaster failed: %v", tablet.Alias, err) - // goto Cleanup - // } - // movedReplicas = append(movedReplicas, instance) - // - // Cleanup: - // tmc.StartReplication(context.Background(), tablet) - // return err - // } - // - unmovedReplicas = append(unmovedReplicas, aheadReplicas...) - unmovedReplicas = append(unmovedReplicas, cannotReplicateReplicas...) - // if err != nil { - // log.Errore(err) - // } - // return nil - //} - // - //if postponedFunctionsContainer != nil && postponeAllMatchOperations != nil && postponeAllMatchOperations(candidateReplica, hasBestPromotionRule) { - // postponedFunctionsContainer.AddPostponedFunction(moveGTIDFunc, fmt.Sprintf("regroup-replicas-gtid %+v", candidateReplica.Key)) - //} else { - // err = moveGTIDFunc() - //} - // - //inst.StartReplication(&candidateReplica.Key) - // - //log.Debugf("RegroupReplicasGTID: done") - //inst.AuditOperation("regroup-replicas-gtid", masterKey, fmt.Sprintf("regrouped replicas of %+v via GTID; promoted %+v", *masterKey, candidateReplica.Key)) - return unmovedReplicas, candidateReplica, hasBestPromotionRule, err + lostReplicas = append(lostReplicas, aheadReplicas...) + lostReplicas = append(lostReplicas, cannotReplicateReplicas...) + return lostReplicas, candidateReplica, hasBestPromotionRule, nil } // PromotedReplicaIsIdeal implements the ReparentFunctions interface -func (vtOrcReparent *VtOrcReparentFunctions) PromotedReplicaIsIdeal(newPrimary, oldPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, validCandidates map[string]mysql.Position) bool { - AuditTopologyRecovery(vtOrcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: promotedReplicaIsIdeal(%+v)", newPrimary.Alias)) +func (vtorcReparent *VtOrcReparentFunctions) PromotedReplicaIsIdeal(newPrimary, oldPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, validCandidates map[string]mysql.Position) bool { + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: promotedReplicaIsIdeal(%+v)", newPrimary.Alias)) newPrimaryKey := &inst.InstanceKey{ Hostname: newPrimary.MysqlHostname, Port: int(newPrimary.MysqlPort), } newPrimaryInst, _, _ := inst.ReadInstance(newPrimaryKey) - if vtOrcReparent.candidateInstanceKey != nil { //explicit request to promote a specific server - return newPrimaryKey.Equals(vtOrcReparent.candidateInstanceKey) + if vtorcReparent.candidateInstanceKey != nil { + // explicit request to promote a specific server + return newPrimaryKey.Equals(vtorcReparent.candidateInstanceKey) } - if newPrimaryInst.DataCenter == vtOrcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstanceDataCenter && - newPrimaryInst.PhysicalEnvironment == vtOrcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstancePhysicalEnvironment { + if newPrimaryInst.DataCenter == vtorcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstanceDataCenter && + newPrimaryInst.PhysicalEnvironment == vtorcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstancePhysicalEnvironment { if newPrimaryInst.PromotionRule == inst.MustPromoteRule || newPrimaryInst.PromotionRule == inst.PreferPromoteRule || - (vtOrcReparent.hasBestPromotionRule && newPrimaryInst.PromotionRule != inst.MustNotPromoteRule) { - AuditTopologyRecovery(vtOrcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: found %+v to be ideal candidate; will optimize recovery", newPrimaryInst.Key)) + (vtorcReparent.hasBestPromotionRule && newPrimaryInst.PromotionRule != inst.MustNotPromoteRule) { + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: found %+v to be ideal candidate; will optimize recovery", newPrimaryInst.Key)) return true } } @@ -382,7 +317,7 @@ func (vtOrcReparent *VtOrcReparentFunctions) PromotedReplicaIsIdeal(newPrimary, } // PostReplicationChangeHook implements the ReparentFunctions interface -func (vtOrcReparent *VtOrcReparentFunctions) PostTabletChangeHook(tablet *topodatapb.Tablet) { +func (vtorcReparent *VtOrcReparentFunctions) PostTabletChangeHook(tablet *topodatapb.Tablet) { instanceKey := &inst.InstanceKey{ Hostname: tablet.MysqlHostname, Port: int(tablet.MysqlPort), @@ -544,6 +479,7 @@ func getReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, newPri return newPrimary } +// TODO: handle error from this func getInstanceFromTablet(tablet *topodatapb.Tablet) *inst.Instance { instance, _, _ := inst.ReadInstance(&inst.InstanceKey{ Hostname: tablet.MysqlHostname, diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 50414267424..dd1a4410b13 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -128,29 +128,36 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } - if err := reparentFunctions.CheckPrimaryRecoveryType(); err != nil { + // check that the primary recovery type is a valid one + if err := reparentFunctions.CheckPrimaryRecoveryType(erp.logger); err != nil { return err } + // read all the tablets and there information event.DispatchUpdate(ev, "reading all tablets") tabletMap, err := erp.ts.GetTabletMapForShard(ctx, keyspace, shard) if err != nil { return vterrors.Wrapf(err, "failed to get tablet map for %v/%v: %v", keyspace, shard, err) } + // Stop replication on all the tablets and build their status map statusMap, primaryStatusMap, err := StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, tabletMap, reparentFunctions.GetWaitReplicasTimeout(), reparentFunctions.GetIgnoreReplicas(), erp.logger) if err != nil { return vterrors.Wrapf(err, "failed to stop replication and build status maps: %v", err) } + // check that we still have the shard lock. If we don't then we can terminate at this point if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } + // find the valid candidates for becoming the primary + // this is where we check for errant GTIDs and remove the tablets that have them from consideration validCandidates, err := FindValidEmergencyReparentCandidates(statusMap, primaryStatusMap) if err != nil { return err } + // Now, we restrict the valid candidates list according to the ReparentFunctions implementations validCandidates, err = reparentFunctions.RestrictValidCandidates(validCandidates, tabletMap) if err != nil { return err @@ -160,18 +167,21 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // Wait for all candidates to apply relay logs if err := waitForAllRelayLogsToApply(ctx, erp.logger, erp.tmc, validCandidates, tabletMap, statusMap, reparentFunctions.GetWaitForRelayLogsTimeout()); err != nil { - err = reparentFunctions.HandleRelayLogFailure(err) + // We handle the error from relay logs separately for each implementation of ReparentFunctions + err = reparentFunctions.HandleRelayLogFailure(erp.logger, err) if err != nil { return err } } + // find the primary candidate that we want to promote var newPrimary *topodatapb.Tablet - newPrimary, tabletMap, err = reparentFunctions.FindPrimaryCandidates(ctx, erp.logger, erp.tmc, validCandidates, tabletMap) + newPrimary, tabletMap, err = reparentFunctions.FindPrimaryCandidate(ctx, erp.logger, erp.tmc, validCandidates, tabletMap) if err != nil { return err } + // check weather the primary candidate selected is ideal or if it can be improved later isIdeal := reparentFunctions.PromotedReplicaIsIdeal(newPrimary, prevPrimary, tabletMap, validCandidates) // TODO := LockAction and RP diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 56bf5875a8c..edd0f23d715 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -44,13 +44,13 @@ type ( LockShard(context.Context, logutil.Logger, *topo.Server, string, string) (context.Context, func(*error), error) CheckIfFixed() bool PreRecoveryProcesses(context.Context) error - CheckPrimaryRecoveryType() error + CheckPrimaryRecoveryType(logutil.Logger) error GetWaitReplicasTimeout() time.Duration GetWaitForRelayLogsTimeout() time.Duration - HandleRelayLogFailure(err error) error + HandleRelayLogFailure(logutil.Logger, error) error GetIgnoreReplicas() sets.String RestrictValidCandidates(map[string]mysql.Position, map[string]*topo.TabletInfo) (map[string]mysql.Position, error) - FindPrimaryCandidates(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) + FindPrimaryCandidate(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) PromotedReplicaIsIdeal(*topodatapb.Tablet, *topodatapb.Tablet, map[string]*topo.TabletInfo, map[string]mysql.Position) bool PostTabletChangeHook(*topodatapb.Tablet) GetBetterCandidate(*topodatapb.Tablet, *topodatapb.Tablet, []*topodatapb.Tablet, map[string]*topo.TabletInfo) *topodatapb.Tablet @@ -97,49 +97,52 @@ func (vtctlReparent *VtctlReparentFunctions) PreRecoveryProcesses(ctx context.Co return nil } -func (vtctlReparentFunctions *VtctlReparentFunctions) GetWaitReplicasTimeout() time.Duration { - return vtctlReparentFunctions.waitReplicasTimeout +// GetWaitReplicasTimeout implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) GetWaitReplicasTimeout() time.Duration { + return vtctlReparent.waitReplicasTimeout } -func (vtctlReparentFunctions *VtctlReparentFunctions) GetWaitForRelayLogsTimeout() time.Duration { - return vtctlReparentFunctions.waitReplicasTimeout +// GetWaitForRelayLogsTimeout implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) GetWaitForRelayLogsTimeout() time.Duration { + return vtctlReparent.waitReplicasTimeout } -func (vtctlReparentFunctions *VtctlReparentFunctions) HandleRelayLogFailure(err error) error { +// HandleRelayLogFailure implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) HandleRelayLogFailure(logger logutil.Logger, err error) error { + // in case of failure in applying relay logs, vtctl should return the error + // and let the user decide weather they want to ignore the tablets that caused the error in question return err } -func (vtctlReparentFunctions *VtctlReparentFunctions) GetIgnoreReplicas() sets.String { - return vtctlReparentFunctions.ignoreReplicas +// GetIgnoreReplicas implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) GetIgnoreReplicas() sets.String { + return vtctlReparent.ignoreReplicas } // CheckPrimaryRecoveryType implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) CheckPrimaryRecoveryType() error { +func (vtctlReparent *VtctlReparentFunctions) CheckPrimaryRecoveryType(logutil.Logger) error { return nil } // RestrictValidCandidates implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) RestrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { restrictedValidCandidates := make(map[string]mysql.Position) - for candidate, position := range validCandidates { candidateInfo, ok := tabletMap[candidate] if !ok { return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) } - + // We only allow PRIMARY and REPLICA type of tablets to be considered for replication if candidateInfo.Type != topodatapb.TabletType_PRIMARY && candidateInfo.Type != topodatapb.TabletType_REPLICA { continue } - restrictedValidCandidates[candidate] = position } - return restrictedValidCandidates, nil } -// FindPrimaryCandidates implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) { +// FindPrimaryCandidate implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidate(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) { // Elect the candidate with the most up-to-date position. var winningPrimaryTabletAliasStr string var winningPosition mysql.Position @@ -158,14 +161,16 @@ func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidates(ctx context.C pos, ok := validCandidates[winningPrimaryTabletAliasStr] switch { case !ok: - return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v has errant GTIDs", winningPrimaryTabletAliasStr) + return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "primary elect %v has errant GTIDs", winningPrimaryTabletAliasStr) case !pos.AtLeast(winningPosition): - return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "master elect %v at position %v is not fully caught up. Winning position: %v", winningPrimaryTabletAliasStr, pos, winningPosition) + return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "primary elect %v at position %v is not fully caught up. Winning position: %v", winningPrimaryTabletAliasStr, pos, winningPosition) } } - // TODO:= handle not found error - newPrimaryAlias := tabletMap[winningPrimaryTabletAliasStr] + newPrimaryAlias, isFound := tabletMap[winningPrimaryTabletAliasStr] + if !isFound { + return nil, nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", winningPrimaryTabletAliasStr) + } return newPrimaryAlias.Tablet, tabletMap, nil } @@ -176,10 +181,11 @@ func (vtctlReparent *VtctlReparentFunctions) PostTabletChangeHook(*topodatapb.Ta // PromotedReplicaIsIdeal implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) PromotedReplicaIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, validCandidates map[string]mysql.Position) bool { if vtctlReparent.newPrimaryAlias != nil { - //explicit request to promote a specific tablet + // explicit request to promote a specific tablet return true } if prevPrimary != nil { + // check that the newPrimary has the same cell as the previous primary if (newPrimary.Type == topodatapb.TabletType_PRIMARY || newPrimary.Type == topodatapb.TabletType_REPLICA) && newPrimary.Alias.Cell == prevPrimary.Alias.Cell { return true } From 463277fcf1984555ade692f7961d32b683769b94 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 7 Sep 2021 16:35:22 +0530 Subject: [PATCH 075/176] handle todo for lockAction and rp Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 5 +++ .../reparentutil/emergency_reparenter.go | 11 +++-- .../reparentutil/emergency_reparenter_test.go | 4 +- .../vtctl/reparentutil/reparent_functions.go | 23 ++++------ go/vt/vtctl/reparentutil/util.go | 45 +++++++++++-------- 5 files changed, 46 insertions(+), 42 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 5b426c21dd8..7fda5133d5d 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -80,6 +80,11 @@ func (vtorcReparent *VtOrcReparentFunctions) LockShard(ctx context.Context, logg return ctx, unlock, nil } +// LockAction implements the ReparentFunctions interface +func (vtorcReparent *VtOrcReparentFunctions) LockAction() string { + return "Orc Recovery" +} + // CheckIfFixed implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) CheckIfFixed() bool { // Check if someone else fixed the problem. diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index dd1a4410b13..176eda5ccd9 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -184,8 +184,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // check weather the primary candidate selected is ideal or if it can be improved later isIdeal := reparentFunctions.PromotedReplicaIsIdeal(newPrimary, prevPrimary, tabletMap, validCandidates) - // TODO := LockAction and RP - validReplacementCandidates, err := promotePrimaryCandidateAndStartReplication(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, "", "", tabletMap, statusMap, reparentFunctions, isIdeal, true) + validReplacementCandidates, err := promotePrimaryCandidateAndStartReplication(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, reparentFunctions.LockAction(), tabletMap, statusMap, reparentFunctions, isIdeal, true) if err != nil { return err } @@ -201,7 +200,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } if !topoproto.TabletAliasEqual(betterCandidate.Alias, newPrimary.Alias) { - err = replaceWithBetterCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, betterCandidate, "", "", tabletMap, statusMap, reparentFunctions) + err = replaceWithBetterCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, betterCandidate, reparentFunctions.LockAction(), tabletMap, statusMap, reparentFunctions) if err != nil { return err } @@ -211,7 +210,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve errInPromotion := reparentFunctions.CheckIfNeedToOverridePromotion(newPrimary) if errInPromotion != nil { erp.logger.Errorf("have to override promotion because of constraint failure - %v", errInPromotion) - newPrimary, err = erp.undoPromotion(ctx, erp.ts, ev, keyspace, shard, prevPrimary, "", "", tabletMap, statusMap, reparentFunctions) + newPrimary, err = erp.undoPromotion(ctx, erp.ts, ev, keyspace, shard, prevPrimary, reparentFunctions.LockAction(), tabletMap, statusMap, reparentFunctions) if err != nil { return err } @@ -231,11 +230,11 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } func (erp *EmergencyReparenter) undoPromotion(ctx context.Context, ts *topo.Server, ev *events.Reparent, keyspace, shard string, prevPrimary *topodatapb.Tablet, - lockAction, rp string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions) (*topodatapb.Tablet, error) { + lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions) (*topodatapb.Tablet, error) { var primaryAlias *topodatapb.TabletAlias if prevPrimary != nil { - _, err := promotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, prevPrimary, lockAction, rp, tabletMap, statusMap, reparentFunctions, true, false) + _, err := promotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, prevPrimary, lockAction, tabletMap, statusMap, reparentFunctions, true, false) if err == nil { return prevPrimary, nil } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index df02c6ef474..f31539eafea 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -92,15 +92,13 @@ func TestEmergencyReparenter_getLockAction(t *testing.T) { }, } - vtctlReparentFunctions := &VtctlReparentFunctions{} - for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() - actual := vtctlReparentFunctions.getLockAction(tt.alias) + actual := getLockAction(tt.alias) assert.Equal(t, tt.expected, actual, tt.msg) }) } diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index edd0f23d715..f886478b2ca 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -18,7 +18,6 @@ package reparentutil import ( "context" - "fmt" "time" "vitess.io/vitess/go/event" @@ -42,6 +41,7 @@ type ( // ReparentFunctions is an interface which has all the functions implementation required for re-parenting ReparentFunctions interface { LockShard(context.Context, logutil.Logger, *topo.Server, string, string) (context.Context, func(*error), error) + LockAction() string CheckIfFixed() bool PreRecoveryProcesses(context.Context) error CheckPrimaryRecoveryType(logutil.Logger) error @@ -81,7 +81,12 @@ func NewVtctlReparentFunctions(newPrimaryAlias *topodatapb.TabletAlias, ignoreRe // LockShard implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) LockShard(ctx context.Context, logger logutil.Logger, ts *topo.Server, keyspace string, shard string) (context.Context, func(*error), error) { - return ts.LockShard(ctx, keyspace, shard, vtctlReparent.getLockAction(vtctlReparent.newPrimaryAlias)) + return ts.LockShard(ctx, keyspace, shard, getLockAction(vtctlReparent.newPrimaryAlias)) +} + +// LockAction implements the ReparentFunctions interface +func (vtctlReparent *VtctlReparentFunctions) LockAction() string { + return getLockAction(vtctlReparent.newPrimaryAlias) } // CheckIfFixed implements the ReparentFunctions interface @@ -226,16 +231,6 @@ func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePromotion(newP func (vtctlReparent *VtctlReparentFunctions) PostERSCompletionHook(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) { } -func (vtctlReparent *VtctlReparentFunctions) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { - action := "EmergencyReparentShard" - - if newPrimaryAlias != nil { - action += fmt.Sprintf("(%v)", topoproto.TabletAliasString(newPrimaryAlias)) - } - - return action -} - func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, winningPrimaryTabletAliasStr string, statusMap map[string]*replicationdatapb.StopReplicationStatus, tabletMap map[string]*topo.TabletInfo, keyspace, shard string) error { logger.Infof("promoting tablet %v to master", winningPrimaryTabletAliasStr) event.DispatchUpdate(ev, "promoting replica") @@ -245,7 +240,7 @@ func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Conte return vterrors.Errorf(vtrpc.Code_INTERNAL, "attempted to promote master-elect %v that was not in the tablet map; this an impossible situation", winningPrimaryTabletAliasStr) } - rp, err := tmc.PromoteReplica(ctx, newPrimaryTabletInfo.Tablet) + _, err := tmc.PromoteReplica(ctx, newPrimaryTabletInfo.Tablet) if err != nil { return vterrors.Wrapf(err, "master-elect tablet %v failed to be upgraded to master: %v", winningPrimaryTabletAliasStr, err) } @@ -254,6 +249,6 @@ func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Conte return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } - _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, vtctlReparent.getLockAction(vtctlReparent.newPrimaryAlias), rp, tabletMap, statusMap, vtctlReparent, false) + _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, getLockAction(vtctlReparent.newPrimaryAlias), tabletMap, statusMap, vtctlReparent, false) return err } diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 1a3cb644d47..e0d70cac774 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -18,6 +18,7 @@ package reparentutil import ( "context" + "fmt" "sync" "time" @@ -100,9 +101,10 @@ func waitForAllRelayLogsToApply(ctx context.Context, logger logutil.Logger, tmc return nil } -func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent, logger logutil.Logger, - tmc tmclient.TabletManagerClient, newPrimaryTablet *topodatapb.Tablet, lockAction, rp string, - tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions, waitForAllReplicas bool) ([]*topodatapb.Tablet, error) { +func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, + newPrimaryTablet *topodatapb.Tablet, lockAction string, tabletMap map[string]*topo.TabletInfo, + statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions, + waitForAllReplicas bool) ([]*topodatapb.Tablet, error) { var replicasStartedReplication []*topodatapb.Tablet @@ -128,9 +130,13 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent replWg := sync.WaitGroup{} rec := concurrency.AllErrorRecorder{} - handlePrimary := func(alias string, ti *topodatapb.Tablet) error { + handlePrimary := func(alias string, tablet *topodatapb.Tablet) error { + position, err := tmc.MasterPosition(replCtx, tablet) + if err != nil { + return err + } logger.Infof("populating reparent journal on new master %v", alias) - return tmc.PopulateReparentJournal(replCtx, ti, now, lockAction, newPrimaryTablet.Alias, rp) + return tmc.PopulateReparentJournal(replCtx, tablet, now, lockAction, newPrimaryTablet.Alias, position) } handleReplica := func(alias string, ti *topo.TabletInfo) { @@ -291,7 +297,7 @@ func ChooseNewPrimary( // promotePrimaryCandidateAndStartReplication promotes the primary candidate that we have, but it does not set to start accepting writes func promotePrimaryCandidateAndStartReplication(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, - lockAction, rp string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions, isIdeal bool, startReplication bool) ([]*topodatapb.Tablet, error) { + lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions, isIdeal bool, startReplication bool) ([]*topodatapb.Tablet, error) { if err := promotePrimary(ctx, tmc, ts, logger, newPrimary); err != nil { return nil, err } @@ -299,18 +305,11 @@ func promotePrimaryCandidateAndStartReplication(ctx context.Context, tmc tmclien reparentFunctions.PostTabletChangeHook(newPrimary) // if the promoted primary is not ideal then we wait for all the replicas so that we choose a better candidate from them later - replicasStartedReplication, err := reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, rp, tabletMap, statusMap, reparentFunctions, !isIdeal) + replicasStartedReplication, err := reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, reparentFunctions, !isIdeal) if err != nil { return nil, err } - // TODO := add as a postponed function - //if postponedFunctionsContainer != nil && postponeAllMatchOperations != nil && postponeAllMatchOperations(candidateReplica, hasBestPromotionRule) { - // postponedFunctionsContainer.AddPostponedFunction(moveGTIDFunc, fmt.Sprintf("regroup-replicas-gtid %+v", candidateReplica.Key)) - //} else { - // err = moveGTIDFunc() - //} - if startReplication { err = tmc.StartReplication(ctx, newPrimary) if err != nil { @@ -318,9 +317,7 @@ func promotePrimaryCandidateAndStartReplication(ctx context.Context, tmc tmclien } } - //log.Debugf("RegroupReplicasGTID: done") - //inst.AuditOperation("regroup-replicas-gtid", masterKey, fmt.Sprintf("regrouped replicas of %+v via GTID; promoted %+v", *masterKey, candidateReplica.Key)) - return replicasStartedReplication, nil //unmovedReplicas, candidateReplica, err + return replicasStartedReplication, nil } // promotePrimary makes the new tablet the primary and proactively performs @@ -418,7 +415,7 @@ func FindCurrentPrimary(tabletMap map[string]*topo.TabletInfo, logger logutil.Lo // replaceWithBetterCandidate promotes the newer candidate over the primary candidate that we have, but it does not set to start accepting writes func replaceWithBetterCandidate(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, prevPrimary, newPrimary *topodatapb.Tablet, - lockAction, rp string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions) error { + lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions) error { pos, err := tmc.PrimaryPosition(ctx, prevPrimary) if err != nil { @@ -437,7 +434,7 @@ func replaceWithBetterCandidate(ctx context.Context, tmc tmclient.TabletManagerC reparentFunctions.PostTabletChangeHook(newPrimary) // if the promoted primary is not ideal then we wait for all the replicas so that we choose a better candidate from them later - _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, rp, tabletMap, statusMap, reparentFunctions, false) + _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, reparentFunctions, false) if err != nil { return err } @@ -453,3 +450,13 @@ func replaceWithBetterCandidate(ctx context.Context, tmc tmclient.TabletManagerC //inst.AuditOperation("regroup-replicas-gtid", masterKey, fmt.Sprintf("regrouped replicas of %+v via GTID; promoted %+v", *masterKey, candidateReplica.Key)) return nil //unmovedReplicas, candidateReplica, err } + +func getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { + action := "EmergencyReparentShard" + + if newPrimaryAlias != nil { + action += fmt.Sprintf("(%v)", topoproto.TabletAliasString(newPrimaryAlias)) + } + + return action +} From 30065e42d6a9954e7e63ae6520e90c1de4f55b29 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 7 Sep 2021 18:04:32 +0530 Subject: [PATCH 076/176] add comments to remaining part of ERS Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 51 +++++----- .../reparentutil/emergency_reparenter.go | 19 +++- .../vtctl/reparentutil/reparent_functions.go | 9 +- go/vt/vtctl/reparentutil/util.go | 98 +++++-------------- 4 files changed, 67 insertions(+), 110 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 7fda5133d5d..bfefdc0c9fb 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -61,6 +61,7 @@ type VtOrcReparentFunctions struct { hasBestPromotionRule bool } +// NewVtorcReparentFunctions is used to create a new VtOrcReparentFunctions func NewVtorcReparentFunctions(analysisEntry inst.ReplicationAnalysis, candidateInstanceKey *inst.InstanceKey, skipProcesses bool, topologyRecovery *TopologyRecovery) *VtOrcReparentFunctions { return &VtOrcReparentFunctions{ analysisEntry: analysisEntry, @@ -321,7 +322,7 @@ func (vtorcReparent *VtOrcReparentFunctions) PromotedReplicaIsIdeal(newPrimary, return false } -// PostReplicationChangeHook implements the ReparentFunctions interface +// PostTabletChangeHook implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) PostTabletChangeHook(tablet *topodatapb.Tablet) { instanceKey := &inst.InstanceKey{ Hostname: tablet.MysqlHostname, @@ -331,6 +332,7 @@ func (vtorcReparent *VtOrcReparentFunctions) PostTabletChangeHook(tablet *topoda TabletRefresh(*instanceKey) } +// GetBetterCandidate implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) GetBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo) *topodatapb.Tablet { if vtorcReparent.candidateInstanceKey != nil { candidateTablet, _ := inst.ReadTablet(*vtorcReparent.candidateInstanceKey) @@ -347,6 +349,7 @@ func (vtorcReparent *VtOrcReparentFunctions) GetBetterCandidate(newPrimary, prev return replacementCandidate } +// TODO: make simpler func getReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, newPrimary, oldPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet) *topodatapb.Tablet { var preferredCandidates []*topodatapb.Tablet var neutralReplicas []*topodatapb.Tablet @@ -493,48 +496,33 @@ func getInstanceFromTablet(tablet *topodatapb.Tablet) *inst.Instance { return instance } -// CheckIfNeedToOverridePrimary implements the ReparentFunctions interface +// CheckIfNeedToOverridePromotion implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) CheckIfNeedToOverridePromotion(newPrimary *topodatapb.Tablet) error { - // TODO : use fixing code outside - //if vtorcReparent.promotedReplica == nil { - // err := TabletUndoDemoteMaster(vtorcReparent.analysisEntry.AnalyzedInstanceKey) - // AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: TabletUndoDemoteMaster: %v", err)) - // message := "Failure: no replica promoted." - // AuditTopologyRecovery(vtorcReparent.topologyRecovery, message) - // inst.AuditOperation("recover-dead-master", &vtorcReparent.analysisEntry.AnalyzedInstanceKey, message) - // return err - //} - - // TODO: Move out to post-change code - message := fmt.Sprintf("promoted replica: %+v", vtorcReparent.promotedReplica.Key) - AuditTopologyRecovery(vtorcReparent.topologyRecovery, message) - inst.AuditOperation("recover-dead-master", &vtorcReparent.analysisEntry.AnalyzedInstanceKey, message) - newPrimaryInstance := getInstanceFromTablet(newPrimary) - overrideMasterPromotion := func() error { + overridePrimaryPromotion := func() error { // Scenarios where we might cancel the promotion. if satisfied, reason := PrimaryFailoverGeographicConstraintSatisfied(&vtorcReparent.analysisEntry, newPrimaryInstance); !satisfied { - return fmt.Errorf("RecoverDeadMaster: failed %+v promotion; %s", newPrimaryInstance.Key, reason) + return fmt.Errorf("RecoverDeadPrimary: failed %+v promotion; %s", newPrimaryInstance.Key, reason) } if config.Config.FailPrimaryPromotionOnLagMinutes > 0 && time.Duration(newPrimaryInstance.ReplicationLagSeconds.Int64)*time.Second >= time.Duration(config.Config.FailPrimaryPromotionOnLagMinutes)*time.Minute { // candidate replica lags too much - return fmt.Errorf("RecoverDeadMaster: failed promotion. FailPrimaryPromotionOnLagMinutes is set to %d (minutes) and promoted replica %+v 's lag is %d (seconds)", config.Config.FailPrimaryPromotionOnLagMinutes, newPrimaryInstance.Key, newPrimaryInstance.ReplicationLagSeconds.Int64) + return fmt.Errorf("RecoverDeadPrimary: failed promotion. FailPrimaryPromotionOnLagMinutes is set to %d (minutes) and promoted replica %+v 's lag is %d (seconds)", config.Config.FailPrimaryPromotionOnLagMinutes, newPrimaryInstance.Key, newPrimaryInstance.ReplicationLagSeconds.Int64) } if config.Config.FailPrimaryPromotionIfSQLThreadNotUpToDate && !newPrimaryInstance.SQLThreadUpToDate() { - return fmt.Errorf("RecoverDeadMaster: failed promotion. FailPrimaryPromotionIfSQLThreadNotUpToDate is set and promoted replica %+v 's sql thread is not up to date (relay logs still unapplied). Aborting promotion", newPrimaryInstance.Key) + return fmt.Errorf("RecoverDeadPrimary: failed promotion. FailPrimaryPromotionIfSQLThreadNotUpToDate is set and promoted replica %+v 's sql thread is not up to date (relay logs still unapplied). Aborting promotion", newPrimaryInstance.Key) } if config.Config.DelayPrimaryPromotionIfSQLThreadNotUpToDate && !newPrimaryInstance.SQLThreadUpToDate() { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("DelayMasterPromotionIfSQLThreadNotUpToDate: waiting for SQL thread on %+v", newPrimaryInstance.Key)) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("DelayPrimaryPromotionIfSQLThreadNotUpToDate: waiting for SQL thread on %+v", newPrimaryInstance.Key)) if _, err := inst.WaitForSQLThreadUpToDate(&newPrimaryInstance.Key, 0, 0); err != nil { - return fmt.Errorf("DelayMasterPromotionIfSQLThreadNotUpToDate error: %+v", err) + return fmt.Errorf("DelayPrimaryPromotionIfSQLThreadNotUpToDate error: %+v", err) } - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("DelayMasterPromotionIfSQLThreadNotUpToDate: SQL thread caught up on %+v", newPrimaryInstance.Key)) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("DelayPrimaryPromotionIfSQLThreadNotUpToDate: SQL thread caught up on %+v", newPrimaryInstance.Key)) } // All seems well. No override done. return nil } - if err := overrideMasterPromotion(); err != nil { + if err := overridePrimaryPromotion(); err != nil { AuditTopologyRecovery(vtorcReparent.topologyRecovery, err.Error()) vtorcReparent.promotedReplica = nil return err @@ -544,14 +532,19 @@ func (vtorcReparent *VtOrcReparentFunctions) CheckIfNeedToOverridePromotion(newP // PostERSCompletionHook implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) PostERSCompletionHook(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) { + if vtorcReparent.promotedReplica != nil { + message := fmt.Sprintf("promoted replica: %+v", vtorcReparent.promotedReplica.Key) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, message) + inst.AuditOperation("recover-dead-master", &vtorcReparent.analysisEntry.AnalyzedInstanceKey, message) + } // And this is the end; whether successful or not, we're done. resolveRecovery(vtorcReparent.topologyRecovery, vtorcReparent.promotedReplica) // Now, see whether we are successful or not. From this point there's no going back. if vtorcReparent.promotedReplica != nil { // Success! recoverDeadPrimarySuccessCounter.Inc(1) - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadMaster: successfully promoted %+v", vtorcReparent.promotedReplica.Key)) - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: promoted server coordinates: %+v", vtorcReparent.promotedReplica.SelfBinlogCoordinates)) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: successfully promoted %+v", vtorcReparent.promotedReplica.Key)) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadPrimary: promoted server coordinates: %+v", vtorcReparent.promotedReplica.SelfBinlogCoordinates)) kvPairs := inst.GetClusterPrimaryKVPairs(vtorcReparent.analysisEntry.ClusterDetails.ClusterAlias, &vtorcReparent.promotedReplica.Key) AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("Writing KV %+v", kvPairs)) @@ -570,12 +563,12 @@ func (vtorcReparent *VtOrcReparentFunctions) PostERSCompletionHook(ctx context.C inst.DetachReplicaPrimaryHost(&vtorcReparent.promotedReplica.Key) return nil } - vtorcReparent.topologyRecovery.AddPostponedFunction(postponedFunction, fmt.Sprintf("RecoverDeadMaster, detaching promoted master host %+v", vtorcReparent.promotedReplica.Key)) + vtorcReparent.topologyRecovery.AddPostponedFunction(postponedFunction, fmt.Sprintf("RecoverDeadPrimary, detaching promoted master host %+v", vtorcReparent.promotedReplica.Key)) } func() error { before := vtorcReparent.analysisEntry.AnalyzedInstanceKey.StringCode() after := vtorcReparent.promotedReplica.Key.StringCode() - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadMaster: updating cluster_alias: %v -> %v", before, after)) + AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadPrimary: updating cluster_alias: %v -> %v", before, after)) //~~~inst.ReplaceClusterName(before, after) if alias := vtorcReparent.analysisEntry.ClusterDetails.ClusterAlias; alias != "" { inst.SetClusterAlias(vtorcReparent.promotedReplica.Key.StringCode(), alias) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 176eda5ccd9..e911f4d4b65 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -184,7 +184,8 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // check weather the primary candidate selected is ideal or if it can be improved later isIdeal := reparentFunctions.PromotedReplicaIsIdeal(newPrimary, prevPrimary, tabletMap, validCandidates) - validReplacementCandidates, err := promotePrimaryCandidateAndStartReplication(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, reparentFunctions.LockAction(), tabletMap, statusMap, reparentFunctions, isIdeal, true) + // we now promote our primary candidate and also reparent all the other tablets to start replicating from this candidate + validReplacementCandidates, err := promotePrimaryCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, reparentFunctions.LockAction(), tabletMap, statusMap, reparentFunctions, isIdeal) if err != nil { return err } @@ -194,11 +195,13 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } + // try to find a better candidate if we do not already have the most ideal one betterCandidate := newPrimary if !isIdeal { betterCandidate = reparentFunctions.GetBetterCandidate(newPrimary, prevPrimary, validReplacementCandidates, tabletMap) } + // if our better candidate is different from our previous candidate, then we replace our primary if !topoproto.TabletAliasEqual(betterCandidate.Alias, newPrimary.Alias) { err = replaceWithBetterCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, betterCandidate, reparentFunctions.LockAction(), tabletMap, statusMap, reparentFunctions) if err != nil { @@ -207,9 +210,11 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve newPrimary = betterCandidate } + // now we check if there is a need to override the promotion of the newPrimary errInPromotion := reparentFunctions.CheckIfNeedToOverridePromotion(newPrimary) if errInPromotion != nil { erp.logger.Errorf("have to override promotion because of constraint failure - %v", errInPromotion) + // we try and undo the promotion newPrimary, err = erp.undoPromotion(ctx, erp.ts, ev, keyspace, shard, prevPrimary, reparentFunctions.LockAction(), tabletMap, statusMap, reparentFunctions) if err != nil { return err @@ -219,10 +224,12 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } } + // Finally, we call PromoteReplica which fixes the semi-sync, set the primary to read-write and flushes the binlogs _, err = erp.tmc.PromoteReplica(ctx, newPrimary) if err != nil { return vterrors.Wrapf(err, "primary-elect tablet %v failed to be upgraded to primary: %v", newPrimary.Alias, err) } + // call the post tablet change hook on the new primary reparentFunctions.PostTabletChangeHook(newPrimary) ev.NewPrimary = proto.Clone(newPrimary).(*topodatapb.Tablet) @@ -232,9 +239,10 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve func (erp *EmergencyReparenter) undoPromotion(ctx context.Context, ts *topo.Server, ev *events.Reparent, keyspace, shard string, prevPrimary *topodatapb.Tablet, lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions) (*topodatapb.Tablet, error) { var primaryAlias *topodatapb.TabletAlias - + var err error if prevPrimary != nil { - _, err := promotePrimaryCandidateAndStartReplication(ctx, erp.tmc, ts, ev, erp.logger, prevPrimary, lockAction, tabletMap, statusMap, reparentFunctions, true, false) + // promote the original primary back + _, err = promotePrimaryCandidate(ctx, erp.tmc, ts, ev, erp.logger, prevPrimary, lockAction, tabletMap, statusMap, reparentFunctions, true) if err == nil { return prevPrimary, nil } @@ -243,7 +251,7 @@ func (erp *EmergencyReparenter) undoPromotion(ctx context.Context, ts *topo.Serv } newTerm := time.Now() - _, err := ts.UpdateShardFields(ctx, keyspace, shard, func(si *topo.ShardInfo) error { + _, errInUpdate := ts.UpdateShardFields(ctx, keyspace, shard, func(si *topo.ShardInfo) error { if proto.Equal(si.PrimaryAlias, primaryAlias) { return nil } @@ -251,5 +259,8 @@ func (erp *EmergencyReparenter) undoPromotion(ctx context.Context, ts *topo.Serv si.PrimaryTermStartTime = logutil.TimeToProto(newTerm) return nil }) + if errInUpdate != nil { + return nil, errInUpdate + } return prevPrimary, err } diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index f886478b2ca..d0f478a0918 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -179,11 +179,11 @@ func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidate(ctx context.Co return newPrimaryAlias.Tablet, tabletMap, nil } -// PostReplicationChangeHook implements the ReparentFunctions interface +// PostTabletChangeHook implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) PostTabletChangeHook(*topodatapb.Tablet) { } -// PromotedReplicaIsIdeal implements the ReparentFunctions interface +// PromotedReplicaIsIdeal implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) PromotedReplicaIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, validCandidates map[string]mysql.Position) bool { if vtctlReparent.newPrimaryAlias != nil { // explicit request to promote a specific tablet @@ -203,9 +203,8 @@ func (vtctlReparent *VtctlReparentFunctions) PromotedReplicaIsIdeal(newPrimary, return false } -// GetBetterCandidate implements the ReparentFunctions interface +// GetBetterCandidate implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) GetBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo) *topodatapb.Tablet { - if prevPrimary != nil { // find one which is of the correct type and matches the cell of the previous primary for _, candidate := range validCandidates { @@ -222,7 +221,7 @@ func (vtctlReparent *VtctlReparentFunctions) GetBetterCandidate(newPrimary, prev return newPrimary } -// CheckIfNeedToOverridePrimary implements the ReparentFunctions interface +// CheckIfNeedToOverridePromotion implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePromotion(newPrimary *topodatapb.Tablet) error { return nil } diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index e0d70cac774..e214d3aa2bb 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -22,8 +22,6 @@ import ( "sync" "time" - "google.golang.org/protobuf/proto" - "vitess.io/vitess/go/event" "vitess.io/vitess/go/vt/concurrency" @@ -74,7 +72,7 @@ func waitForAllRelayLogsToApply(ctx context.Context, logger logutil.Logger, tmc // place, so we skip it, and log that we did. status, ok := statusMap[candidate] if !ok { - logger.Infof("EmergencyReparent candidate %v not in replica status map; this means it was not running replication (because it was formerly MASTER), so skipping WaitForRelayLogsToApply step for this candidate", candidate) + logger.Infof("EmergencyReparent candidate %v not in replica status map; this means it was not running replication (because it was formerly PRIMARY), so skipping WaitForRelayLogsToApply step for this candidate", candidate) continue } @@ -135,13 +133,13 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent if err != nil { return err } - logger.Infof("populating reparent journal on new master %v", alias) + logger.Infof("populating reparent journal on new primary %v", alias) return tmc.PopulateReparentJournal(replCtx, tablet, now, lockAction, newPrimaryTablet.Alias, position) } handleReplica := func(alias string, ti *topo.TabletInfo) { defer replWg.Done() - logger.Infof("setting new master on replica %v", alias) + logger.Infof("setting new primary on replica %v", alias) forceStart := false if status, ok := statusMap[alias]; ok { @@ -156,7 +154,7 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent forceStart = fs } - err := tmc.SetReplicationSource(replCtx, ti.Tablet, newPrimaryTablet.Alias, 0, "", forceStart) + err := tmc.SetMaster(replCtx, ti.Tablet, newPrimaryTablet.Alias, 0, "", forceStart) if err != nil { err = vterrors.Wrapf(err, "tablet %v SetReplicationSource failed: %v", alias, err) rec.RecordError(err) @@ -165,10 +163,11 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent } replicasStartedReplication = append(replicasStartedReplication, ti.Tablet) + // We call PostTabletChangeHook every time there is an update to a tablet's replication or type reparentFunctions.PostTabletChangeHook(ti.Tablet) // Signal that at least one goroutine succeeded to SetReplicationSource. - // We do this only when we do not want to wair for all the replicas + // We do this only when we do not want to wait for all the replicas if !waitForAllReplicas { replSuccessCancel() } @@ -203,10 +202,10 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent primaryErr := handlePrimary(topoproto.TabletAliasString(newPrimaryTablet.Alias), newPrimaryTablet) if primaryErr != nil { - logger.Warningf("master failed to PopulateReparentJournal") + logger.Warningf("primary failed to PopulateReparentJournal") replCancel() - return nil, vterrors.Wrapf(primaryErr, "failed to PopulateReparentJournal on master: %v", primaryErr) + return nil, vterrors.Wrapf(primaryErr, "failed to PopulateReparentJournal on primary: %v", primaryErr) } select { @@ -295,70 +294,31 @@ func ChooseNewPrimary( return nil, nil } -// promotePrimaryCandidateAndStartReplication promotes the primary candidate that we have, but it does not set to start accepting writes -func promotePrimaryCandidateAndStartReplication(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, - lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions, isIdeal bool, startReplication bool) ([]*topodatapb.Tablet, error) { - if err := promotePrimary(ctx, tmc, ts, logger, newPrimary); err != nil { +// promotePrimaryCandidate promotes the primary candidate that we have, but it does not yet set to start accepting writes +func promotePrimaryCandidate(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, + lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions, isIdeal bool) ([]*topodatapb.Tablet, error) { + // first step is change the type of the newPrimary tablet to PRIMARY + if err := changeTypeToPrimary(ctx, tmc, newPrimary); err != nil { return nil, err } + // We call PostTabletChangeHook every time there is an update to a tablet's replication or type reparentFunctions.PostTabletChangeHook(newPrimary) + // now we reparent all the other tablets to start replication from our new primary // if the promoted primary is not ideal then we wait for all the replicas so that we choose a better candidate from them later replicasStartedReplication, err := reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, reparentFunctions, !isIdeal) if err != nil { return nil, err } - if startReplication { - err = tmc.StartReplication(ctx, newPrimary) - if err != nil { - return nil, err - } - } - return replicasStartedReplication, nil } -// promotePrimary makes the new tablet the primary and proactively performs -// the necessary propagation to the old primary. The propagation is best -// effort. If it fails, the tablet's shard sync will eventually converge. -func promotePrimary(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, logger logutil.Logger, newPrimary *topodatapb.Tablet) error { - err := tmc.ChangeType(ctx, newPrimary, topodatapb.TabletType_PRIMARY) - if err != nil { - return err - } - ti, err := ts.GetTablet(ctx, newPrimary.Alias) - if err != nil { - return err - } - newPrimary = ti.Tablet - ctx, cancel := context.WithTimeout(ctx, *topo.RemoteOperationTimeout) - defer cancel() - _, err = ts.UpdateShardFields(ctx, newPrimary.Keyspace, newPrimary.Shard, func(si *topo.ShardInfo) error { - if proto.Equal(si.PrimaryAlias, newPrimary.Alias) && proto.Equal(si.PrimaryTermStartTime, newPrimary.PrimaryTermStartTime) { - return topo.NewError(topo.NoUpdateNeeded, "") - } - - // We just successfully reparented. We should check timestamps, but always overwrite. - lastTerm := si.GetPrimaryTermStartTime() - newTerm := logutil.ProtoToTime(newPrimary.PrimaryTermStartTime) - if !newTerm.After(lastTerm) { - logger.Errorf("Possible clock skew. New master start time is before previous one: %v vs %v", newTerm, lastTerm) - } - - aliasStr := topoproto.TabletAliasString(newPrimary.Alias) - logger.Infof("Updating shard record: master_alias=%v, primary_term_start_time=%v", aliasStr, newTerm) - si.PrimaryAlias = newPrimary.Alias - si.PrimaryTermStartTime = newPrimary.PrimaryTermStartTime - return nil - }) - // Log any error but do not abort - if err != nil { - logger.Error(err) - return nil - } - return nil +// changeTypeToPrimary changes the type of the tablet to primary +// the tablet's shard sync will update the topo server for us. +func changeTypeToPrimary(ctx context.Context, tmc tmclient.TabletManagerClient, newPrimary *topodatapb.Tablet) error { + return tmc.ChangeType(ctx, newPrimary, topodatapb.TabletType_PRIMARY) } // FindCurrentPrimary returns the current primary tablet of a shard, if any. The @@ -416,39 +376,33 @@ func FindCurrentPrimary(tabletMap map[string]*topo.TabletInfo, logger logutil.Lo // replaceWithBetterCandidate promotes the newer candidate over the primary candidate that we have, but it does not set to start accepting writes func replaceWithBetterCandidate(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, prevPrimary, newPrimary *topodatapb.Tablet, lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions) error { - + // Find the primary position of the previous primary pos, err := tmc.PrimaryPosition(ctx, prevPrimary) if err != nil { return err } + // Wait until the new primary has caught upto that position err = tmc.WaitForPosition(ctx, newPrimary, pos) if err != nil { return err } - if err := promotePrimary(ctx, tmc, ts, logger, newPrimary); err != nil { + // Now change the type of the new primary tablet to PRIMARY + if err := changeTypeToPrimary(ctx, tmc, newPrimary); err != nil { return err } + // We call PostTabletChangeHook every time there is an update to a tablet's replication or type reparentFunctions.PostTabletChangeHook(newPrimary) - // if the promoted primary is not ideal then we wait for all the replicas so that we choose a better candidate from them later + // we now reparent the other tablets, but this time we do not need to wait for all of them _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, reparentFunctions, false) if err != nil { return err } - // TODO := add as a postponed function - //if postponedFunctionsContainer != nil && postponeAllMatchOperations != nil && postponeAllMatchOperations(candidateReplica, hasBestPromotionRule) { - // postponedFunctionsContainer.AddPostponedFunction(moveGTIDFunc, fmt.Sprintf("regroup-replicas-gtid %+v", candidateReplica.Key)) - //} else { - // err = moveGTIDFunc() - //} - - //log.Debugf("RegroupReplicasGTID: done") - //inst.AuditOperation("regroup-replicas-gtid", masterKey, fmt.Sprintf("regrouped replicas of %+v via GTID; promoted %+v", *masterKey, candidateReplica.Key)) - return nil //unmovedReplicas, candidateReplica, err + return nil } func getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { From 74423b27a53fa3588c580bf9e192f40752fc3392 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 8 Sep 2021 13:05:32 +0530 Subject: [PATCH 077/176] revert change to promoteReplica Signed-off-by: Manan Gupta --- go/vt/vttablet/tabletmanager/rpc_replication.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index dfad7dc45f0..ec16d8003b8 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -795,7 +795,7 @@ func (tm *TabletManager) PromoteReplica(ctx context.Context) (string, error) { return "", err } - if err := tm.MysqlDaemon.SetReadOnly(false); err != nil { + if err := tm.changeTypeLocked(ctx, topodatapb.TabletType_PRIMARY, DBActionSetReadWrite); err != nil { return "", err } From cf12b41e25bf37e32e3e40822a0c8eae3bdaec88 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 8 Sep 2021 16:21:16 +0530 Subject: [PATCH 078/176] fixed emergency-reparent unit tests Signed-off-by: Manan Gupta --- .../testutil/test_tmclient.go | 7 + .../reparentutil/emergency_reparenter_test.go | 276 ++++++++++-------- 2 files changed, 158 insertions(+), 125 deletions(-) diff --git a/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go b/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go index 01c83a0700a..0b7be7fbbdc 100644 --- a/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go +++ b/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go @@ -128,6 +128,8 @@ type TabletManagerClient struct { // a test, set tmc.TopoServer = nil. TopoServer *topo.Server // keyed by tablet alias. + ChangeTabletTypeResult map[string]error + // keyed by tablet alias. DemoteMasterDelays map[string]time.Duration // keyed by tablet alias. DemoteMasterResults map[string]struct { @@ -209,6 +211,10 @@ type TabletManagerClient struct { // ChangeType is part of the tmclient.TabletManagerClient interface. func (fake *TabletManagerClient) ChangeType(ctx context.Context, tablet *topodatapb.Tablet, newType topodatapb.TabletType) error { + if result, ok := fake.ChangeTabletTypeResult[topoproto.TabletAliasString(tablet.Alias)]; ok { + return result + } + if fake.TopoServer == nil { return assert.AnError } @@ -435,6 +441,7 @@ func (fake *TabletManagerClient) SetMaster(ctx context.Context, tablet *topodata return assert.AnError } +// StartReplication is part of the tmclient.TabletManagerClient interface. func (fake *TabletManagerClient) StartReplication(ctx context.Context, tablet *topodatapb.Tablet) error { if tablet == nil { return assert.AnError diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index f31539eafea..b97bdad7e8a 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -18,6 +18,7 @@ package reparentutil import ( "context" + "fmt" "testing" "time" @@ -119,7 +120,8 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { shards []*vtctldatapb.Shard tablets []*topodatapb.Tablet // results - shouldErr bool + shouldErr bool + errShouldContain string }{ { name: "success", @@ -137,6 +139,14 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Error: nil, }, }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000102": { + Error: nil, + }, + }, SetMasterResults: map[string]error{ "zone1-0000000100": nil, "zone1-0000000101": nil, @@ -236,6 +246,14 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { PopulateReparentJournalResults: map[string]error{ "zone1-0000000101": nil, }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000101": { + Error: nil, + }, + }, PromoteReplicaResults: map[string]struct { Result string Error error @@ -345,6 +363,14 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, }, }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000102": { + Error: nil, + }, + }, PopulateReparentJournalResults: map[string]error{ "zone1-0000000102": nil, }, @@ -446,6 +472,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { shard: "-", ts: memorytopo.NewServer("zone1"), shouldErr: true, + errShouldContain: "node doesn't exist: keyspaces/testkeyspace/shards/-/Shard", }, { name: "cannot stop replication", @@ -500,10 +527,11 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Shard: "-", }, }, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: true, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: "failed to stop replication and build status maps", }, { name: "lost topo lock", @@ -558,10 +586,11 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Shard: "-", }, }, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: true, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: "lost topology lock, aborting", }, { name: "cannot get reparent candidates", @@ -628,10 +657,11 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Hostname: "has a zero relay log position", }, }, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: true, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: "encountered tablet zone1-0000000102 with no relay log position", }, { name: "zero valid reparent candidates", @@ -643,10 +673,11 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Name: "-", }, }, - shouldErr: true, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), + shouldErr: true, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + errShouldContain: "no valid candidates for emergency reparent", }, { name: "error waiting for relay logs to apply", @@ -732,10 +763,11 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Hostname: "fails to apply relay logs", }, }, - shouldErr: true, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), + shouldErr: true, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + errShouldContain: "could not apply all relay logs within the provided waitReplicasTimeout", }, { name: "requested primary-elect is not in tablet map", @@ -818,10 +850,11 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Shard: "-", }, }, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: true, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: "primary elect zone1-0000000200 has errant GTIDs", }, { name: "requested primary-elect is not winning primary-elect", @@ -908,7 +941,8 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Hostname: "not most up-to-date position", }, }, - shouldErr: true, + shouldErr: true, + errShouldContain: "primary elect zone1-0000000102 at position 3e11fa47-71ca-11e1-9e33-c80aa9429562:1-20 is not fully caught up", }, { name: "cannot promote new primary", @@ -929,6 +963,17 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Error: assert.AnError, }, }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000102": { + Error: nil, + }, + }, + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000102": nil, + }, StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status StopStatus *replicationdatapb.StopReplicationStatus @@ -936,6 +981,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }{ "zone1-0000000100": { StopStatus: &replicationdatapb.StopReplicationStatus{ + Before: &replicationdatapb.Status{}, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", @@ -944,6 +990,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, "zone1-0000000101": { StopStatus: &replicationdatapb.StopReplicationStatus{ + Before: &replicationdatapb.Status{}, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", @@ -952,6 +999,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, "zone1-0000000102": { StopStatus: &replicationdatapb.StopReplicationStatus{ + Before: &replicationdatapb.Status{}, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", @@ -970,6 +1018,11 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21": nil, }, }, + SetMasterResults: map[string]error{ + "zone1-0000000100": nil, + "zone1-0000000101": nil, + "zone1-0000000102": nil, + }, }, shards: []*vtctldatapb.Shard{ { @@ -1004,10 +1057,11 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Hostname: "not most up-to-date position", }, }, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: true, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: "failed to be upgraded to primary", }, } @@ -1047,6 +1101,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { err := erp.reparentShardLocked(ctx, ev, tt.keyspace, tt.shard, tt.vtctlReparentFunctions) if tt.shouldErr { assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errShouldContain) return } @@ -1055,7 +1110,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { } } -func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { +func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { t.Parallel() tests := []struct { @@ -1067,9 +1122,11 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { ts *topo.Server keyspace string shard string + tablets []*topodatapb.Tablet tabletMap map[string]*topo.TabletInfo statusMap map[string]*replicationdatapb.StopReplicationStatus shouldErr bool + errShouldContain string }{ { name: "success", @@ -1078,9 +1135,9 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, }, - PromoteReplicaResults: map[string]struct { - Result string - Error error + MasterPositionResults: map[string]struct { + Position string + Error error }{ "zone1-0000000100": { Error: nil, @@ -1091,6 +1148,9 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { "zone1-0000000102": nil, "zone1-0000000404": assert.AnError, // okay, because we're ignoring it. }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000100": nil, + }, }, newPrimaryTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ @@ -1150,72 +1210,21 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { shouldErr: false, }, { - name: "primary not in tablet map", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), - tmc: &testutil.TabletManagerClient{}, - newPrimaryTabletAlias: "zone2-0000000200", - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": {}, - "zone1-0000000101": {}, - }, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - shouldErr: true, - }, - { - name: "PromoteReplica error", + name: "MasterPosition error", vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ - PromoteReplicaResults: map[string]struct { - Result string - Error error + MasterPositionResults: map[string]struct { + Position string + Error error }{ "zone1-0000000100": { - Error: assert.AnError, - }, - }, - }, - newPrimaryTabletAlias: "zone1-0000000100", - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, - "zone1-0000000101": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, + Error: fmt.Errorf("primary position error"), }, }, - }, - statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: true, - }, - { - name: "lost topology lock", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), - tmc: &testutil.TabletManagerClient{ - PromoteReplicaResults: map[string]struct { - Result string - Error error - }{ - "zone1-0000000100": { - Error: nil, - }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000100": nil, }, }, - unlockTopo: true, newPrimaryTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { @@ -1235,11 +1244,12 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, }, - statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: true, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: "primary position error", }, { name: "cannot repopulate reparent journal on new primary", @@ -1248,13 +1258,15 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": assert.AnError, }, - PromoteReplicaResults: map[string]struct { - Result string - Error error + MasterPositionResults: map[string]struct { + Position string + Error error }{ "zone1-0000000100": { Error: nil, }, + }, ChangeTabletTypeResult: map[string]error{ + "zone1-0000000100": nil, }, }, newPrimaryTabletAlias: "zone1-0000000100", @@ -1276,11 +1288,12 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, }, - statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: true, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: "failed to PopulateReparentJournal on primary", }, { name: "all replicas failing to SetMaster does fail the promotion", @@ -1289,9 +1302,9 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, }, - PromoteReplicaResults: map[string]struct { - Result string - Error error + MasterPositionResults: map[string]struct { + Position string + Error error }{ "zone1-0000000100": { Error: nil, @@ -1302,6 +1315,9 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { "zone1-0000000101": assert.AnError, "zone1-0000000102": assert.AnError, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000100": nil, + }, }, newPrimaryTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ @@ -1330,11 +1346,12 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, }, - statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: true, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: " replica(s) failed", }, { name: "all replicas slow to SetMaster does fail the promotion", @@ -1343,9 +1360,9 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, }, - PromoteReplicaResults: map[string]struct { - Result string - Error error + MasterPositionResults: map[string]struct { + Position string + Error error }{ "zone1-0000000100": { Error: nil, @@ -1360,6 +1377,9 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { "zone1-0000000101": nil, "zone1-0000000102": nil, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000100": nil, + }, }, newPrimaryTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ @@ -1388,11 +1408,12 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, }, }, - statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - shouldErr: true, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + shouldErr: true, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + errShouldContain: "context deadline exceeded", }, { name: "one replica failing to SetMaster does not fail the promotion", @@ -1401,14 +1422,17 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, }, - PromoteReplicaResults: map[string]struct { - Result string - Error error + MasterPositionResults: map[string]struct { + Position string + Error error }{ "zone1-0000000100": { Error: nil, }, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000100": nil, + }, SetMasterResults: map[string]error{ "zone1-0000000101": nil, // this one succeeds, so we're good "zone1-0000000102": assert.AnError, @@ -1478,10 +1502,12 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { require.NoError(t, lerr, "could not unlock %s/%s after test", tt.keyspace, tt.shard) }() } + tabletInfo := tt.tabletMap[tt.newPrimaryTabletAlias] - err := tt.vtctlReparentFunctions.promoteNewPrimary(ctx, ev, logger, tt.tmc, tt.newPrimaryTabletAlias, tt.statusMap, tt.tabletMap, tt.keyspace, tt.shard) + _, err := promotePrimaryCandidate(ctx, tt.tmc, tt.ts, ev, logger, tabletInfo.Tablet, "", tt.tabletMap, tt.statusMap, tt.vtctlReparentFunctions, true) if tt.shouldErr { assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errShouldContain) return } From 08d0e5424c8224d81da67f08da084cd35cd94db0 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 8 Sep 2021 16:21:48 +0530 Subject: [PATCH 079/176] remove unused code Signed-off-by: Manan Gupta --- .../vtctl/reparentutil/reparent_functions.go | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index d0f478a0918..c6fe1448541 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -20,8 +20,6 @@ import ( "context" "time" - "vitess.io/vitess/go/event" - "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vterrors" @@ -30,7 +28,6 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/vt/logutil" - replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topotools/events" @@ -229,25 +226,3 @@ func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePromotion(newP // PostERSCompletionHook implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) PostERSCompletionHook(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) { } - -func (vtctlReparent *VtctlReparentFunctions) promoteNewPrimary(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, winningPrimaryTabletAliasStr string, statusMap map[string]*replicationdatapb.StopReplicationStatus, tabletMap map[string]*topo.TabletInfo, keyspace, shard string) error { - logger.Infof("promoting tablet %v to master", winningPrimaryTabletAliasStr) - event.DispatchUpdate(ev, "promoting replica") - - newPrimaryTabletInfo, ok := tabletMap[winningPrimaryTabletAliasStr] - if !ok { - return vterrors.Errorf(vtrpc.Code_INTERNAL, "attempted to promote master-elect %v that was not in the tablet map; this an impossible situation", winningPrimaryTabletAliasStr) - } - - _, err := tmc.PromoteReplica(ctx, newPrimaryTabletInfo.Tablet) - if err != nil { - return vterrors.Wrapf(err, "master-elect tablet %v failed to be upgraded to master: %v", winningPrimaryTabletAliasStr, err) - } - - if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { - return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) - } - - _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimaryTabletInfo.Tablet, getLockAction(vtctlReparent.newPrimaryAlias), tabletMap, statusMap, vtctlReparent, false) - return err -} From faf94b85d09c55b7dc01eae32d22bf52de164d83 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 8 Sep 2021 16:50:13 +0530 Subject: [PATCH 080/176] fixed planned_reparenter unit tests Signed-off-by: Manan Gupta --- .../reparentutil/planned_reparenter_test.go | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/go/vt/vtctl/reparentutil/planned_reparenter_test.go b/go/vt/vtctl/reparentutil/planned_reparenter_test.go index 101346dce10..a9cc0f869a4 100644 --- a/go/vt/vtctl/reparentutil/planned_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/planned_reparenter_test.go @@ -721,6 +721,9 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { "position1": nil, }, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000200": nil, + }, }, ev: &events.Reparent{}, keyspace: "testkeyspace", @@ -1096,6 +1099,9 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { Error: nil, }, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000200": nil, + }, SetMasterResults: map[string]error{ "zone1-0000000200": nil, }, @@ -1291,6 +1297,9 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { Error: assert.AnError, }, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000200": nil, + }, SetMasterResults: map[string]error{ "zone1-0000000200": nil, }, @@ -1359,6 +1368,9 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { Error: nil, }, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000200": nil, + }, SetMasterResults: map[string]error{ "zone1-0000000200": nil, }, @@ -1666,6 +1678,9 @@ func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { Error: nil, }, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000100": nil, + }, }, unlockTopo: false, keyspace: "testkeyspace", @@ -2000,6 +2015,9 @@ func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { Error: assert.AnError, }, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000100": nil, + }, }, unlockTopo: false, keyspace: "testkeyspace", @@ -2077,6 +2095,9 @@ func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { Error: nil, }, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000100": nil, + }, }, timeout: time.Millisecond * 50, unlockTopo: false, @@ -2218,6 +2239,9 @@ func TestPlannedReparenter_reparentShardLocked(t *testing.T) { Error: nil, }, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000200": nil, + }, SetMasterResults: map[string]error{ "zone1-0000000100": nil, // zone1-100 gets reparented under zone1-200 }, @@ -2377,6 +2401,9 @@ func TestPlannedReparenter_reparentShardLocked(t *testing.T) { Error: nil, }, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000200": nil, + }, SetMasterResults: map[string]error{ "zone1-0000000100": nil, // called during reparentTablets to make oldPrimary a replica of newPrimary "zone1-0000000200": nil, // called during performGracefulPromotion to ensure newPrimary is caught up From 34f4ac7b17f8f9a0b0539fdd90971bcd4166c74c Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 8 Sep 2021 17:18:42 +0530 Subject: [PATCH 081/176] fixed reparent shard test Signed-off-by: Manan Gupta --- go/vt/wrangler/testlib/planned_reparent_shard_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/go/vt/wrangler/testlib/planned_reparent_shard_test.go b/go/vt/wrangler/testlib/planned_reparent_shard_test.go index da4035cdbfd..87e6bb2bd0c 100644 --- a/go/vt/wrangler/testlib/planned_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/planned_reparent_shard_test.go @@ -753,6 +753,9 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { "STOP SLAVE", "FAKE SET MASTER", "START SLAVE", + "CREATE DATABASE IF NOT EXISTS _vt", + "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", + "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES", // extra commands because of retry "STOP SLAVE", "FAKE SET MASTER", @@ -762,6 +765,8 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES", } oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "FAKE SET MASTER", + "START SLAVE", "FAKE SET MASTER", "START SLAVE", // extra commands because of retry From a5b777af5ad8dee97b38bf29b8316de0ebfc8e60 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 8 Sep 2021 19:37:46 +0530 Subject: [PATCH 082/176] bug fix in finding errant GTIDs Signed-off-by: Manan Gupta --- go/mysql/replication_status.go | 11 +------ go/mysql/replication_status_test.go | 47 ++++++++++++++++++----------- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/go/mysql/replication_status.go b/go/mysql/replication_status.go index 852e68d3f3c..801d38a680f 100644 --- a/go/mysql/replication_status.go +++ b/go/mysql/replication_status.go @@ -123,16 +123,7 @@ func (s *ReplicationStatus) FindErrantGTIDs(otherReplicaStatuses []*ReplicationS if !ok { panic("The receiver ReplicationStatus contained a Mysql56GTIDSet in its relay log, but a replica's ReplicationStatus is of another flavor. This should never happen.") } - // Copy and throw out primary SID from consideration, so we don't mutate input. - otherSetNoPrimarySID := make(Mysql56GTIDSet, len(otherSet)) - for sid, intervals := range otherSet { - if sid == status.SourceUUID { - continue - } - otherSetNoPrimarySID[sid] = intervals - } - - otherSets = append(otherSets, otherSetNoPrimarySID) + otherSets = append(otherSets, otherSet) } // Copy set for final diffSet so we don't mutate receiver. diff --git a/go/mysql/replication_status_test.go b/go/mysql/replication_status_test.go index 0a9358fb912..6c23eee3aef 100644 --- a/go/mysql/replication_status_test.go +++ b/go/mysql/replication_status_test.go @@ -18,6 +18,8 @@ package mysql import ( "testing" + + "github.com/stretchr/testify/require" ) func TestStatusReplicationRunning(t *testing.T) { @@ -81,22 +83,33 @@ func TestFindErrantGTIDs(t *testing.T) { sourceSID: []interval{{2, 6}, {15, 45}}, } - status1 := ReplicationStatus{SourceUUID: sourceSID, RelayLogPosition: Position{GTIDSet: set1}} - status2 := ReplicationStatus{SourceUUID: sourceSID, RelayLogPosition: Position{GTIDSet: set2}} - status3 := ReplicationStatus{SourceUUID: sourceSID, RelayLogPosition: Position{GTIDSet: set3}} - - got, err := status1.FindErrantGTIDs([]*ReplicationStatus{&status2, &status3}) - if err != nil { - t.Errorf("%v", err) - } - - want := Mysql56GTIDSet{ - sid1: []interval{{39, 39}, {40, 49}, {71, 75}}, - sid2: []interval{{1, 2}, {6, 7}, {20, 21}, {26, 31}, {38, 50}, {60, 66}}, - sid4: []interval{{1, 30}}, - } - - if !got.Equal(want) { - t.Errorf("got %#v; want %#v", got, want) + testcases := []struct { + mainRepStatus *ReplicationStatus + otherRepStatuses []*ReplicationStatus + want Mysql56GTIDSet + }{{ + mainRepStatus: &ReplicationStatus{SourceUUID: sourceSID, RelayLogPosition: Position{GTIDSet: set1}}, + otherRepStatuses: []*ReplicationStatus{ + {SourceUUID: sourceSID, RelayLogPosition: Position{GTIDSet: set2}}, + {SourceUUID: sourceSID, RelayLogPosition: Position{GTIDSet: set3}}, + }, + want: Mysql56GTIDSet{ + sid1: []interval{{39, 39}, {40, 49}, {71, 75}}, + sid2: []interval{{1, 2}, {6, 7}, {20, 21}, {26, 31}, {38, 50}, {60, 66}}, + sid4: []interval{{1, 30}}, + }, + }, { + mainRepStatus: &ReplicationStatus{SourceUUID: sourceSID, RelayLogPosition: Position{GTIDSet: set1}}, + otherRepStatuses: []*ReplicationStatus{{SourceUUID: sid1, RelayLogPosition: Position{GTIDSet: set1}}}, + // servers with the same GTID sets should not be diagnosed with errant GTIDs + want: nil, + }} + + for _, testcase := range testcases { + t.Run("", func(t *testing.T) { + got, err := testcase.mainRepStatus.FindErrantGTIDs(testcase.otherRepStatuses) + require.NoError(t, err) + require.Equal(t, testcase.want, got) + }) } } From 07d2e88b2e0038aa02f74db197c7f178c83a3753 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 9 Sep 2021 11:52:35 +0530 Subject: [PATCH 083/176] fix slow server tests Signed-off-by: Manan Gupta --- .../grpcvtctldserver/server_slow_test.go | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server_slow_test.go b/go/vt/vtctl/grpcvtctldserver/server_slow_test.go index 001daec8cf5..bab7af82196 100644 --- a/go/vt/vtctl/grpcvtctldserver/server_slow_test.go +++ b/go/vt/vtctl/grpcvtctldserver/server_slow_test.go @@ -42,7 +42,7 @@ func TestEmergencyReparentShardSlow(t *testing.T) { tests := []struct { name string ts *topo.Server - tmc *testutil.TabletManagerClient + tmc tmclient.TabletManagerClient tablets []*topodatapb.Tablet req *vtctldatapb.EmergencyReparentShardRequest @@ -116,6 +116,15 @@ func TestEmergencyReparentShardSlow(t *testing.T) { }{ "zone1-0000000200": {}, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000200": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000200": {}, + }, SetMasterResults: map[string]error{ "zone1-0000000100": nil, "zone1-0000000101": nil, @@ -229,6 +238,15 @@ func TestEmergencyReparentShardSlow(t *testing.T) { }{ "zone1-0000000200": {}, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000200": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000200": {}, + }, SetMasterResults: map[string]error{ "zone1-0000000100": nil, "zone1-0000000101": nil, @@ -296,8 +314,6 @@ func TestEmergencyReparentShardSlow(t *testing.T) { SkipShardCreation: false, }, tt.tablets...) - tt.tmc.TopoServer = tt.ts - vtctld := testutil.NewVtctldServerWithTabletManagerClient(t, tt.ts, tt.tmc, func(ts *topo.Server) vtctlservicepb.VtctldServer { return NewVtctldServer(ts) }) @@ -416,6 +432,9 @@ func TestPlannedReparentShardSlow(t *testing.T) { Error: nil, }, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000200": nil, + }, SetMasterResults: map[string]error{ "zone1-0000000200": nil, // waiting for master-position during promotion // reparent SetMaster calls @@ -519,6 +538,9 @@ func TestPlannedReparentShardSlow(t *testing.T) { Error: nil, }, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000200": nil, + }, SetMasterResults: map[string]error{ "zone1-0000000200": nil, // waiting for master-position during promotion // reparent SetMaster calls From 536b93377227d2081772669d4e7659e33577e717 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 9 Sep 2021 13:21:04 +0530 Subject: [PATCH 084/176] added test in ERS for testing preference of a candidate in the same cell Signed-off-by: Manan Gupta --- go/test/endtoend/reparent/ers_test.go | 134 +++++++++++++++++++++ go/test/endtoend/reparent/reparent_test.go | 85 ------------- 2 files changed, 134 insertions(+), 85 deletions(-) create mode 100644 go/test/endtoend/reparent/ers_test.go diff --git a/go/test/endtoend/reparent/ers_test.go b/go/test/endtoend/reparent/ers_test.go new file mode 100644 index 00000000000..dc5570f7bcc --- /dev/null +++ b/go/test/endtoend/reparent/ers_test.go @@ -0,0 +1,134 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reparent + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/vt/log" +) + +func TestTrivialERS(t *testing.T) { + defer cluster.PanicHandler(t) + setupReparentCluster(t) + defer teardownCluster() + + confirmReplication(t, tab1, []*cluster.Vttablet{tab2, tab3, tab4}) + + // We should be able to do a series of ERS-es, even if nothing + // is down, without issue + for i := 1; i <= 4; i++ { + out, err := ers(t, nil, "30s") + log.Infof("ERS loop %d. EmergencyReparentShard Output: %v", i, out) + require.NoError(t, err) + time.Sleep(5 * time.Second) + } +} + +func TestReparentIgnoreReplicas(t *testing.T) { + defer cluster.PanicHandler(t) + setupReparentCluster(t) + defer teardownCluster() + var err error + + ctx := context.Background() + + confirmReplication(t, tab1, []*cluster.Vttablet{tab2, tab3, tab4}) + + // Make the current primary agent and database unavailable. + stopTablet(t, tab1, true) + + // Take down a replica - this should cause the emergency reparent to fail. + stopTablet(t, tab3, true) + + // We expect this one to fail because we have an unreachable replica + out, err := ers(t, nil, "30s") + require.NotNil(t, err, out) + + // Now let's run it again, but set the command to ignore the unreachable replica. + out, err = ersIgnoreTablet(t, nil, "30s", []*cluster.Vttablet{tab3}) + require.Nil(t, err, out) + + // We'll bring back the replica we took down. + restartTablet(t, tab3) + + // Check that old primary tablet is left around for human intervention. + confirmOldPrimaryIsHangingAround(t) + deleteTablet(t, tab1) + validateTopology(t, false) + + newPrimary := getNewPrimary(t) + // Check new primary has latest transaction. + err = checkInsertedValues(ctx, t, newPrimary, insertVal) + require.Nil(t, err) + + // bring back the old primary as a replica, check that it catches up + resurrectTablet(ctx, t, tab1) +} + +// TestERSPromoteRdonly tests that we never end up promoting a rdonly instance as the primary +func TestERSPromoteRdonly(t *testing.T) { + defer cluster.PanicHandler(t) + setupReparentCluster(t) + defer teardownCluster() + var err error + + err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", tab2.Alias, "rdonly") + require.NoError(t, err) + + err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", tab3.Alias, "rdonly") + require.NoError(t, err) + + confirmReplication(t, tab1, []*cluster.Vttablet{tab2, tab3, tab4}) + + // Make the current primary agent and database unavailable. + stopTablet(t, tab1, true) + + // We expect this one to fail because we have ignored all the replicas and have only the rdonly's which should not be promoted + out, err := ersIgnoreTablet(t, nil, "30s", []*cluster.Vttablet{tab4}) + require.NotNil(t, err, out) + + out, err = clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetShard", keyspaceShard) + require.NoError(t, err) + require.Contains(t, out, `"uid": 101`, "the primary should still be 101 in the shard info") +} + +// TestERSPrefersSameCell tests that we prefer to promote a replica in the same cell as the previous primary +func TestERSPrefersSameCell(t *testing.T) { + defer cluster.PanicHandler(t) + setupReparentCluster(t) + defer teardownCluster() + var err error + + // confirm that replication is going smoothly + confirmReplication(t, tab1, []*cluster.Vttablet{tab2, tab3, tab4}) + + // Make the current primary agent and database unavailable. + stopTablet(t, tab1, true) + + // We expect that tab3 will be promoted since it is in the same cell as the previous primary + out, err := ersIgnoreTablet(t, nil, "30s", []*cluster.Vttablet{tab2}) + require.NoError(t, err, out) + + newPrimary := getNewPrimary(t) + require.Equal(t, newPrimary.Alias, tab3.Alias, "tab3 should be the promoted primary") +} diff --git a/go/test/endtoend/reparent/reparent_test.go b/go/test/endtoend/reparent/reparent_test.go index 0dbdea87bbf..12220cba3b0 100644 --- a/go/test/endtoend/reparent/reparent_test.go +++ b/go/test/endtoend/reparent/reparent_test.go @@ -118,91 +118,6 @@ func TestReparentNoChoiceDownPrimary(t *testing.T) { resurrectTablet(ctx, t, tab1) } -func TestTrivialERS(t *testing.T) { - defer cluster.PanicHandler(t) - setupReparentCluster(t) - defer teardownCluster() - - confirmReplication(t, tab1, []*cluster.Vttablet{tab2, tab3, tab4}) - - // We should be able to do a series of ERS-es, even if nothing - // is down, without issue - for i := 1; i <= 4; i++ { - out, err := ers(t, nil, "30s") - log.Infof("ERS loop %d. EmergencyReparentShard Output: %v", i, out) - require.NoError(t, err) - time.Sleep(5 * time.Second) - } -} - -func TestReparentIgnoreReplicas(t *testing.T) { - defer cluster.PanicHandler(t) - setupReparentCluster(t) - defer teardownCluster() - var err error - - ctx := context.Background() - - confirmReplication(t, tab1, []*cluster.Vttablet{tab2, tab3, tab4}) - - // Make the current primary agent and database unavailable. - stopTablet(t, tab1, true) - - // Take down a replica - this should cause the emergency reparent to fail. - stopTablet(t, tab3, true) - - // We expect this one to fail because we have an unreachable replica - out, err := ers(t, nil, "30s") - require.NotNil(t, err, out) - - // Now let's run it again, but set the command to ignore the unreachable replica. - out, err = ersIgnoreTablet(t, nil, "30s", []*cluster.Vttablet{tab3}) - require.Nil(t, err, out) - - // We'll bring back the replica we took down. - restartTablet(t, tab3) - - // Check that old primary tablet is left around for human intervention. - confirmOldPrimaryIsHangingAround(t) - deleteTablet(t, tab1) - validateTopology(t, false) - - newPrimary := getNewPrimary(t) - // Check new primary has latest transaction. - err = checkInsertedValues(ctx, t, newPrimary, insertVal) - require.Nil(t, err) - - // bring back the old primary as a replica, check that it catches up - resurrectTablet(ctx, t, tab1) -} - -// TestERSPromoteRdonly tests that we never end up promoting a rdonly instance as the primary -func TestERSPromoteRdonly(t *testing.T) { - defer cluster.PanicHandler(t) - setupReparentCluster(t) - defer teardownCluster() - var err error - - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", tab2.Alias, "rdonly") - require.NoError(t, err) - - err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", tab3.Alias, "rdonly") - require.NoError(t, err) - - confirmReplication(t, tab1, []*cluster.Vttablet{tab2, tab3, tab4}) - - // Make the current primary agent and database unavailable. - stopTablet(t, tab1, true) - - // We expect this one to fail because we have ignored all the replicas and have only the rdonly's which should not be promoted - out, err := ersIgnoreTablet(t, nil, "30s", []*cluster.Vttablet{tab4}) - require.NotNil(t, err, out) - - out, err = clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetShard", keyspaceShard) - require.NoError(t, err) - require.Contains(t, out, `"uid": 101`, "the primary should still be 101 in the shard info") -} - func TestReparentCrossCell(t *testing.T) { defer cluster.PanicHandler(t) setupReparentCluster(t) From 9a654d58f7c60f2a6c759c01fdff923148319646 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 9 Sep 2021 13:54:30 +0530 Subject: [PATCH 085/176] fix server unit test for vtctldserver Signed-off-by: Manan Gupta --- go/vt/vtctl/grpcvtctldserver/server_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/go/vt/vtctl/grpcvtctldserver/server_test.go b/go/vt/vtctl/grpcvtctldserver/server_test.go index 1518503ebd2..270a3de25a5 100644 --- a/go/vt/vtctl/grpcvtctldserver/server_test.go +++ b/go/vt/vtctl/grpcvtctldserver/server_test.go @@ -2652,6 +2652,15 @@ func TestEmergencyReparentShard(t *testing.T) { }{ "zone1-0000000200": {}, }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000200": {}, + }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000200": nil, + }, SetMasterResults: map[string]error{ "zone1-0000000100": nil, "zone1-0000000101": nil, @@ -4677,6 +4686,9 @@ func TestPlannedReparentShard(t *testing.T) { Error: nil, }, }, + ChangeTabletTypeResult: map[string]error{ + "zone1-0000000200": nil, + }, SetMasterResults: map[string]error{ "zone1-0000000200": nil, // waiting for master-position during promotion // reparent SetMaster calls From c68f8647a9d318da2b8ce88b16688463749c7897 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 9 Sep 2021 15:02:27 +0530 Subject: [PATCH 086/176] found data race Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/util.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index e214d3aa2bb..d16a8de2dde 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -162,6 +162,7 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent return } + // TODO: data race over here replicasStartedReplication = append(replicasStartedReplication, ti.Tablet) // We call PostTabletChangeHook every time there is an update to a tablet's replication or type reparentFunctions.PostTabletChangeHook(ti.Tablet) From 644e0d62eac19e366d5b13758ce4124cffbfcb9e Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 9 Sep 2021 15:22:27 +0530 Subject: [PATCH 087/176] added a todo Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/util.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index d16a8de2dde..551e98ceb2e 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -384,6 +384,7 @@ func replaceWithBetterCandidate(ctx context.Context, tmc tmclient.TabletManagerC } // Wait until the new primary has caught upto that position + // TODO - discuss, what happens in case of timeout err = tmc.WaitForPosition(ctx, newPrimary, pos) if err != nil { return err From 115e57b1ab9080ab5909ee2f6bf58b7c808a9823 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 10 Sep 2021 10:22:52 +0530 Subject: [PATCH 088/176] handle data race Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/util.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 551e98ceb2e..19b747b9b6f 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -99,12 +99,15 @@ func waitForAllRelayLogsToApply(ctx context.Context, logger logutil.Logger, tmc return nil } +// reparentReplicasAndPopulateJournal reparents all the replicas provided and populates the reparent journal on the primary. +// Also, it returns the replicas which started replicating only in the case where we wait for all the replicas func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, newPrimaryTablet *topodatapb.Tablet, lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions, waitForAllReplicas bool) ([]*topodatapb.Tablet, error) { var replicasStartedReplication []*topodatapb.Tablet + var replicaMutex sync.Mutex replCtx, replCancel := context.WithTimeout(ctx, reparentFunctions.GetWaitReplicasTimeout()) defer replCancel() @@ -162,8 +165,9 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent return } - // TODO: data race over here + replicaMutex.Lock() replicasStartedReplication = append(replicasStartedReplication, ti.Tablet) + replicaMutex.Unlock() // We call PostTabletChangeHook every time there is an update to a tablet's replication or type reparentFunctions.PostTabletChangeHook(ti.Tablet) @@ -212,7 +216,8 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent select { case <-replSuccessCtx.Done(): // At least one replica was able to SetMaster successfully - return replicasStartedReplication, nil + // Here we do not need to return the replicas which started replicating + return nil, nil case <-allReplicasDoneCtx.Done(): // There are certain timing issues between replSuccessCtx.Done firing // and allReplicasDoneCtx.Done firing, so we check again if truly all From 1ccb8ef38095670aab2041898ccd93964eb3ff46 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 10 Sep 2021 13:07:42 +0530 Subject: [PATCH 089/176] fix vtorc tests Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/main_test.go | 2 +- .../endtoend/vtorc/primary_failure_test.go | 20 ++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/go/test/endtoend/vtorc/main_test.go b/go/test/endtoend/vtorc/main_test.go index 1e540f15415..77ba9425172 100644 --- a/go/test/endtoend/vtorc/main_test.go +++ b/go/test/endtoend/vtorc/main_test.go @@ -330,7 +330,7 @@ func TestMain(m *testing.M) { }) cellInfos = append(cellInfos, &cellInfo{ cellName: cell2, - numReplicas: 1, + numReplicas: 2, numRdonly: 0, uidBase: 200, }) diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index f4295dd069d..49a3956acf7 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -131,9 +131,10 @@ func TestCrossDataCenterFailureError(t *testing.T) { } assert.NotNil(t, rdonly, "could not find rdonly tablet") - crossCellReplica := startVttablet(t, cell2, false) + crossCellReplica1 := startVttablet(t, cell2, false) + crossCellReplica2 := startVttablet(t, cell2, false) // newly started tablet does not replicate from anyone yet, we will allow orchestrator to fix this too - checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{crossCellReplica, rdonly}, 25*time.Second) + checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{crossCellReplica1, crossCellReplica2, rdonly}, 25*time.Second) // Make the current primary database unavailable. err := curPrimary.MysqlctlProcess.Stop() @@ -275,7 +276,7 @@ func TestPromotionLagSuccess(t *testing.T) { // covers the test case master-failover-fail-promotion-lag-minutes-failure from orchestrator func TestPromotionLagFailure(t *testing.T) { defer cluster.PanicHandler(t) - setupVttabletsAndVtorc(t, 2, 1, nil, "test_config_promotion_failure.json") + setupVttabletsAndVtorc(t, 3, 1, nil, "test_config_promotion_failure.json") keyspace := &clusterInstance.Keyspaces[0] shard0 := &keyspace.Shards[0] // find primary from topo @@ -283,21 +284,26 @@ func TestPromotionLagFailure(t *testing.T) { assert.NotNil(t, curPrimary, "should have elected a primary") // find the replica and rdonly tablets - var replica, rdonly *cluster.Vttablet + var replica1, replica2, rdonly *cluster.Vttablet for _, tablet := range shard0.Vttablets { // we know we have only two replcia tablets, so the one not the primary must be the other replica if tablet.Alias != curPrimary.Alias && tablet.Type == "replica" { - replica = tablet + if replica1 == nil { + replica1 = tablet + } else { + replica2 = tablet + } } if tablet.Type == "rdonly" { rdonly = tablet } } - assert.NotNil(t, replica, "could not find replica tablet") + assert.NotNil(t, replica1, "could not find replica tablet") + assert.NotNil(t, replica2, "could not find second replica tablet") assert.NotNil(t, rdonly, "could not find rdonly tablet") // check that the replication is setup correctly before we failover - checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{rdonly, replica}, 10*time.Second) + checkReplication(t, clusterInstance, curPrimary, []*cluster.Vttablet{rdonly, replica1, replica2}, 10*time.Second) // Make the current primary database unavailable. err := curPrimary.MysqlctlProcess.Stop() From 488ae35966e10de63d35852aae77b53e95d1f37c Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 13 Sep 2021 17:38:24 +0530 Subject: [PATCH 090/176] remove lock shard from interface Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/reparent_functions.go | 10 ---------- go/vt/orchestrator/logic/topology_recovery.go | 10 ++++++++++ go/vt/vtctl/reparentutil/emergency_reparenter.go | 2 +- go/vt/vtctl/reparentutil/reparent_functions.go | 6 ------ 4 files changed, 11 insertions(+), 17 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index bfefdc0c9fb..84212418365 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -71,16 +71,6 @@ func NewVtorcReparentFunctions(analysisEntry inst.ReplicationAnalysis, candidate } } -// LockShard implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) LockShard(ctx context.Context, logger logutil.Logger, ts *topo.Server, keyspace string, shard string) (context.Context, func(*error), error) { - ctx, unlock, err := LockShard(ctx, vtorcReparent.analysisEntry.AnalyzedInstanceKey) - if err != nil { - logger.Infof("could not obtain shard lock (%v)", err) - return nil, nil, err - } - return ctx, unlock, nil -} - // LockAction implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) LockAction() string { return "Orc Recovery" diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 2a924964ccb..033ca09b393 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -19,6 +19,7 @@ package logic import ( "context" "encoding/json" + "errors" "fmt" "math/rand" goos "os" @@ -628,6 +629,15 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat } log.Infof("Analysis: %v, deadprimary %+v", analysisEntry.Analysis, analysisEntry.AnalyzedInstanceKey) + // check if we have received SIGTERM, if we have, we should not continue with the recovery + val := atomic.LoadInt32(&hasReceivedSIGTERM) + if val > 0 { + return false, topologyRecovery, errors.New("Can't lock shard: SIGTERM received") + } + // add to the shard lock counter since ERS will lock the shard + atomic.AddInt32(&shardsLockCounter, -1) + defer atomic.AddInt32(&shardsLockCounter, -1) + reparentFunctions := NewVtorcReparentFunctions(analysisEntry, candidateInstanceKey, skipProcesses, topologyRecovery) _, err = reparentutil.NewEmergencyReparenter(ts, tmclient.NewTabletManagerClient(), logutil.NewCallbackLogger(func(event *logutilpb.Event) { level := event.GetLevel() diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index e911f4d4b65..e8c133fe7f0 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -73,7 +73,7 @@ func NewEmergencyReparenter(ts *topo.Server, tmc tmclient.TabletManagerClient, l // keyspace and shard. func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, shard string, reparentFunctions ReparentFunctions) (*events.Reparent, error) { // First step is to lock the shard for the given operation - ctx, unlock, err := reparentFunctions.LockShard(ctx, erp.logger, erp.ts, keyspace, shard) + ctx, unlock, err := erp.ts.LockShard(ctx, keyspace, shard, reparentFunctions.LockAction()) if err != nil { return nil, err } diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index c6fe1448541..3a56c464e2d 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -37,7 +37,6 @@ import ( type ( // ReparentFunctions is an interface which has all the functions implementation required for re-parenting ReparentFunctions interface { - LockShard(context.Context, logutil.Logger, *topo.Server, string, string) (context.Context, func(*error), error) LockAction() string CheckIfFixed() bool PreRecoveryProcesses(context.Context) error @@ -76,11 +75,6 @@ func NewVtctlReparentFunctions(newPrimaryAlias *topodatapb.TabletAlias, ignoreRe } } -// LockShard implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) LockShard(ctx context.Context, logger logutil.Logger, ts *topo.Server, keyspace string, shard string) (context.Context, func(*error), error) { - return ts.LockShard(ctx, keyspace, shard, getLockAction(vtctlReparent.newPrimaryAlias)) -} - // LockAction implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) LockAction() string { return getLockAction(vtctlReparent.newPrimaryAlias) From 6d65dfdbba23b864f2820356873ab3d4f9e00475 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 13 Sep 2021 17:47:00 +0530 Subject: [PATCH 091/176] add atomic counter so that only 1 ers is issued at a time Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/orchestrator.go | 1 + go/vt/orchestrator/logic/topology_recovery.go | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/go/vt/orchestrator/logic/orchestrator.go b/go/vt/orchestrator/logic/orchestrator.go index 1ae49e0cb93..44223237b25 100644 --- a/go/vt/orchestrator/logic/orchestrator.go +++ b/go/vt/orchestrator/logic/orchestrator.go @@ -50,6 +50,7 @@ var discoveryQueue *discovery.Queue var snapshotDiscoveryKeys chan inst.InstanceKey var snapshotDiscoveryKeysMutex sync.Mutex var hasReceivedSIGTERM int32 +var ersInProgress int32 var discoveriesCounter = metrics.NewCounter() var failedDiscoveriesCounter = metrics.NewCounter() diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 033ca09b393..72f20d8b38e 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -634,6 +634,17 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat if val > 0 { return false, topologyRecovery, errors.New("Can't lock shard: SIGTERM received") } + + // check if we have received an ERS in progress, if we do, we should not continue with the recovery + val = atomic.LoadInt32(&ersInProgress) + if val > 0 { + AuditTopologyRecovery(topologyRecovery, "an ERS is already in progress, not issuing another") + return false, topologyRecovery, nil + } + // set the ers in progress + atomic.StoreInt32(&ersInProgress, 1) + defer atomic.StoreInt32(&ersInProgress, 0) + // add to the shard lock counter since ERS will lock the shard atomic.AddInt32(&shardsLockCounter, -1) defer atomic.AddInt32(&shardsLockCounter, -1) From cf2c99ef87880102482d698a882d351f03edc6a3 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 13 Sep 2021 18:00:54 +0530 Subject: [PATCH 092/176] remove the pre-recovery processes from ers Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/reparent_functions.go | 13 ------------- go/vt/vtctl/reparentutil/emergency_reparenter.go | 5 ----- go/vt/vtctl/reparentutil/reparent_functions.go | 7 ------- 3 files changed, 25 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 84212418365..ff4af6f8271 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -92,19 +92,6 @@ func (vtorcReparent *VtOrcReparentFunctions) CheckIfFixed() bool { return false } -// PreRecoveryProcesses implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) PreRecoveryProcesses(ctx context.Context) error { - inst.AuditOperation("recover-dead-primary", &vtorcReparent.analysisEntry.AnalyzedInstanceKey, "problem found; will recover") - if !vtorcReparent.skipProcesses { - if err := executeProcesses(config.Config.PreFailoverProcesses, "PreFailoverProcesses", vtorcReparent.topologyRecovery, true); err != nil { - return vtorcReparent.topologyRecovery.AddError(err) - } - } - - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: will recover %+v", vtorcReparent.analysisEntry.AnalyzedInstanceKey)) - return nil -} - // GetWaitReplicasTimeout implements the ReparentFunctions interface // TODO : Discuss correct way func (vtorcReparent *VtOrcReparentFunctions) GetWaitReplicasTimeout() time.Duration { diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index e8c133fe7f0..c555c5b5040 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -123,11 +123,6 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve prevPrimary = prevPrimaryInfo.Tablet } - // run the pre recovery processes - if err := reparentFunctions.PreRecoveryProcesses(ctx); err != nil { - return err - } - // check that the primary recovery type is a valid one if err := reparentFunctions.CheckPrimaryRecoveryType(erp.logger); err != nil { return err diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 3a56c464e2d..f2ab2d9714b 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -39,7 +39,6 @@ type ( ReparentFunctions interface { LockAction() string CheckIfFixed() bool - PreRecoveryProcesses(context.Context) error CheckPrimaryRecoveryType(logutil.Logger) error GetWaitReplicasTimeout() time.Duration GetWaitForRelayLogsTimeout() time.Duration @@ -87,12 +86,6 @@ func (vtctlReparent *VtctlReparentFunctions) CheckIfFixed() bool { return false } -// PreRecoveryProcesses implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) PreRecoveryProcesses(ctx context.Context) error { - // there are no pre-recovery processes to be run - return nil -} - // GetWaitReplicasTimeout implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) GetWaitReplicasTimeout() time.Duration { return vtctlReparent.waitReplicasTimeout From 8100ffd7be31bcc9fda79a55594b636eaec0b89d Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 13 Sep 2021 18:02:19 +0530 Subject: [PATCH 093/176] remove the check for primary recovery type from ers Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/reparent_functions.go | 12 ------------ go/vt/vtctl/reparentutil/emergency_reparenter.go | 5 ----- go/vt/vtctl/reparentutil/reparent_functions.go | 6 ------ 3 files changed, 23 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index ff4af6f8271..ee3c154337c 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -119,18 +119,6 @@ func (vtorcReparent *VtOrcReparentFunctions) GetIgnoreReplicas() sets.String { return nil } -// CheckPrimaryRecoveryType implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) CheckPrimaryRecoveryType(logger logutil.Logger) error { - vtorcReparent.topologyRecovery.RecoveryType = GetPrimaryRecoveryType(&vtorcReparent.topologyRecovery.AnalysisEntry) - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: primaryRecoveryType=%+v", vtorcReparent.topologyRecovery.RecoveryType)) - if vtorcReparent.topologyRecovery.RecoveryType != PrimaryRecoveryGTID { - err := fmt.Errorf("RecoveryType unknown/unsupported") - logger.Error(err) - return vtorcReparent.topologyRecovery.AddError(err) - } - return nil -} - // RestrictValidCandidates implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) RestrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { // we do not restrict the valid candidates for VtOrc for 2 reasons - diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index c555c5b5040..e89ed1ead97 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -123,11 +123,6 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve prevPrimary = prevPrimaryInfo.Tablet } - // check that the primary recovery type is a valid one - if err := reparentFunctions.CheckPrimaryRecoveryType(erp.logger); err != nil { - return err - } - // read all the tablets and there information event.DispatchUpdate(ev, "reading all tablets") tabletMap, err := erp.ts.GetTabletMapForShard(ctx, keyspace, shard) diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index f2ab2d9714b..cdb41557506 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -39,7 +39,6 @@ type ( ReparentFunctions interface { LockAction() string CheckIfFixed() bool - CheckPrimaryRecoveryType(logutil.Logger) error GetWaitReplicasTimeout() time.Duration GetWaitForRelayLogsTimeout() time.Duration HandleRelayLogFailure(logutil.Logger, error) error @@ -108,11 +107,6 @@ func (vtctlReparent *VtctlReparentFunctions) GetIgnoreReplicas() sets.String { return vtctlReparent.ignoreReplicas } -// CheckPrimaryRecoveryType implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) CheckPrimaryRecoveryType(logutil.Logger) error { - return nil -} - // RestrictValidCandidates implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) RestrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { restrictedValidCandidates := make(map[string]mysql.Position) From cabc4a785c641de7ba87009e97cc1f1bbf115ba8 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 15 Sep 2021 12:17:50 +0530 Subject: [PATCH 094/176] removed check fixed function from the interface Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/reparent_functions.go | 16 ---------------- go/vt/vtctl/reparentutil/emergency_reparenter.go | 14 ++++++++++---- go/vt/vtctl/reparentutil/reparent_functions.go | 8 -------- 3 files changed, 10 insertions(+), 28 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index ee3c154337c..89904ba0d1d 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -76,22 +76,6 @@ func (vtorcReparent *VtOrcReparentFunctions) LockAction() string { return "Orc Recovery" } -// CheckIfFixed implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) CheckIfFixed() bool { - // Check if someone else fixed the problem. - tablet, err := TabletRefresh(vtorcReparent.analysisEntry.AnalyzedInstanceKey) - if err == nil && tablet.Type != topodatapb.TabletType_PRIMARY { - // TODO(sougou); use a version that only refreshes the current shard. - RefreshTablets() - AuditTopologyRecovery(vtorcReparent.topologyRecovery, "another agent seems to have fixed the problem") - // TODO(sougou): see if we have to reset the cluster as healthy. - return true - } - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("will handle DeadPrimary event on %+v", vtorcReparent.analysisEntry.ClusterDetails.ClusterName)) - recoverDeadPrimaryCounter.Inc(1) - return false -} - // GetWaitReplicasTimeout implements the ReparentFunctions interface // TODO : Discuss correct way func (vtorcReparent *VtOrcReparentFunctions) GetWaitReplicasTimeout() time.Duration { diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index e89ed1ead97..a3509c6154f 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -20,6 +20,8 @@ import ( "context" "time" + "vitess.io/vitess/go/stats" + replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" "vitess.io/vitess/go/vt/topo/topoproto" @@ -48,6 +50,11 @@ type EmergencyReparenter struct { logger logutil.Logger } +// counters for Emergency Reparent Shard +var ( + ersCounter = stats.NewGauge("ers_counter", "Number of time Emergency Reparent Shard has been run") +) + // NewEmergencyReparenter returns a new EmergencyReparenter object, ready to // perform EmergencyReparentShard operations using the given topo.Server, // TabletManagerClient, and logger. @@ -101,10 +108,9 @@ func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, sha // reparentShardLocked performs Emergency Reparent Shard operation assuming that the shard is already locked func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *events.Reparent, keyspace, shard string, reparentFunctions ReparentFunctions) error { - // check whether ERS is required or it has been fixed via an external agent - if reparentFunctions.CheckIfFixed() { - return nil - } + // log the starting of the operation and increment the counter + erp.logger.Infof("will initiate emergency reparent shard in keyspace - %s, shard - %s", keyspace, shard) + ersCounter.Add(1) // get the shard information from the topology server shardInfo, err := erp.ts.GetShard(ctx, keyspace, shard) diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index cdb41557506..2ccef5d826c 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -38,7 +38,6 @@ type ( // ReparentFunctions is an interface which has all the functions implementation required for re-parenting ReparentFunctions interface { LockAction() string - CheckIfFixed() bool GetWaitReplicasTimeout() time.Duration GetWaitForRelayLogsTimeout() time.Duration HandleRelayLogFailure(logutil.Logger, error) error @@ -78,13 +77,6 @@ func (vtctlReparent *VtctlReparentFunctions) LockAction() string { return getLockAction(vtctlReparent.newPrimaryAlias) } -// CheckIfFixed implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) CheckIfFixed() bool { - // For vtctl command, we know there is no other third party to fix this. - // If a user has called this command, then we should execute EmergencyReparentShard - return false -} - // GetWaitReplicasTimeout implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) GetWaitReplicasTimeout() time.Duration { return vtctlReparent.waitReplicasTimeout From 7c19858310fa5be6df77ce7db05c4822b4599637 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 15 Sep 2021 12:36:42 +0530 Subject: [PATCH 095/176] bug fix for shard counter Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/topology_recovery.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 72f20d8b38e..a054c38a708 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -187,7 +187,6 @@ func (this InstancesByCountReplicas) Less(i, j int) bool { return len(this[i].Replicas) < len(this[j].Replicas) } -var recoverDeadPrimaryCounter = metrics.NewCounter() var recoverDeadPrimarySuccessCounter = metrics.NewCounter() var recoverDeadPrimaryFailureCounter = metrics.NewCounter() var recoverDeadIntermediatePrimaryCounter = metrics.NewCounter() @@ -199,7 +198,6 @@ var recoverDeadCoPrimaryFailureCounter = metrics.NewCounter() var countPendingRecoveriesGauge = metrics.NewGauge() func init() { - metrics.Register("recover.dead_primary.start", recoverDeadPrimaryCounter) metrics.Register("recover.dead_primary.success", recoverDeadPrimarySuccessCounter) metrics.Register("recover.dead_primary.fail", recoverDeadPrimaryFailureCounter) metrics.Register("recover.dead_intermediate_primary.start", recoverDeadIntermediatePrimaryCounter) @@ -646,7 +644,7 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat defer atomic.StoreInt32(&ersInProgress, 0) // add to the shard lock counter since ERS will lock the shard - atomic.AddInt32(&shardsLockCounter, -1) + atomic.AddInt32(&shardsLockCounter, 1) defer atomic.AddInt32(&shardsLockCounter, -1) reparentFunctions := NewVtorcReparentFunctions(analysisEntry, candidateInstanceKey, skipProcesses, topologyRecovery) From 136f825da25752f351d3940454a8afd14d1b789a Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 15 Sep 2021 12:53:36 +0530 Subject: [PATCH 096/176] move vtorc to use vtctl ers Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/topology_recovery.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index a054c38a708..aeaca6d8baf 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -620,6 +620,14 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat return false, nil, err } + var candidateTabletAlias *topodatapb.TabletAlias + if candidateInstanceKey != nil { + candidateTablet, err := inst.ReadTablet(*candidateInstanceKey) + if err != nil { + return false, nil, err + } + candidateTabletAlias = candidateTablet.Alias + } topologyRecovery, err = AttemptRecoveryRegistration(&analysisEntry, !forceInstanceRecovery, !forceInstanceRecovery) if topologyRecovery == nil { AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("found an active or recent recovery on %+v. Will not issue another RecoverDeadPrimary.", analysisEntry.AnalyzedInstanceKey)) @@ -647,7 +655,8 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat atomic.AddInt32(&shardsLockCounter, 1) defer atomic.AddInt32(&shardsLockCounter, -1) - reparentFunctions := NewVtorcReparentFunctions(analysisEntry, candidateInstanceKey, skipProcesses, topologyRecovery) + // TODO: Fix durations + reparentFunctions := reparentutil.NewVtctlReparentFunctions(candidateTabletAlias, nil, 1*time.Second) _, err = reparentutil.NewEmergencyReparenter(ts, tmclient.NewTabletManagerClient(), logutil.NewCallbackLogger(func(event *logutilpb.Event) { level := event.GetLevel() value := event.GetValue() @@ -661,7 +670,8 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat } })).ReparentShard(context.Background(), tablet.Keyspace, tablet.Shard, reparentFunctions) - return reparentFunctions.recoveryAttempted, topologyRecovery, err + // TODO: fix recovery attempted + return true, topologyRecovery, err } // isGenerallyValidAsCandidateSiblingOfIntermediatePrimary sees that basic server configuration and state are valid From aaee0ae5c093944f1339822f4a0e57206e527798 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 15 Sep 2021 12:58:08 +0530 Subject: [PATCH 097/176] also audit the steps in the callback logger Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/topology_recovery.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index aeaca6d8baf..18706eeab89 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -668,6 +668,7 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat default: log.Infof("ERP - %s", value) } + AuditTopologyRecovery(topologyRecovery, value) })).ReparentShard(context.Background(), tablet.Keyspace, tablet.Shard, reparentFunctions) // TODO: fix recovery attempted From ca7c79a3a960c35a5c1ff24464e0b9a62ec09f04 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 15 Sep 2021 13:04:36 +0530 Subject: [PATCH 098/176] removed handle relay log failure from the interface Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/reparent_functions.go | 9 --------- go/vt/vtctl/reparentutil/emergency_reparenter.go | 6 +----- go/vt/vtctl/reparentutil/reparent_functions.go | 8 -------- 3 files changed, 1 insertion(+), 22 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 89904ba0d1d..0069ae35778 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -88,15 +88,6 @@ func (vtorcReparent *VtOrcReparentFunctions) GetWaitForRelayLogsTimeout() time.D return 1 * time.Second } -// HandleRelayLogFailure implements the ReparentFunctions interface -// TODO : Discuss correct way -func (vtorcReparent *VtOrcReparentFunctions) HandleRelayLogFailure(logger logutil.Logger, err error) error { - // We do not want to throw an error from vtorc, since there could be replicas which are - // so far lagging that they may take days to apply all their relay logs - logger.Infof("failed to apply all relay logs - %v", err) - return nil -} - // GetIgnoreReplicas implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) GetIgnoreReplicas() sets.String { // vtorc does not ignore any replicas diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index a3509c6154f..b06c6947531 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -163,11 +163,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // Wait for all candidates to apply relay logs if err := waitForAllRelayLogsToApply(ctx, erp.logger, erp.tmc, validCandidates, tabletMap, statusMap, reparentFunctions.GetWaitForRelayLogsTimeout()); err != nil { - // We handle the error from relay logs separately for each implementation of ReparentFunctions - err = reparentFunctions.HandleRelayLogFailure(erp.logger, err) - if err != nil { - return err - } + return err } // find the primary candidate that we want to promote diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 2ccef5d826c..ff2725bdbef 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -40,7 +40,6 @@ type ( LockAction() string GetWaitReplicasTimeout() time.Duration GetWaitForRelayLogsTimeout() time.Duration - HandleRelayLogFailure(logutil.Logger, error) error GetIgnoreReplicas() sets.String RestrictValidCandidates(map[string]mysql.Position, map[string]*topo.TabletInfo) (map[string]mysql.Position, error) FindPrimaryCandidate(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) @@ -87,13 +86,6 @@ func (vtctlReparent *VtctlReparentFunctions) GetWaitForRelayLogsTimeout() time.D return vtctlReparent.waitReplicasTimeout } -// HandleRelayLogFailure implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) HandleRelayLogFailure(logger logutil.Logger, err error) error { - // in case of failure in applying relay logs, vtctl should return the error - // and let the user decide weather they want to ignore the tablets that caused the error in question - return err -} - // GetIgnoreReplicas implements the ReparentFunctions interface func (vtctlReparent *VtctlReparentFunctions) GetIgnoreReplicas() sets.String { return vtctlReparent.ignoreReplicas From 9b2e13a3692fc308f21a47fc0620f6f458f7fd3c Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 15 Sep 2021 15:50:30 +0530 Subject: [PATCH 099/176] remove the ReparentFunctions interface Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 4 ---- .../reparentutil/emergency_reparenter.go | 6 +++--- .../vtctl/reparentutil/reparent_functions.go | 19 ------------------- go/vt/vtctl/reparentutil/util.go | 6 +++--- 4 files changed, 6 insertions(+), 29 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 0069ae35778..2b486c6e46c 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -32,8 +32,6 @@ import ( "vitess.io/vitess/go/vt/orchestrator/attributes" "vitess.io/vitess/go/vt/orchestrator/kv" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - "vitess.io/vitess/go/vt/vttablet/tmclient" "vitess.io/vitess/go/vt/logutil" @@ -48,8 +46,6 @@ import ( "vitess.io/vitess/go/vt/topo" ) -var _ reparentutil.ReparentFunctions = (*VtOrcReparentFunctions)(nil) - // VtOrcReparentFunctions is the VtOrc implementation for ReparentFunctions type VtOrcReparentFunctions struct { analysisEntry inst.ReplicationAnalysis diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index b06c6947531..38fb8d0a08b 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -78,7 +78,7 @@ func NewEmergencyReparenter(ts *topo.Server, tmc tmclient.TabletManagerClient, l // ReparentShard performs the EmergencyReparentShard operation on the given // keyspace and shard. -func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, shard string, reparentFunctions ReparentFunctions) (*events.Reparent, error) { +func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, shard string, reparentFunctions *VtctlReparentFunctions) (*events.Reparent, error) { // First step is to lock the shard for the given operation ctx, unlock, err := erp.ts.LockShard(ctx, keyspace, shard, reparentFunctions.LockAction()) if err != nil { @@ -107,7 +107,7 @@ func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, sha } // reparentShardLocked performs Emergency Reparent Shard operation assuming that the shard is already locked -func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *events.Reparent, keyspace, shard string, reparentFunctions ReparentFunctions) error { +func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *events.Reparent, keyspace, shard string, reparentFunctions *VtctlReparentFunctions) error { // log the starting of the operation and increment the counter erp.logger.Infof("will initiate emergency reparent shard in keyspace - %s, shard - %s", keyspace, shard) ersCounter.Add(1) @@ -229,7 +229,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } func (erp *EmergencyReparenter) undoPromotion(ctx context.Context, ts *topo.Server, ev *events.Reparent, keyspace, shard string, prevPrimary *topodatapb.Tablet, - lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions) (*topodatapb.Tablet, error) { + lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions *VtctlReparentFunctions) (*topodatapb.Tablet, error) { var primaryAlias *topodatapb.TabletAlias var err error if prevPrimary != nil { diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index ff2725bdbef..180d0153eac 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -35,21 +35,6 @@ import ( ) type ( - // ReparentFunctions is an interface which has all the functions implementation required for re-parenting - ReparentFunctions interface { - LockAction() string - GetWaitReplicasTimeout() time.Duration - GetWaitForRelayLogsTimeout() time.Duration - GetIgnoreReplicas() sets.String - RestrictValidCandidates(map[string]mysql.Position, map[string]*topo.TabletInfo) (map[string]mysql.Position, error) - FindPrimaryCandidate(context.Context, logutil.Logger, tmclient.TabletManagerClient, map[string]mysql.Position, map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) - PromotedReplicaIsIdeal(*topodatapb.Tablet, *topodatapb.Tablet, map[string]*topo.TabletInfo, map[string]mysql.Position) bool - PostTabletChangeHook(*topodatapb.Tablet) - GetBetterCandidate(*topodatapb.Tablet, *topodatapb.Tablet, []*topodatapb.Tablet, map[string]*topo.TabletInfo) *topodatapb.Tablet - CheckIfNeedToOverridePromotion(newPrimary *topodatapb.Tablet) error - PostERSCompletionHook(context.Context, *events.Reparent, logutil.Logger, tmclient.TabletManagerClient) - } - // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions VtctlReparentFunctions struct { newPrimaryAlias *topodatapb.TabletAlias @@ -58,10 +43,6 @@ type ( } ) -var ( - _ ReparentFunctions = (*VtctlReparentFunctions)(nil) -) - // NewVtctlReparentFunctions creates a new VtctlReparentFunctions which is used in ERS ans PRS func NewVtctlReparentFunctions(newPrimaryAlias *topodatapb.TabletAlias, ignoreReplicas sets.String, waitReplicasTimeout time.Duration) *VtctlReparentFunctions { return &VtctlReparentFunctions{ diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 19b747b9b6f..54151c52f50 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -103,7 +103,7 @@ func waitForAllRelayLogsToApply(ctx context.Context, logger logutil.Logger, tmc // Also, it returns the replicas which started replicating only in the case where we wait for all the replicas func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, newPrimaryTablet *topodatapb.Tablet, lockAction string, tabletMap map[string]*topo.TabletInfo, - statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions, + statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions *VtctlReparentFunctions, waitForAllReplicas bool) ([]*topodatapb.Tablet, error) { var replicasStartedReplication []*topodatapb.Tablet @@ -302,7 +302,7 @@ func ChooseNewPrimary( // promotePrimaryCandidate promotes the primary candidate that we have, but it does not yet set to start accepting writes func promotePrimaryCandidate(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, - lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions, isIdeal bool) ([]*topodatapb.Tablet, error) { + lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions *VtctlReparentFunctions, isIdeal bool) ([]*topodatapb.Tablet, error) { // first step is change the type of the newPrimary tablet to PRIMARY if err := changeTypeToPrimary(ctx, tmc, newPrimary); err != nil { return nil, err @@ -381,7 +381,7 @@ func FindCurrentPrimary(tabletMap map[string]*topo.TabletInfo, logger logutil.Lo // replaceWithBetterCandidate promotes the newer candidate over the primary candidate that we have, but it does not set to start accepting writes func replaceWithBetterCandidate(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, prevPrimary, newPrimary *topodatapb.Tablet, - lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions ReparentFunctions) error { + lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions *VtctlReparentFunctions) error { // Find the primary position of the previous primary pos, err := tmc.PrimaryPosition(ctx, prevPrimary) if err != nil { From 60097390f48ef59c9cd05071690158203c26cfc4 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 15 Sep 2021 16:01:15 +0530 Subject: [PATCH 100/176] renamed reparent functions to EmergencyReparentOptions Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/topology_recovery.go | 2 +- go/vt/vtctl/grpcvtctldserver/server.go | 2 +- .../reparentutil/emergency_reparenter.go | 36 +++++++------- .../reparentutil/emergency_reparenter_test.go | 40 ++++++++-------- .../vtctl/reparentutil/reparent_functions.go | 48 ++++++++++--------- go/vt/vtctl/reparentutil/util.go | 20 ++++---- go/vt/wrangler/reparent.go | 2 +- 7 files changed, 76 insertions(+), 74 deletions(-) diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 18706eeab89..80fbbb9860c 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -656,7 +656,7 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat defer atomic.AddInt32(&shardsLockCounter, -1) // TODO: Fix durations - reparentFunctions := reparentutil.NewVtctlReparentFunctions(candidateTabletAlias, nil, 1*time.Second) + reparentFunctions := reparentutil.NewEmergencyReparentOptions(candidateTabletAlias, nil, 1*time.Second) _, err = reparentutil.NewEmergencyReparenter(ts, tmclient.NewTabletManagerClient(), logutil.NewCallbackLogger(func(event *logutilpb.Event) { level := event.GetLevel() value := event.GetValue() diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 3e0b7e4e8da..0bb1e6ed924 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -607,7 +607,7 @@ func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldat ev, err := reparentutil.NewEmergencyReparenter(s.ts, s.tmc, logger).ReparentShard(ctx, req.Keyspace, req.Shard, - reparentutil.NewVtctlReparentFunctions(req.NewPrimary, sets.NewString(ignoreReplicaAliases...), waitReplicasTimeout)) + reparentutil.NewEmergencyReparentOptions(req.NewPrimary, sets.NewString(ignoreReplicaAliases...), waitReplicasTimeout)) resp := &vtctldatapb.EmergencyReparentShardResponse{ Keyspace: req.Keyspace, diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 38fb8d0a08b..8c004efa287 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -78,9 +78,9 @@ func NewEmergencyReparenter(ts *topo.Server, tmc tmclient.TabletManagerClient, l // ReparentShard performs the EmergencyReparentShard operation on the given // keyspace and shard. -func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, shard string, reparentFunctions *VtctlReparentFunctions) (*events.Reparent, error) { +func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, shard string, opts EmergencyReparentOptions) (*events.Reparent, error) { // First step is to lock the shard for the given operation - ctx, unlock, err := erp.ts.LockShard(ctx, keyspace, shard, reparentFunctions.LockAction()) + ctx, unlock, err := erp.ts.LockShard(ctx, keyspace, shard, opts.LockAction()) if err != nil { return nil, err } @@ -99,15 +99,15 @@ func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, sha }() // run ERS with shard already locked - err = erp.reparentShardLocked(ctx, ev, keyspace, shard, reparentFunctions) + err = erp.reparentShardLocked(ctx, ev, keyspace, shard, opts) - reparentFunctions.PostERSCompletionHook(ctx, ev, erp.logger, erp.tmc) + opts.PostERSCompletionHook(ctx, ev, erp.logger, erp.tmc) return ev, err } // reparentShardLocked performs Emergency Reparent Shard operation assuming that the shard is already locked -func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *events.Reparent, keyspace, shard string, reparentFunctions *VtctlReparentFunctions) error { +func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *events.Reparent, keyspace, shard string, opts EmergencyReparentOptions) error { // log the starting of the operation and increment the counter erp.logger.Infof("will initiate emergency reparent shard in keyspace - %s, shard - %s", keyspace, shard) ersCounter.Add(1) @@ -137,7 +137,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // Stop replication on all the tablets and build their status map - statusMap, primaryStatusMap, err := StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, tabletMap, reparentFunctions.GetWaitReplicasTimeout(), reparentFunctions.GetIgnoreReplicas(), erp.logger) + statusMap, primaryStatusMap, err := StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, tabletMap, opts.GetWaitReplicasTimeout(), opts.GetIgnoreReplicas(), erp.logger) if err != nil { return vterrors.Wrapf(err, "failed to stop replication and build status maps: %v", err) } @@ -154,7 +154,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } // Now, we restrict the valid candidates list according to the ReparentFunctions implementations - validCandidates, err = reparentFunctions.RestrictValidCandidates(validCandidates, tabletMap) + validCandidates, err = opts.RestrictValidCandidates(validCandidates, tabletMap) if err != nil { return err } else if len(validCandidates) == 0 { @@ -162,22 +162,22 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // Wait for all candidates to apply relay logs - if err := waitForAllRelayLogsToApply(ctx, erp.logger, erp.tmc, validCandidates, tabletMap, statusMap, reparentFunctions.GetWaitForRelayLogsTimeout()); err != nil { + if err := waitForAllRelayLogsToApply(ctx, erp.logger, erp.tmc, validCandidates, tabletMap, statusMap, opts.GetWaitForRelayLogsTimeout()); err != nil { return err } // find the primary candidate that we want to promote var newPrimary *topodatapb.Tablet - newPrimary, tabletMap, err = reparentFunctions.FindPrimaryCandidate(ctx, erp.logger, erp.tmc, validCandidates, tabletMap) + newPrimary, tabletMap, err = opts.FindPrimaryCandidate(ctx, erp.logger, erp.tmc, validCandidates, tabletMap) if err != nil { return err } // check weather the primary candidate selected is ideal or if it can be improved later - isIdeal := reparentFunctions.PromotedReplicaIsIdeal(newPrimary, prevPrimary, tabletMap, validCandidates) + isIdeal := opts.PromotedReplicaIsIdeal(newPrimary, prevPrimary, tabletMap, validCandidates) // we now promote our primary candidate and also reparent all the other tablets to start replicating from this candidate - validReplacementCandidates, err := promotePrimaryCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, reparentFunctions.LockAction(), tabletMap, statusMap, reparentFunctions, isIdeal) + validReplacementCandidates, err := promotePrimaryCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, opts.LockAction(), tabletMap, statusMap, opts, isIdeal) if err != nil { return err } @@ -190,12 +190,12 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // try to find a better candidate if we do not already have the most ideal one betterCandidate := newPrimary if !isIdeal { - betterCandidate = reparentFunctions.GetBetterCandidate(newPrimary, prevPrimary, validReplacementCandidates, tabletMap) + betterCandidate = opts.GetBetterCandidate(newPrimary, prevPrimary, validReplacementCandidates, tabletMap) } // if our better candidate is different from our previous candidate, then we replace our primary if !topoproto.TabletAliasEqual(betterCandidate.Alias, newPrimary.Alias) { - err = replaceWithBetterCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, betterCandidate, reparentFunctions.LockAction(), tabletMap, statusMap, reparentFunctions) + err = replaceWithBetterCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, betterCandidate, opts.LockAction(), tabletMap, statusMap, opts) if err != nil { return err } @@ -203,11 +203,11 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // now we check if there is a need to override the promotion of the newPrimary - errInPromotion := reparentFunctions.CheckIfNeedToOverridePromotion(newPrimary) + errInPromotion := opts.CheckIfNeedToOverridePromotion(newPrimary) if errInPromotion != nil { erp.logger.Errorf("have to override promotion because of constraint failure - %v", errInPromotion) // we try and undo the promotion - newPrimary, err = erp.undoPromotion(ctx, erp.ts, ev, keyspace, shard, prevPrimary, reparentFunctions.LockAction(), tabletMap, statusMap, reparentFunctions) + newPrimary, err = erp.undoPromotion(ctx, erp.ts, ev, keyspace, shard, prevPrimary, opts.LockAction(), tabletMap, statusMap, opts) if err != nil { return err } @@ -222,19 +222,19 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return vterrors.Wrapf(err, "primary-elect tablet %v failed to be upgraded to primary: %v", newPrimary.Alias, err) } // call the post tablet change hook on the new primary - reparentFunctions.PostTabletChangeHook(newPrimary) + opts.PostTabletChangeHook(newPrimary) ev.NewPrimary = proto.Clone(newPrimary).(*topodatapb.Tablet) return errInPromotion } func (erp *EmergencyReparenter) undoPromotion(ctx context.Context, ts *topo.Server, ev *events.Reparent, keyspace, shard string, prevPrimary *topodatapb.Tablet, - lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions *VtctlReparentFunctions) (*topodatapb.Tablet, error) { + lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions) (*topodatapb.Tablet, error) { var primaryAlias *topodatapb.TabletAlias var err error if prevPrimary != nil { // promote the original primary back - _, err = promotePrimaryCandidate(ctx, erp.tmc, ts, ev, erp.logger, prevPrimary, lockAction, tabletMap, statusMap, reparentFunctions, true) + _, err = promotePrimaryCandidate(ctx, erp.tmc, ts, ev, erp.logger, prevPrimary, lockAction, tabletMap, statusMap, opts, true) if err == nil { return prevPrimary, nil } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index b97bdad7e8a..374f6c7d124 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -110,7 +110,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { tests := []struct { name string - vtctlReparentFunctions *VtctlReparentFunctions + vtctlReparentFunctions EmergencyReparentOptions tmc *testutil.TabletManagerClient // setup ts *topo.Server @@ -125,7 +125,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }{ { name: "success", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000102": nil, @@ -238,7 +238,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { // Here, all our tablets are tied, so we're going to explicitly pick // zone1-101. name: "success with requested primary-elect", - vtctlReparentFunctions: NewVtctlReparentFunctions(&topodatapb.TabletAlias{ + vtctlReparentFunctions: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ Cell: "zone1", Uid: 101, }, nil, 0), @@ -351,7 +351,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "success with existing primary", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.PrimaryStatus @@ -464,7 +464,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "shard not found", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), tmc: &testutil.TabletManagerClient{}, unlockTopo: true, // we shouldn't try to lock the nonexistent shard shards: nil, @@ -476,7 +476,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "cannot stop replication", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -535,7 +535,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "lost topo lock", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -594,7 +594,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "cannot get reparent candidates", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -665,7 +665,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "zero valid reparent candidates", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), tmc: &testutil.TabletManagerClient{}, shards: []*vtctldatapb.Shard{ { @@ -682,7 +682,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { { name: "error waiting for relay logs to apply", // one replica is going to take a minute to apply relay logs - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, time.Millisecond*50), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, time.Millisecond*50), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -771,7 +771,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "requested primary-elect is not in tablet map", - vtctlReparentFunctions: NewVtctlReparentFunctions(&topodatapb.TabletAlias{ + vtctlReparentFunctions: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ Cell: "zone1", Uid: 200, }, nil, 0), @@ -858,7 +858,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "requested primary-elect is not winning primary-elect", - vtctlReparentFunctions: NewVtctlReparentFunctions(&topodatapb.TabletAlias{ // we're requesting a tablet that's behind in replication + vtctlReparentFunctions: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ // we're requesting a tablet that's behind in replication Cell: "zone1", Uid: 102, }, nil, 0), @@ -946,7 +946,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "cannot promote new primary", - vtctlReparentFunctions: NewVtctlReparentFunctions( // We're explicitly requesting a primary-elect in this test case + vtctlReparentFunctions: NewEmergencyReparentOptions( // We're explicitly requesting a primary-elect in this test case // because we don't care about the correctness of the selection // code (it's covered by other test cases), and it simplifies // the error mocking. @@ -1115,7 +1115,7 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { tests := []struct { name string - vtctlReparentFunctions *VtctlReparentFunctions + vtctlReparentFunctions EmergencyReparentOptions tmc *testutil.TabletManagerClient unlockTopo bool newPrimaryTabletAlias string @@ -1130,7 +1130,7 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { }{ { name: "success", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, sets.NewString("zone1-0000000404"), 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, sets.NewString("zone1-0000000404"), 0), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1211,7 +1211,7 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { }, { name: "MasterPosition error", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string @@ -1253,7 +1253,7 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { }, { name: "cannot repopulate reparent journal on new primary", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": assert.AnError, @@ -1297,7 +1297,7 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { }, { name: "all replicas failing to SetMaster does fail the promotion", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1355,7 +1355,7 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { }, { name: "all replicas slow to SetMaster does fail the promotion", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, time.Millisecond*10), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, time.Millisecond*10), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1417,7 +1417,7 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { }, { name: "one replica failing to SetMaster does not fail the promotion", - vtctlReparentFunctions: NewVtctlReparentFunctions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 180d0153eac..7805de1bdcc 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -35,17 +35,19 @@ import ( ) type ( - // VtctlReparentFunctions is the Vtctl implementation for ReparentFunctions - VtctlReparentFunctions struct { + // EmergencyReparentOptions provides optional parameters to + // EmergencyReparentShard operations. Options are passed by value, so it is safe + // for callers to mutate and reuse options structs for multiple calls. + EmergencyReparentOptions struct { newPrimaryAlias *topodatapb.TabletAlias ignoreReplicas sets.String waitReplicasTimeout time.Duration } ) -// NewVtctlReparentFunctions creates a new VtctlReparentFunctions which is used in ERS ans PRS -func NewVtctlReparentFunctions(newPrimaryAlias *topodatapb.TabletAlias, ignoreReplicas sets.String, waitReplicasTimeout time.Duration) *VtctlReparentFunctions { - return &VtctlReparentFunctions{ +// NewEmergencyReparentOptions creates a new EmergencyReparentOptions which is used in ERS ans PRS +func NewEmergencyReparentOptions(newPrimaryAlias *topodatapb.TabletAlias, ignoreReplicas sets.String, waitReplicasTimeout time.Duration) EmergencyReparentOptions { + return EmergencyReparentOptions{ newPrimaryAlias: newPrimaryAlias, ignoreReplicas: ignoreReplicas, waitReplicasTimeout: waitReplicasTimeout, @@ -53,27 +55,27 @@ func NewVtctlReparentFunctions(newPrimaryAlias *topodatapb.TabletAlias, ignoreRe } // LockAction implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) LockAction() string { - return getLockAction(vtctlReparent.newPrimaryAlias) +func (opts *EmergencyReparentOptions) LockAction() string { + return getLockAction(opts.newPrimaryAlias) } // GetWaitReplicasTimeout implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) GetWaitReplicasTimeout() time.Duration { - return vtctlReparent.waitReplicasTimeout +func (opts *EmergencyReparentOptions) GetWaitReplicasTimeout() time.Duration { + return opts.waitReplicasTimeout } // GetWaitForRelayLogsTimeout implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) GetWaitForRelayLogsTimeout() time.Duration { - return vtctlReparent.waitReplicasTimeout +func (opts *EmergencyReparentOptions) GetWaitForRelayLogsTimeout() time.Duration { + return opts.waitReplicasTimeout } // GetIgnoreReplicas implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) GetIgnoreReplicas() sets.String { - return vtctlReparent.ignoreReplicas +func (opts *EmergencyReparentOptions) GetIgnoreReplicas() sets.String { + return opts.ignoreReplicas } // RestrictValidCandidates implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) RestrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { +func (opts *EmergencyReparentOptions) RestrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { restrictedValidCandidates := make(map[string]mysql.Position) for candidate, position := range validCandidates { candidateInfo, ok := tabletMap[candidate] @@ -90,7 +92,7 @@ func (vtctlReparent *VtctlReparentFunctions) RestrictValidCandidates(validCandid } // FindPrimaryCandidate implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidate(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) { +func (opts *EmergencyReparentOptions) FindPrimaryCandidate(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) { // Elect the candidate with the most up-to-date position. var winningPrimaryTabletAliasStr string var winningPosition mysql.Position @@ -104,8 +106,8 @@ func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidate(ctx context.Co // If we were requested to elect a particular primary, verify it's a valid // candidate (non-zero position, no errant GTIDs) and is at least as // advanced as the winning position. - if vtctlReparent.newPrimaryAlias != nil { - winningPrimaryTabletAliasStr = topoproto.TabletAliasString(vtctlReparent.newPrimaryAlias) + if opts.newPrimaryAlias != nil { + winningPrimaryTabletAliasStr = topoproto.TabletAliasString(opts.newPrimaryAlias) pos, ok := validCandidates[winningPrimaryTabletAliasStr] switch { case !ok: @@ -123,12 +125,12 @@ func (vtctlReparent *VtctlReparentFunctions) FindPrimaryCandidate(ctx context.Co } // PostTabletChangeHook implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) PostTabletChangeHook(*topodatapb.Tablet) { +func (opts *EmergencyReparentOptions) PostTabletChangeHook(*topodatapb.Tablet) { } // PromotedReplicaIsIdeal implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) PromotedReplicaIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, validCandidates map[string]mysql.Position) bool { - if vtctlReparent.newPrimaryAlias != nil { +func (opts *EmergencyReparentOptions) PromotedReplicaIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, validCandidates map[string]mysql.Position) bool { + if opts.newPrimaryAlias != nil { // explicit request to promote a specific tablet return true } @@ -147,7 +149,7 @@ func (vtctlReparent *VtctlReparentFunctions) PromotedReplicaIsIdeal(newPrimary, } // GetBetterCandidate implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) GetBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo) *topodatapb.Tablet { +func (opts *EmergencyReparentOptions) GetBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo) *topodatapb.Tablet { if prevPrimary != nil { // find one which is of the correct type and matches the cell of the previous primary for _, candidate := range validCandidates { @@ -165,10 +167,10 @@ func (vtctlReparent *VtctlReparentFunctions) GetBetterCandidate(newPrimary, prev } // CheckIfNeedToOverridePromotion implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) CheckIfNeedToOverridePromotion(newPrimary *topodatapb.Tablet) error { +func (opts *EmergencyReparentOptions) CheckIfNeedToOverridePromotion(newPrimary *topodatapb.Tablet) error { return nil } // PostERSCompletionHook implements the ReparentFunctions interface -func (vtctlReparent *VtctlReparentFunctions) PostERSCompletionHook(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) { +func (opts *EmergencyReparentOptions) PostERSCompletionHook(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) { } diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 54151c52f50..777b6aa49db 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -103,13 +103,13 @@ func waitForAllRelayLogsToApply(ctx context.Context, logger logutil.Logger, tmc // Also, it returns the replicas which started replicating only in the case where we wait for all the replicas func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, newPrimaryTablet *topodatapb.Tablet, lockAction string, tabletMap map[string]*topo.TabletInfo, - statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions *VtctlReparentFunctions, + statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions, waitForAllReplicas bool) ([]*topodatapb.Tablet, error) { var replicasStartedReplication []*topodatapb.Tablet var replicaMutex sync.Mutex - replCtx, replCancel := context.WithTimeout(ctx, reparentFunctions.GetWaitReplicasTimeout()) + replCtx, replCancel := context.WithTimeout(ctx, opts.GetWaitReplicasTimeout()) defer replCancel() event.DispatchUpdate(ev, "reparenting all tablets") @@ -169,7 +169,7 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent replicasStartedReplication = append(replicasStartedReplication, ti.Tablet) replicaMutex.Unlock() // We call PostTabletChangeHook every time there is an update to a tablet's replication or type - reparentFunctions.PostTabletChangeHook(ti.Tablet) + opts.PostTabletChangeHook(ti.Tablet) // Signal that at least one goroutine succeeded to SetReplicationSource. // We do this only when we do not want to wait for all the replicas @@ -184,7 +184,7 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent switch { case alias == topoproto.TabletAliasString(newPrimaryTablet.Alias): continue - case !reparentFunctions.GetIgnoreReplicas().Has(alias): + case !opts.GetIgnoreReplicas().Has(alias): replWg.Add(1) numReplicas++ go handleReplica(alias, ti) @@ -302,18 +302,18 @@ func ChooseNewPrimary( // promotePrimaryCandidate promotes the primary candidate that we have, but it does not yet set to start accepting writes func promotePrimaryCandidate(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, - lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions *VtctlReparentFunctions, isIdeal bool) ([]*topodatapb.Tablet, error) { + lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions, isIdeal bool) ([]*topodatapb.Tablet, error) { // first step is change the type of the newPrimary tablet to PRIMARY if err := changeTypeToPrimary(ctx, tmc, newPrimary); err != nil { return nil, err } // We call PostTabletChangeHook every time there is an update to a tablet's replication or type - reparentFunctions.PostTabletChangeHook(newPrimary) + opts.PostTabletChangeHook(newPrimary) // now we reparent all the other tablets to start replication from our new primary // if the promoted primary is not ideal then we wait for all the replicas so that we choose a better candidate from them later - replicasStartedReplication, err := reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, reparentFunctions, !isIdeal) + replicasStartedReplication, err := reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, opts, !isIdeal) if err != nil { return nil, err } @@ -381,7 +381,7 @@ func FindCurrentPrimary(tabletMap map[string]*topo.TabletInfo, logger logutil.Lo // replaceWithBetterCandidate promotes the newer candidate over the primary candidate that we have, but it does not set to start accepting writes func replaceWithBetterCandidate(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, prevPrimary, newPrimary *topodatapb.Tablet, - lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, reparentFunctions *VtctlReparentFunctions) error { + lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions) error { // Find the primary position of the previous primary pos, err := tmc.PrimaryPosition(ctx, prevPrimary) if err != nil { @@ -401,10 +401,10 @@ func replaceWithBetterCandidate(ctx context.Context, tmc tmclient.TabletManagerC } // We call PostTabletChangeHook every time there is an update to a tablet's replication or type - reparentFunctions.PostTabletChangeHook(newPrimary) + opts.PostTabletChangeHook(newPrimary) // we now reparent the other tablets, but this time we do not need to wait for all of them - _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, reparentFunctions, false) + _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, opts, false) if err != nil { return err } diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index ade96c09c14..d34b2e17936 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -150,7 +150,7 @@ func (wr *Wrangler) EmergencyReparentShard(ctx context.Context, keyspace, shard ctx, keyspace, shard, - reparentutil.NewVtctlReparentFunctions(primaryElectTabletAlias, ignoredTablets, waitReplicasTimeout)) + reparentutil.NewEmergencyReparentOptions(primaryElectTabletAlias, ignoredTablets, waitReplicasTimeout)) return err } From 570866bc856e74f77727748fab0f4f87ef51e21a Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 15 Sep 2021 16:12:33 +0530 Subject: [PATCH 101/176] moved lockAction and restrictValidCandidates from the emergencyReparentOptions struct to common util code Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter.go | 11 +++++---- .../vtctl/reparentutil/reparent_functions.go | 24 ++----------------- go/vt/vtctl/reparentutil/util.go | 17 +++++++++++++ 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 8c004efa287..f76b8d24ad7 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -80,7 +80,8 @@ func NewEmergencyReparenter(ts *topo.Server, tmc tmclient.TabletManagerClient, l // keyspace and shard. func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, shard string, opts EmergencyReparentOptions) (*events.Reparent, error) { // First step is to lock the shard for the given operation - ctx, unlock, err := erp.ts.LockShard(ctx, keyspace, shard, opts.LockAction()) + opts.lockAction = getLockAction(opts.newPrimaryAlias) + ctx, unlock, err := erp.ts.LockShard(ctx, keyspace, shard, opts.lockAction) if err != nil { return nil, err } @@ -154,7 +155,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } // Now, we restrict the valid candidates list according to the ReparentFunctions implementations - validCandidates, err = opts.RestrictValidCandidates(validCandidates, tabletMap) + validCandidates, err = restrictValidCandidates(validCandidates, tabletMap) if err != nil { return err } else if len(validCandidates) == 0 { @@ -177,7 +178,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve isIdeal := opts.PromotedReplicaIsIdeal(newPrimary, prevPrimary, tabletMap, validCandidates) // we now promote our primary candidate and also reparent all the other tablets to start replicating from this candidate - validReplacementCandidates, err := promotePrimaryCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, opts.LockAction(), tabletMap, statusMap, opts, isIdeal) + validReplacementCandidates, err := promotePrimaryCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, opts.lockAction, tabletMap, statusMap, opts, isIdeal) if err != nil { return err } @@ -195,7 +196,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // if our better candidate is different from our previous candidate, then we replace our primary if !topoproto.TabletAliasEqual(betterCandidate.Alias, newPrimary.Alias) { - err = replaceWithBetterCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, betterCandidate, opts.LockAction(), tabletMap, statusMap, opts) + err = replaceWithBetterCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, betterCandidate, opts.lockAction, tabletMap, statusMap, opts) if err != nil { return err } @@ -207,7 +208,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve if errInPromotion != nil { erp.logger.Errorf("have to override promotion because of constraint failure - %v", errInPromotion) // we try and undo the promotion - newPrimary, err = erp.undoPromotion(ctx, erp.ts, ev, keyspace, shard, prevPrimary, opts.LockAction(), tabletMap, statusMap, opts) + newPrimary, err = erp.undoPromotion(ctx, erp.ts, ev, keyspace, shard, prevPrimary, opts.lockAction, tabletMap, statusMap, opts) if err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 7805de1bdcc..f0731e152a7 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -42,6 +42,8 @@ type ( newPrimaryAlias *topodatapb.TabletAlias ignoreReplicas sets.String waitReplicasTimeout time.Duration + + lockAction string } ) @@ -54,11 +56,6 @@ func NewEmergencyReparentOptions(newPrimaryAlias *topodatapb.TabletAlias, ignore } } -// LockAction implements the ReparentFunctions interface -func (opts *EmergencyReparentOptions) LockAction() string { - return getLockAction(opts.newPrimaryAlias) -} - // GetWaitReplicasTimeout implements the ReparentFunctions interface func (opts *EmergencyReparentOptions) GetWaitReplicasTimeout() time.Duration { return opts.waitReplicasTimeout @@ -74,23 +71,6 @@ func (opts *EmergencyReparentOptions) GetIgnoreReplicas() sets.String { return opts.ignoreReplicas } -// RestrictValidCandidates implements the ReparentFunctions interface -func (opts *EmergencyReparentOptions) RestrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { - restrictedValidCandidates := make(map[string]mysql.Position) - for candidate, position := range validCandidates { - candidateInfo, ok := tabletMap[candidate] - if !ok { - return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) - } - // We only allow PRIMARY and REPLICA type of tablets to be considered for replication - if candidateInfo.Type != topodatapb.TabletType_PRIMARY && candidateInfo.Type != topodatapb.TabletType_REPLICA { - continue - } - restrictedValidCandidates[candidate] = position - } - return restrictedValidCandidates, nil -} - // FindPrimaryCandidate implements the ReparentFunctions interface func (opts *EmergencyReparentOptions) FindPrimaryCandidate(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) { // Elect the candidate with the most up-to-date position. diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 777b6aa49db..f738725131f 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -421,3 +421,20 @@ func getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { return action } + +// restrictValidCandidates implements the ReparentFunctions interface +func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { + restrictedValidCandidates := make(map[string]mysql.Position) + for candidate, position := range validCandidates { + candidateInfo, ok := tabletMap[candidate] + if !ok { + return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) + } + // We only allow PRIMARY and REPLICA type of tablets to be considered for replication + if candidateInfo.Type != topodatapb.TabletType_PRIMARY && candidateInfo.Type != topodatapb.TabletType_REPLICA { + continue + } + restrictedValidCandidates[candidate] = position + } + return restrictedValidCandidates, nil +} From 6ae51e749c95f55f18464fada30afd84728d3e68 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Sun, 19 Sep 2021 10:46:51 +0530 Subject: [PATCH 102/176] moved durability policy to reparentUtil Signed-off-by: Manan Gupta --- go/vt/orchestrator/app/cli.go | 4 +- go/vt/orchestrator/http/api.go | 4 +- go/vt/orchestrator/inst/analysis_dao.go | 6 +- .../inst/candidate_database_instance.go | 6 +- .../inst/candidate_database_instance_dao.go | 3 +- go/vt/orchestrator/inst/durability.go | 213 +---------------- go/vt/orchestrator/inst/instance.go | 4 +- go/vt/orchestrator/inst/instance_dao.go | 11 +- go/vt/orchestrator/inst/instance_topology.go | 4 +- .../inst/instance_topology_test.go | 30 +-- go/vt/orchestrator/logic/orchestrator.go | 4 +- .../orchestrator/logic/reparent_functions.go | 14 +- go/vt/orchestrator/logic/topology_recovery.go | 2 +- go/vt/vtctl/reparentutil/durability.go | 226 ++++++++++++++++++ .../reparentutil}/durability_test.go | 2 +- .../reparentutil}/promotion_rule.go | 2 +- 16 files changed, 293 insertions(+), 242 deletions(-) create mode 100644 go/vt/vtctl/reparentutil/durability.go rename go/vt/{orchestrator/inst => vtctl/reparentutil}/durability_test.go (99%) rename go/vt/{orchestrator/inst => vtctl/reparentutil}/promotion_rule.go (98%) diff --git a/go/vt/orchestrator/app/cli.go b/go/vt/orchestrator/app/cli.go index b3bbec86b13..f7e9f0d9b58 100644 --- a/go/vt/orchestrator/app/cli.go +++ b/go/vt/orchestrator/app/cli.go @@ -26,6 +26,8 @@ import ( "strings" "time" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/orchestrator/config" "vitess.io/vitess/go/vt/orchestrator/external/golib/log" "vitess.io/vitess/go/vt/orchestrator/external/golib/util" @@ -1273,7 +1275,7 @@ func Cli(command string, strict bool, instance string, destination string, owner case registerCliCommand("register-candidate", "Instance, meta", `Indicate that a specific instance is a preferred candidate for primary promotion`): { instanceKey, _ = inst.FigureInstanceKey(instanceKey, thisInstanceKey) - promotionRule, err := inst.ParseCandidatePromotionRule(*config.RuntimeCLIFlags.PromotionRule) + promotionRule, err := reparentutil.ParseCandidatePromotionRule(*config.RuntimeCLIFlags.PromotionRule) if err != nil { log.Fatale(err) } diff --git a/go/vt/orchestrator/http/api.go b/go/vt/orchestrator/http/api.go index 0d33f5c81d5..e2c6224aa94 100644 --- a/go/vt/orchestrator/http/api.go +++ b/go/vt/orchestrator/http/api.go @@ -25,6 +25,8 @@ import ( "strings" "time" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + "github.com/go-martini/martini" "github.com/martini-contrib/auth" "github.com/martini-contrib/render" @@ -2386,7 +2388,7 @@ func (this *HttpAPI) RegisterCandidate(params martini.Params, r render.Render, r Respond(r, &APIResponse{Code: ERROR, Message: err.Error()}) return } - promotionRule, err := inst.ParseCandidatePromotionRule(params["promotionRule"]) + promotionRule, err := reparentutil.ParseCandidatePromotionRule(params["promotionRule"]) if err != nil { Respond(r, &APIResponse{Code: ERROR, Message: err.Error()}) return diff --git a/go/vt/orchestrator/inst/analysis_dao.go b/go/vt/orchestrator/inst/analysis_dao.go index cc05aa219d1..cf9e7c95301 100644 --- a/go/vt/orchestrator/inst/analysis_dao.go +++ b/go/vt/orchestrator/inst/analysis_dao.go @@ -21,6 +21,8 @@ import ( "regexp" "time" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + "google.golang.org/protobuf/encoding/prototext" "vitess.io/vitess/go/vt/orchestrator/config" @@ -531,11 +533,11 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints) a.Analysis = ReplicationStopped a.Description = "Replication is stopped" // - } else if topo.IsReplicaType(a.TabletType) && !a.IsPrimary && ReplicaSemiSyncFromTablet(primaryTablet, tablet) && !a.SemiSyncReplicaEnabled { + } else if topo.IsReplicaType(a.TabletType) && !a.IsPrimary && reparentutil.ReplicaSemiSyncFromTablet(primaryTablet, tablet) && !a.SemiSyncReplicaEnabled { a.Analysis = ReplicaSemiSyncMustBeSet a.Description = "Replica semi-sync must be set" // - } else if topo.IsReplicaType(a.TabletType) && !a.IsPrimary && !ReplicaSemiSyncFromTablet(primaryTablet, tablet) && a.SemiSyncReplicaEnabled { + } else if topo.IsReplicaType(a.TabletType) && !a.IsPrimary && !reparentutil.ReplicaSemiSyncFromTablet(primaryTablet, tablet) && a.SemiSyncReplicaEnabled { a.Analysis = ReplicaSemiSyncMustNotBeSet a.Description = "Replica semi-sync must not be set" // diff --git a/go/vt/orchestrator/inst/candidate_database_instance.go b/go/vt/orchestrator/inst/candidate_database_instance.go index 493a1e7034b..54ea0c1f8ac 100644 --- a/go/vt/orchestrator/inst/candidate_database_instance.go +++ b/go/vt/orchestrator/inst/candidate_database_instance.go @@ -19,6 +19,8 @@ package inst import ( "fmt" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/orchestrator/db" ) @@ -26,12 +28,12 @@ import ( type CandidateDatabaseInstance struct { Hostname string Port int - PromotionRule CandidatePromotionRule + PromotionRule reparentutil.CandidatePromotionRule LastSuggestedString string PromotionRuleExpiry string // generated when retrieved from database for consistency reasons } -func NewCandidateDatabaseInstance(instanceKey *InstanceKey, promotionRule CandidatePromotionRule) *CandidateDatabaseInstance { +func NewCandidateDatabaseInstance(instanceKey *InstanceKey, promotionRule reparentutil.CandidatePromotionRule) *CandidateDatabaseInstance { return &CandidateDatabaseInstance{ Hostname: instanceKey.Hostname, Port: instanceKey.Port, diff --git a/go/vt/orchestrator/inst/candidate_database_instance_dao.go b/go/vt/orchestrator/inst/candidate_database_instance_dao.go index 30a2220eb95..2b228a0de39 100644 --- a/go/vt/orchestrator/inst/candidate_database_instance_dao.go +++ b/go/vt/orchestrator/inst/candidate_database_instance_dao.go @@ -19,6 +19,7 @@ package inst import ( "vitess.io/vitess/go/vt/orchestrator/external/golib/log" "vitess.io/vitess/go/vt/orchestrator/external/golib/sqlutils" + "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/orchestrator/config" "vitess.io/vitess/go/vt/orchestrator/db" @@ -93,7 +94,7 @@ func BulkReadCandidateDatabaseInstance() ([]CandidateDatabaseInstance, error) { cdi := CandidateDatabaseInstance{ Hostname: m.GetString("hostname"), Port: m.GetInt("port"), - PromotionRule: CandidatePromotionRule(m.GetString("promotion_rule")), + PromotionRule: reparentutil.CandidatePromotionRule(m.GetString("promotion_rule")), LastSuggestedString: m.GetString("last_suggested"), PromotionRuleExpiry: m.GetString("promotion_rule_expiry"), } diff --git a/go/vt/orchestrator/inst/durability.go b/go/vt/orchestrator/inst/durability.go index 6036094e793..4307d228904 100644 --- a/go/vt/orchestrator/inst/durability.go +++ b/go/vt/orchestrator/inst/durability.go @@ -17,80 +17,9 @@ limitations under the License. package inst import ( - "fmt" - - "vitess.io/vitess/go/vt/topo/topoproto" - - "vitess.io/vitess/go/vt/orchestrator/external/golib/log" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" -) - -//======================================================================= - -// A newDurabler is a function that creates a new durabler based on the -// properties specified in the input map. Every durabler must -// register a newDurabler function. -type newDurabler func(map[string]string) durabler - -var ( - // durabilityPolicies is a map that stores the functions needed to create a new durabler - durabilityPolicies = make(map[string]newDurabler) - // curDurabilityPolicy is the current durability policy in use - curDurabilityPolicy durabler + "vitess.io/vitess/go/vt/vtctl/reparentutil" ) -func init() { - // register all the durability rules with their functions to create them - registerDurability("none", func(map[string]string) durabler { - return &durabilityNone{} - }) - registerDurability("semi_sync", func(map[string]string) durabler { - return &durabilitySemiSync{} - }) - registerDurability("cross_cell", func(map[string]string) durabler { - return &durabilityCrossCell{} - }) - registerDurability("specified", newDurabilitySpecified) -} - -// durabler is the interface which is used to get the promotion rules for candidates and the semi sync setup -type durabler interface { - promotionRule(*topodatapb.Tablet) CandidatePromotionRule - primarySemiSync(InstanceKey) int - replicaSemiSync(primary, replica *topodatapb.Tablet) bool -} - -func registerDurability(name string, newDurablerFunc newDurabler) { - if durabilityPolicies[name] != nil { - log.Fatalf("durability policy %v already registered", name) - } - durabilityPolicies[name] = newDurablerFunc -} - -//======================================================================= - -// SetDurabilityPolicy is used to set the durability policy from the registered policies -func SetDurabilityPolicy(name string, durabilityParams map[string]string) error { - newDurabilityCreationFunc, found := durabilityPolicies[name] - if !found { - return fmt.Errorf("durability policy %v not found", name) - } - log.Infof("Durability setting: %v", name) - curDurabilityPolicy = newDurabilityCreationFunc(durabilityParams) - return nil -} - -// PromotionRule returns the promotion rule for the instance. -func PromotionRule(tablet *topodatapb.Tablet) CandidatePromotionRule { - return curDurabilityPolicy.promotionRule(tablet) -} - -// PrimarySemiSync returns the primary semi-sync setting for the instance. -// 0 means none. Non-zero specifies the number of required ackers. -func PrimarySemiSync(instanceKey InstanceKey) int { - return curDurabilityPolicy.primarySemiSync(instanceKey) -} - // ReplicaSemiSync returns the replica semi-sync setting for the instance. func ReplicaSemiSync(primaryKey, replicaKey InstanceKey) bool { primary, err := ReadTablet(primaryKey) @@ -101,139 +30,15 @@ func ReplicaSemiSync(primaryKey, replicaKey InstanceKey) bool { if err != nil { return false } - return curDurabilityPolicy.replicaSemiSync(primary, replica) + return reparentutil.ReplicaSemiSyncFromTablet(primary, replica) } -// ReplicaSemiSyncFromTablet returns the replica semi-sync setting from the tablet record. -// Prefer using this function if tablet record is available. -func ReplicaSemiSyncFromTablet(primary, replica *topodatapb.Tablet) bool { - return curDurabilityPolicy.replicaSemiSync(primary, replica) -} - -//======================================================================= - -// durabilityNone has no semi-sync and returns NeutralPromoteRule for Primary and Replica tablet types, MustNotPromoteRule for everything else -type durabilityNone struct{} - -func (d *durabilityNone) promotionRule(tablet *topodatapb.Tablet) CandidatePromotionRule { - switch tablet.Type { - case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: - return NeutralPromoteRule - } - return MustNotPromoteRule -} - -func (d *durabilityNone) primarySemiSync(instanceKey InstanceKey) int { - return 0 -} - -func (d *durabilityNone) replicaSemiSync(primary, replica *topodatapb.Tablet) bool { - return false -} - -//======================================================================= - -// durabilitySemiSync has 1 semi-sync setup. It only allows Primary and Replica type servers to acknowledge semi sync -// It returns NeutralPromoteRule for Primary and Replica tablet types, MustNotPromoteRule for everything else -type durabilitySemiSync struct{} - -func (d *durabilitySemiSync) promotionRule(tablet *topodatapb.Tablet) CandidatePromotionRule { - switch tablet.Type { - case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: - return NeutralPromoteRule - } - return MustNotPromoteRule -} - -func (d *durabilitySemiSync) primarySemiSync(instanceKey InstanceKey) int { - return 1 -} - -func (d *durabilitySemiSync) replicaSemiSync(primary, replica *topodatapb.Tablet) bool { - switch replica.Type { - case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: - return true - } - return false -} - -//======================================================================= - -// durabilityCrossCell has 1 semi-sync setup. It only allows Primary and Replica type servers from a different cell to acknowledge semi sync. -// This means that a transaction must be in two cells for it to be acknowledged -// It returns NeutralPromoteRule for Primary and Replica tablet types, MustNotPromoteRule for everything else -type durabilityCrossCell struct{} - -func (d *durabilityCrossCell) promotionRule(tablet *topodatapb.Tablet) CandidatePromotionRule { - switch tablet.Type { - case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: - return NeutralPromoteRule - } - return MustNotPromoteRule -} - -func (d *durabilityCrossCell) primarySemiSync(instanceKey InstanceKey) int { - return 1 -} - -func (d *durabilityCrossCell) replicaSemiSync(primary, replica *topodatapb.Tablet) bool { - // Prevent panics. - if primary.Alias == nil || replica.Alias == nil { - return false - } - switch replica.Type { - case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: - return primary.Alias.Cell != replica.Alias.Cell - } - return false -} - -//======================================================================= - -// durabilitySpecified is like durabilityNone. It has an additional map which it first queries with the tablet alias as the key -// If a CandidatePromotionRule is found in that map, then that is used as the promotion rule. Otherwise, it reverts to the same logic as durabilityNone -type durabilitySpecified struct { - promotionRules map[string]CandidatePromotionRule -} - -func (d *durabilitySpecified) promotionRule(tablet *topodatapb.Tablet) CandidatePromotionRule { - promoteRule, isFound := d.promotionRules[topoproto.TabletAliasString(tablet.Alias)] - if isFound { - return promoteRule - } - - switch tablet.Type { - case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: - return NeutralPromoteRule - } - return MustNotPromoteRule -} - -func (d *durabilitySpecified) primarySemiSync(instanceKey InstanceKey) int { - return 0 -} - -func (d *durabilitySpecified) replicaSemiSync(primary, replica *topodatapb.Tablet) bool { - return false -} - -// newDurabilitySpecified is a function that is used to create a new durabilitySpecified struct -func newDurabilitySpecified(m map[string]string) durabler { - promotionRules := map[string]CandidatePromotionRule{} - // range over the map given by the user - for tabletAliasStr, promotionRuleStr := range m { - // parse the promotion rule - promotionRule, err := ParseCandidatePromotionRule(promotionRuleStr) - // if parsing is not successful, skip over this rule - if err != nil { - log.Errorf("invalid promotion rule %s found, received error - %v", promotionRuleStr, err) - continue - } - // set the promotion rule in the map at the given tablet alias - promotionRules[tabletAliasStr] = promotionRule - } - - return &durabilitySpecified{ - promotionRules: promotionRules, +// PrimarySemiSync returns the primary semi-sync setting for the instance. +// 0 means none. Non-zero specifies the number of required ackers. +func PrimarySemiSync(instanceKey InstanceKey) int { + primary, err := ReadTablet(instanceKey) + if err != nil { + return 0 } + return reparentutil.PrimarySemiSyncFromTablet(primary) } diff --git a/go/vt/orchestrator/inst/instance.go b/go/vt/orchestrator/inst/instance.go index 1f6e7dd453a..a1fc44b6267 100644 --- a/go/vt/orchestrator/inst/instance.go +++ b/go/vt/orchestrator/inst/instance.go @@ -24,6 +24,8 @@ import ( "strings" "time" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/orchestrator/config" "vitess.io/vitess/go/vt/orchestrator/external/golib/math" ) @@ -110,7 +112,7 @@ type Instance struct { // be picked up from daabase_candidate_instance's value when // reading an instance from the db. IsCandidate bool - PromotionRule CandidatePromotionRule + PromotionRule reparentutil.CandidatePromotionRule IsDowntimed bool DowntimeReason string DowntimeOwner string diff --git a/go/vt/orchestrator/inst/instance_dao.go b/go/vt/orchestrator/inst/instance_dao.go index 8f0de503743..c0d1d0000fc 100644 --- a/go/vt/orchestrator/inst/instance_dao.go +++ b/go/vt/orchestrator/inst/instance_dao.go @@ -41,6 +41,7 @@ import ( "vitess.io/vitess/go/vt/orchestrator/external/golib/sqlutils" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/orchestrator/attributes" "vitess.io/vitess/go/vt/orchestrator/collection" @@ -679,7 +680,7 @@ func ReadTopologyInstanceBufferable(instanceKey *InstanceKey, bufferWrites bool, // We need to update candidate_database_instance. // We register the rule even if it hasn't changed, // to bump the last_suggested time. - instance.PromotionRule = PromotionRule(tablet) + instance.PromotionRule = reparentutil.PromotionRule(tablet) err = RegisterCandidateInstance(NewCandidateDatabaseInstance(instanceKey, instance.PromotionRule).WithCurrentTime()) logReadTopologyInstanceError(instanceKey, "RegisterCandidateInstance", err) @@ -932,7 +933,7 @@ func BulkReadInstance() ([](*InstanceKey), error) { } func ReadInstancePromotionRule(instance *Instance) (err error) { - var promotionRule CandidatePromotionRule = NeutralPromoteRule + var promotionRule reparentutil.CandidatePromotionRule = reparentutil.NeutralPromoteRule query := ` select ifnull(nullif(promotion_rule, ''), 'neutral') as promotion_rule @@ -942,7 +943,7 @@ func ReadInstancePromotionRule(instance *Instance) (err error) { args := sqlutils.Args(instance.Key.Hostname, instance.Key.Port) err = db.QueryOrchestrator(query, args, func(m sqlutils.RowMap) error { - promotionRule = CandidatePromotionRule(m.GetString("promotion_rule")) + promotionRule = reparentutil.CandidatePromotionRule(m.GetString("promotion_rule")) return nil }) instance.PromotionRule = promotionRule @@ -1022,7 +1023,7 @@ func readInstanceRow(m sqlutils.RowMap) *Instance { instance.IsLastCheckValid = m.GetBool("is_last_check_valid") instance.SecondsSinceLastSeen = m.GetNullInt64("seconds_since_last_seen") instance.IsCandidate = m.GetBool("is_candidate") - instance.PromotionRule = CandidatePromotionRule(m.GetString("promotion_rule")) + instance.PromotionRule = reparentutil.CandidatePromotionRule(m.GetString("promotion_rule")) instance.IsDowntimed = m.GetBool("is_downtimed") instance.DowntimeReason = m.GetString("downtime_reason") instance.DowntimeOwner = m.GetString("downtime_owner") @@ -1426,7 +1427,7 @@ func ReadClusterNeutralPromotionRuleInstances(clusterName string) (neutralInstan return neutralInstances, err } for _, instance := range instances { - if instance.PromotionRule == NeutralPromoteRule { + if instance.PromotionRule == reparentutil.NeutralPromoteRule { neutralInstances = append(neutralInstances, instance) } } diff --git a/go/vt/orchestrator/inst/instance_topology.go b/go/vt/orchestrator/inst/instance_topology.go index 8f4534acd0d..f8db0e92693 100644 --- a/go/vt/orchestrator/inst/instance_topology.go +++ b/go/vt/orchestrator/inst/instance_topology.go @@ -25,6 +25,8 @@ import ( "sync" "time" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/orchestrator/config" "vitess.io/vitess/go/vt/orchestrator/external/golib/log" "vitess.io/vitess/go/vt/orchestrator/external/golib/math" @@ -1609,7 +1611,7 @@ func isValidAsCandidatePrimaryInBinlogServerTopology(replica *Instance) bool { } func IsBannedFromBeingCandidateReplica(replica *Instance) bool { - if replica.PromotionRule == MustNotPromoteRule { + if replica.PromotionRule == reparentutil.MustNotPromoteRule { log.Debugf("instance %+v is banned because of promotion rule", replica.Key) return true } diff --git a/go/vt/orchestrator/inst/instance_topology_test.go b/go/vt/orchestrator/inst/instance_topology_test.go index daafec6c9a9..f36b41fdf8b 100644 --- a/go/vt/orchestrator/inst/instance_topology_test.go +++ b/go/vt/orchestrator/inst/instance_topology_test.go @@ -3,6 +3,8 @@ package inst import ( "math/rand" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + "testing" "vitess.io/vitess/go/vt/orchestrator/config" @@ -251,7 +253,7 @@ func TestIsBannedFromBeingCandidateReplica(t *testing.T) { { instances, _ := generateTestInstances() for _, instance := range instances { - instance.PromotionRule = MustNotPromoteRule + instance.PromotionRule = reparentutil.MustNotPromoteRule } for _, instance := range instances { test.S(t).ExpectTrue(IsBannedFromBeingCandidateReplica(instance)) @@ -471,7 +473,7 @@ func TestChooseCandidateReplicaPriorityBinlogFormatRowOverrides(t *testing.T) { func TestChooseCandidateReplicaMustNotPromoteRule(t *testing.T) { instances, instancesMap := generateTestInstances() applyGeneralGoodToGoReplicationParams(instances) - instancesMap[i830Key.StringCode()].PromotionRule = MustNotPromoteRule + instancesMap[i830Key.StringCode()].PromotionRule = reparentutil.MustNotPromoteRule instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) @@ -485,8 +487,8 @@ func TestChooseCandidateReplicaMustNotPromoteRule(t *testing.T) { func TestChooseCandidateReplicaPreferNotPromoteRule(t *testing.T) { instances, instancesMap := generateTestInstances() applyGeneralGoodToGoReplicationParams(instances) - instancesMap[i830Key.StringCode()].PromotionRule = MustNotPromoteRule - instancesMap[i820Key.StringCode()].PromotionRule = PreferNotPromoteRule + instancesMap[i830Key.StringCode()].PromotionRule = reparentutil.MustNotPromoteRule + instancesMap[i820Key.StringCode()].PromotionRule = reparentutil.PreferNotPromoteRule instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) @@ -501,9 +503,9 @@ func TestChooseCandidateReplicaPreferNotPromoteRule2(t *testing.T) { instances, instancesMap := generateTestInstances() applyGeneralGoodToGoReplicationParams(instances) for _, instance := range instances { - instance.PromotionRule = PreferNotPromoteRule + instance.PromotionRule = reparentutil.PreferNotPromoteRule } - instancesMap[i830Key.StringCode()].PromotionRule = MustNotPromoteRule + instancesMap[i830Key.StringCode()].PromotionRule = reparentutil.MustNotPromoteRule instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) @@ -519,9 +521,9 @@ func TestChooseCandidateReplicaPromoteRuleOrdering(t *testing.T) { applyGeneralGoodToGoReplicationParams(instances) for _, instance := range instances { instance.ExecBinlogCoordinates = instancesMap[i710Key.StringCode()].ExecBinlogCoordinates - instance.PromotionRule = NeutralPromoteRule + instance.PromotionRule = reparentutil.NeutralPromoteRule } - instancesMap[i830Key.StringCode()].PromotionRule = PreferPromoteRule + instancesMap[i830Key.StringCode()].PromotionRule = reparentutil.PreferPromoteRule instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) @@ -537,9 +539,9 @@ func TestChooseCandidateReplicaPromoteRuleOrdering2(t *testing.T) { applyGeneralGoodToGoReplicationParams(instances) for _, instance := range instances { instance.ExecBinlogCoordinates = instancesMap[i710Key.StringCode()].ExecBinlogCoordinates - instance.PromotionRule = PreferPromoteRule + instance.PromotionRule = reparentutil.PreferPromoteRule } - instancesMap[i820Key.StringCode()].PromotionRule = MustPromoteRule + instancesMap[i820Key.StringCode()].PromotionRule = reparentutil.MustPromoteRule instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) @@ -555,11 +557,11 @@ func TestChooseCandidateReplicaPromoteRuleOrdering3(t *testing.T) { applyGeneralGoodToGoReplicationParams(instances) for _, instance := range instances { instance.ExecBinlogCoordinates = instancesMap[i710Key.StringCode()].ExecBinlogCoordinates - instance.PromotionRule = NeutralPromoteRule + instance.PromotionRule = reparentutil.NeutralPromoteRule } - instancesMap[i730Key.StringCode()].PromotionRule = MustPromoteRule - instancesMap[i810Key.StringCode()].PromotionRule = PreferPromoteRule - instancesMap[i830Key.StringCode()].PromotionRule = PreferNotPromoteRule + instancesMap[i730Key.StringCode()].PromotionRule = reparentutil.MustPromoteRule + instancesMap[i810Key.StringCode()].PromotionRule = reparentutil.PreferPromoteRule + instancesMap[i830Key.StringCode()].PromotionRule = reparentutil.PreferNotPromoteRule instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) diff --git a/go/vt/orchestrator/logic/orchestrator.go b/go/vt/orchestrator/logic/orchestrator.go index 44223237b25..ff3f84fd850 100644 --- a/go/vt/orchestrator/logic/orchestrator.go +++ b/go/vt/orchestrator/logic/orchestrator.go @@ -24,6 +24,8 @@ import ( "syscall" "time" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + "github.com/patrickmn/go-cache" "github.com/rcrowley/go-metrics" "github.com/sjmudd/stopwatch" @@ -414,7 +416,7 @@ func ContinuousDiscovery() { go ometrics.InitMetrics() go acceptSignals() go kv.InitKVStores() - inst.SetDurabilityPolicy(config.Config.Durability, config.Config.DurabilityParams) + reparentutil.SetDurabilityPolicy(config.Config.Durability, config.Config.DurabilityParams) if *config.RuntimeCLIFlags.GrabElection { process.GrabElection() diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 2b486c6e46c..7aeb9e79c3d 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -21,6 +21,8 @@ import ( "fmt" "time" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/proto/vtrpc" @@ -249,8 +251,8 @@ func (vtorcReparent *VtOrcReparentFunctions) PromotedReplicaIsIdeal(newPrimary, } if newPrimaryInst.DataCenter == vtorcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstanceDataCenter && newPrimaryInst.PhysicalEnvironment == vtorcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstancePhysicalEnvironment { - if newPrimaryInst.PromotionRule == inst.MustPromoteRule || newPrimaryInst.PromotionRule == inst.PreferPromoteRule || - (vtorcReparent.hasBestPromotionRule && newPrimaryInst.PromotionRule != inst.MustNotPromoteRule) { + if newPrimaryInst.PromotionRule == reparentutil.MustPromoteRule || newPrimaryInst.PromotionRule == reparentutil.PreferPromoteRule || + (vtorcReparent.hasBestPromotionRule && newPrimaryInst.PromotionRule != reparentutil.MustNotPromoteRule) { AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: found %+v to be ideal candidate; will optimize recovery", newPrimaryInst.Key)) return true } @@ -290,11 +292,11 @@ func getReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, newPri var preferredCandidates []*topodatapb.Tablet var neutralReplicas []*topodatapb.Tablet for _, candidate := range validCandidates { - promotionRule := inst.PromotionRule(candidate) - if promotionRule == inst.MustPromoteRule || promotionRule == inst.PreferPromoteRule { + promotionRule := reparentutil.PromotionRule(candidate) + if promotionRule == reparentutil.MustPromoteRule || promotionRule == reparentutil.PreferPromoteRule { preferredCandidates = append(preferredCandidates, candidate) } - if promotionRule == inst.NeutralPromoteRule { + if promotionRule == reparentutil.NeutralPromoteRule { neutralReplicas = append(neutralReplicas, candidate) } } @@ -378,7 +380,7 @@ func getReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, newPri keepSearchingHint := "" if satisfied, reason := PrimaryFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(newPrimary)); !satisfied { keepSearchingHint = fmt.Sprintf("Will keep searching; %s", reason) - } else if inst.PromotionRule(newPrimary) == inst.PreferNotPromoteRule { + } else if reparentutil.PromotionRule(newPrimary) == reparentutil.PreferNotPromoteRule { keepSearchingHint = fmt.Sprintf("Will keep searching because we have promoted a server with prefer_not rule: %+v", newPrimary.Alias) } if keepSearchingHint != "" { diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 80fbbb9860c..01f1bbf9f33 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -497,7 +497,7 @@ func SuggestReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, de keepSearchingHint := "" if satisfied, reason := PrimaryFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, promotedReplica); !satisfied { keepSearchingHint = fmt.Sprintf("Will keep searching; %s", reason) - } else if promotedReplica.PromotionRule == inst.PreferNotPromoteRule { + } else if promotedReplica.PromotionRule == reparentutil.PreferNotPromoteRule { keepSearchingHint = fmt.Sprintf("Will keep searching because we have promoted a server with prefer_not rule: %+v", promotedReplica.Key) } if keepSearchingHint != "" { diff --git a/go/vt/vtctl/reparentutil/durability.go b/go/vt/vtctl/reparentutil/durability.go new file mode 100644 index 00000000000..16d8d78659a --- /dev/null +++ b/go/vt/vtctl/reparentutil/durability.go @@ -0,0 +1,226 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reparentutil + +import ( + "fmt" + + "vitess.io/vitess/go/vt/topo/topoproto" + + "vitess.io/vitess/go/vt/orchestrator/external/golib/log" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +//======================================================================= + +// A newDurabler is a function that creates a new durabler based on the +// properties specified in the input map. Every durabler must +// register a newDurabler function. +type newDurabler func(map[string]string) durabler + +var ( + // durabilityPolicies is a map that stores the functions needed to create a new durabler + durabilityPolicies = make(map[string]newDurabler) + // curDurabilityPolicy is the current durability policy in use + curDurabilityPolicy durabler +) + +func init() { + // register all the durability rules with their functions to create them + registerDurability("none", func(map[string]string) durabler { + return &durabilityNone{} + }) + registerDurability("semi_sync", func(map[string]string) durabler { + return &durabilitySemiSync{} + }) + registerDurability("cross_cell", func(map[string]string) durabler { + return &durabilityCrossCell{} + }) + registerDurability("specified", newDurabilitySpecified) +} + +// durabler is the interface which is used to get the promotion rules for candidates and the semi sync setup +type durabler interface { + promotionRule(*topodatapb.Tablet) CandidatePromotionRule + primarySemiSync(*topodatapb.Tablet) int + replicaSemiSync(primary, replica *topodatapb.Tablet) bool +} + +func registerDurability(name string, newDurablerFunc newDurabler) { + if durabilityPolicies[name] != nil { + log.Fatalf("durability policy %v already registered", name) + } + durabilityPolicies[name] = newDurablerFunc +} + +//======================================================================= + +// SetDurabilityPolicy is used to set the durability policy from the registered policies +func SetDurabilityPolicy(name string, durabilityParams map[string]string) error { + newDurabilityCreationFunc, found := durabilityPolicies[name] + if !found { + return fmt.Errorf("durability policy %v not found", name) + } + log.Infof("Durability setting: %v", name) + curDurabilityPolicy = newDurabilityCreationFunc(durabilityParams) + return nil +} + +// PromotionRule returns the promotion rule for the instance. +func PromotionRule(tablet *topodatapb.Tablet) CandidatePromotionRule { + return curDurabilityPolicy.promotionRule(tablet) +} + +// PrimarySemiSyncFromTablet returns the primary semi-sync setting for the instance. +// 0 means none. Non-zero specifies the number of required ackers. +func PrimarySemiSyncFromTablet(tablet *topodatapb.Tablet) int { + return curDurabilityPolicy.primarySemiSync(tablet) +} + +// ReplicaSemiSyncFromTablet returns the replica semi-sync setting from the tablet record. +// Prefer using this function if tablet record is available. +func ReplicaSemiSyncFromTablet(primary, replica *topodatapb.Tablet) bool { + return curDurabilityPolicy.replicaSemiSync(primary, replica) +} + +//======================================================================= + +// durabilityNone has no semi-sync and returns NeutralPromoteRule for Primary and Replica tablet types, MustNotPromoteRule for everything else +type durabilityNone struct{} + +func (d *durabilityNone) promotionRule(tablet *topodatapb.Tablet) CandidatePromotionRule { + switch tablet.Type { + case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: + return NeutralPromoteRule + } + return MustNotPromoteRule +} + +func (d *durabilityNone) primarySemiSync(tablet *topodatapb.Tablet) int { + return 0 +} + +func (d *durabilityNone) replicaSemiSync(primary, replica *topodatapb.Tablet) bool { + return false +} + +//======================================================================= + +// durabilitySemiSync has 1 semi-sync setup. It only allows Primary and Replica type servers to acknowledge semi sync +// It returns NeutralPromoteRule for Primary and Replica tablet types, MustNotPromoteRule for everything else +type durabilitySemiSync struct{} + +func (d *durabilitySemiSync) promotionRule(tablet *topodatapb.Tablet) CandidatePromotionRule { + switch tablet.Type { + case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: + return NeutralPromoteRule + } + return MustNotPromoteRule +} + +func (d *durabilitySemiSync) primarySemiSync(tablet *topodatapb.Tablet) int { + return 1 +} + +func (d *durabilitySemiSync) replicaSemiSync(primary, replica *topodatapb.Tablet) bool { + switch replica.Type { + case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: + return true + } + return false +} + +//======================================================================= + +// durabilityCrossCell has 1 semi-sync setup. It only allows Primary and Replica type servers from a different cell to acknowledge semi sync. +// This means that a transaction must be in two cells for it to be acknowledged +// It returns NeutralPromoteRule for Primary and Replica tablet types, MustNotPromoteRule for everything else +type durabilityCrossCell struct{} + +func (d *durabilityCrossCell) promotionRule(tablet *topodatapb.Tablet) CandidatePromotionRule { + switch tablet.Type { + case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: + return NeutralPromoteRule + } + return MustNotPromoteRule +} + +func (d *durabilityCrossCell) primarySemiSync(tablet *topodatapb.Tablet) int { + return 1 +} + +func (d *durabilityCrossCell) replicaSemiSync(primary, replica *topodatapb.Tablet) bool { + // Prevent panics. + if primary.Alias == nil || replica.Alias == nil { + return false + } + switch replica.Type { + case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: + return primary.Alias.Cell != replica.Alias.Cell + } + return false +} + +//======================================================================= + +// durabilitySpecified is like durabilityNone. It has an additional map which it first queries with the tablet alias as the key +// If a CandidatePromotionRule is found in that map, then that is used as the promotion rule. Otherwise, it reverts to the same logic as durabilityNone +type durabilitySpecified struct { + promotionRules map[string]CandidatePromotionRule +} + +func (d *durabilitySpecified) promotionRule(tablet *topodatapb.Tablet) CandidatePromotionRule { + promoteRule, isFound := d.promotionRules[topoproto.TabletAliasString(tablet.Alias)] + if isFound { + return promoteRule + } + + switch tablet.Type { + case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: + return NeutralPromoteRule + } + return MustNotPromoteRule +} + +func (d *durabilitySpecified) primarySemiSync(tablet *topodatapb.Tablet) int { + return 0 +} + +func (d *durabilitySpecified) replicaSemiSync(primary, replica *topodatapb.Tablet) bool { + return false +} + +// newDurabilitySpecified is a function that is used to create a new durabilitySpecified struct +func newDurabilitySpecified(m map[string]string) durabler { + promotionRules := map[string]CandidatePromotionRule{} + // range over the map given by the user + for tabletAliasStr, promotionRuleStr := range m { + // parse the promotion rule + promotionRule, err := ParseCandidatePromotionRule(promotionRuleStr) + // if parsing is not successful, skip over this rule + if err != nil { + log.Errorf("invalid promotion rule %s found, received error - %v", promotionRuleStr, err) + continue + } + // set the promotion rule in the map at the given tablet alias + promotionRules[tabletAliasStr] = promotionRule + } + + return &durabilitySpecified{ + promotionRules: promotionRules, + } +} diff --git a/go/vt/orchestrator/inst/durability_test.go b/go/vt/vtctl/reparentutil/durability_test.go similarity index 99% rename from go/vt/orchestrator/inst/durability_test.go rename to go/vt/vtctl/reparentutil/durability_test.go index 58083760999..7c7649c7056 100644 --- a/go/vt/orchestrator/inst/durability_test.go +++ b/go/vt/vtctl/reparentutil/durability_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package inst +package reparentutil import ( "testing" diff --git a/go/vt/orchestrator/inst/promotion_rule.go b/go/vt/vtctl/reparentutil/promotion_rule.go similarity index 98% rename from go/vt/orchestrator/inst/promotion_rule.go rename to go/vt/vtctl/reparentutil/promotion_rule.go index 8ea89ec4aa9..077eff7f518 100644 --- a/go/vt/orchestrator/inst/promotion_rule.go +++ b/go/vt/vtctl/reparentutil/promotion_rule.go @@ -14,7 +14,7 @@ limitations under the License. */ -package inst +package reparentutil import ( "fmt" From d31fd1b22197632437f291611847c46933f51247 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 20 Sep 2021 17:55:41 +0530 Subject: [PATCH 103/176] added sorter for ERS Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/ers_sorted_test.go | 144 ++++++++++++++++++++ go/vt/vtctl/reparentutil/ers_sorter.go | 105 ++++++++++++++ 2 files changed, 249 insertions(+) create mode 100644 go/vt/vtctl/reparentutil/ers_sorted_test.go create mode 100644 go/vt/vtctl/reparentutil/ers_sorter.go diff --git a/go/vt/vtctl/reparentutil/ers_sorted_test.go b/go/vt/vtctl/reparentutil/ers_sorted_test.go new file mode 100644 index 00000000000..538cb2d834f --- /dev/null +++ b/go/vt/vtctl/reparentutil/ers_sorted_test.go @@ -0,0 +1,144 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reparentutil + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +// TestErsSorter tests that the sorting for ERS works correctly +func TestErsSorter(t *testing.T) { + sid1 := mysql.SID{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + sid2 := mysql.SID{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16} + cell1 := "cell1" + cell2 := "cell2" + tabletReplica1_100 := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: cell1, + Uid: 100, + }, + Type: topodatapb.TabletType_REPLICA, + } + tabletReplica2_100 := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: cell2, + Uid: 100, + }, + Type: topodatapb.TabletType_REPLICA, + } + tabletReplica1_101 := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: cell1, + Uid: 101, + }, + Type: topodatapb.TabletType_REPLICA, + } + tabletRdonly1_102 := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: cell1, + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + } + + mysqlGTID1 := mysql.Mysql56GTID{ + Server: sid1, + Sequence: 10, + } + mysqlGTID2 := mysql.Mysql56GTID{ + Server: sid2, + Sequence: 10, + } + mysqlGTID3 := mysql.Mysql56GTID{ + Server: sid1, + Sequence: 11, + } + + positionMostAdvanced := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID1) + positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID2) + positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID3) + + positionEmpty := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + + positionIntermediate1 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionIntermediate1.GTIDSet = positionIntermediate1.GTIDSet.AddGTID(mysqlGTID1) + + positionIntermediate2 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionIntermediate2.GTIDSet = positionIntermediate2.GTIDSet.AddGTID(mysqlGTID1) + positionIntermediate2.GTIDSet = positionIntermediate2.GTIDSet.AddGTID(mysqlGTID2) + + testcases := []struct { + name string + tablets []*topodatapb.Tablet + positions []mysql.Position + idealCell string + containsErr string + sortedTablets []*topodatapb.Tablet + }{ + { + name: "all advanced, ideal cell 1", + tablets: []*topodatapb.Tablet{nil, tabletReplica2_100, tabletReplica1_100, tabletRdonly1_102}, + positions: []mysql.Position{positionMostAdvanced, positionMostAdvanced, positionMostAdvanced, positionMostAdvanced}, + idealCell: cell1, + sortedTablets: []*topodatapb.Tablet{tabletReplica1_100, tabletRdonly1_102, tabletReplica2_100, nil}, + }, { + name: "ordering by position", + tablets: []*topodatapb.Tablet{tabletReplica1_101, tabletReplica2_100, tabletReplica1_100, tabletRdonly1_102}, + positions: []mysql.Position{positionEmpty, positionIntermediate1, positionIntermediate2, positionMostAdvanced}, + idealCell: cell1, + sortedTablets: []*topodatapb.Tablet{tabletRdonly1_102, tabletReplica1_100, tabletReplica2_100, tabletReplica1_101}, + }, { + name: "tablets and positions count error", + tablets: []*topodatapb.Tablet{tabletReplica1_101, tabletReplica2_100}, + positions: []mysql.Position{positionEmpty, positionIntermediate1, positionMostAdvanced}, + idealCell: cell1, + containsErr: "unequal number of tablets and positions", + }, { + name: "promotion rule check", + tablets: []*topodatapb.Tablet{tabletReplica1_101, tabletRdonly1_102}, + positions: []mysql.Position{positionMostAdvanced, positionMostAdvanced}, + idealCell: cell1, + sortedTablets: []*topodatapb.Tablet{tabletReplica1_101, tabletRdonly1_102}, + }, { + name: "mixed", + tablets: []*topodatapb.Tablet{tabletReplica1_101, tabletReplica2_100, tabletReplica1_100, tabletRdonly1_102}, + positions: []mysql.Position{positionEmpty, positionIntermediate1, positionMostAdvanced, positionIntermediate1}, + idealCell: cell1, + sortedTablets: []*topodatapb.Tablet{tabletReplica1_100, tabletRdonly1_102, tabletReplica2_100, tabletReplica1_101}, + }, + } + + err := SetDurabilityPolicy("none", nil) + require.NoError(t, err) + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + err := sortTabletsForERS(testcase.tablets, testcase.positions, testcase.idealCell) + if testcase.containsErr != "" { + require.EqualError(t, err, testcase.containsErr) + } else { + require.NoError(t, err) + require.Equal(t, testcase.sortedTablets, testcase.tablets) + } + }) + } +} diff --git a/go/vt/vtctl/reparentutil/ers_sorter.go b/go/vt/vtctl/reparentutil/ers_sorter.go new file mode 100644 index 00000000000..fc287d0f6c0 --- /dev/null +++ b/go/vt/vtctl/reparentutil/ers_sorter.go @@ -0,0 +1,105 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reparentutil + +import ( + "sort" + + "vitess.io/vitess/go/mysql" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" +) + +// ErsSorter sorts tablets by GTID positions and Promotion rules aimed at finding the best +// candidate for intermediate promotion in emergency reparent shard +type ErsSorter struct { + tablets []*topodatapb.Tablet + positions []mysql.Position + idealCell string +} + +// NewErsSorter creates a new ErsSorter +func NewErsSorter(tablets []*topodatapb.Tablet, positions []mysql.Position, idealCell string) *ErsSorter { + return &ErsSorter{ + tablets: tablets, + positions: positions, + idealCell: idealCell, + } +} + +// Len implements the Interface for sorting +func (ersSorter *ErsSorter) Len() int { return len(ersSorter.tablets) } + +// Swap implements the Interface for sorting +func (ersSorter *ErsSorter) Swap(i, j int) { + ersSorter.tablets[i], ersSorter.tablets[j] = ersSorter.tablets[j], ersSorter.tablets[i] + ersSorter.positions[i], ersSorter.positions[j] = ersSorter.positions[j], ersSorter.positions[i] +} + +// Less implements the Interface for sorting +func (ersSorter *ErsSorter) Less(i, j int) bool { + // Returning "true" in this function means [i] is before [j], + // which will lead to [i] be a better candidate for promotion + + // Should not happen + // fail-safe code + if ersSorter.tablets[i] == nil { + return false + } + if ersSorter.tablets[j] == nil { + return true + } + + if !ersSorter.positions[i].AtLeast(ersSorter.positions[j]) { + // [i] does not have all GTIDs that [j] does + return false + } + if !ersSorter.positions[j].AtLeast(ersSorter.positions[i]) { + // [j] does not have all GTIDs that [i] does + return true + } + + // at this point, both have the same GTIDs + // So, we will now check which one is in the same cell + iInSameCell := ersSorter.tablets[i].Alias.Cell == ersSorter.idealCell + jInSameCell := ersSorter.tablets[j].Alias.Cell == ersSorter.idealCell + if iInSameCell && !jInSameCell { + return true + } + if jInSameCell && !iInSameCell { + return false + } + + // at this point, either both are in the ideal cell + // or neither is, so we check their promotion rules + jPromotionRule := PromotionRule(ersSorter.tablets[j]) + iPromotionRule := PromotionRule(ersSorter.tablets[i]) + return !jPromotionRule.BetterThan(iPromotionRule) +} + +// sortTabletsForERS sorts the tablets, given their positions for emergency reparent shard +func sortTabletsForERS(tablets []*topodatapb.Tablet, positions []mysql.Position, idealCell string) error { + // throw an error internal error in case of unequal number of tablets and positions + // fail-safe code prevents panic in sorting in case the lengths are unequal + if len(tablets) != len(positions) { + return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unequal number of tablets and positions") + } + + sort.Sort(NewErsSorter(tablets, positions, idealCell)) + return nil +} From d199764b3d812a0e58f24b6b16eb2a91947071b9 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 20 Sep 2021 18:21:55 +0530 Subject: [PATCH 104/176] use sorter in findPrimaryCandidate Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter.go | 2 +- go/vt/vtctl/reparentutil/util.go | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index f76b8d24ad7..8b432af11a6 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -169,7 +169,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // find the primary candidate that we want to promote var newPrimary *topodatapb.Tablet - newPrimary, tabletMap, err = opts.FindPrimaryCandidate(ctx, erp.logger, erp.tmc, validCandidates, tabletMap) + newPrimary, err = findPrimaryCandidate(erp.logger, prevPrimary, validCandidates, tabletMap, opts) if err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index f738725131f..380fcb2ec64 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -438,3 +438,51 @@ func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMa } return restrictedValidCandidates, nil } + +// findPrimaryCandidate implements the ReparentFunctions interface +func findPrimaryCandidate(logger logutil.Logger, prevPrimary *topodatapb.Tablet, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (*topodatapb.Tablet, error) { + var validTablets []*topodatapb.Tablet + var tabletPositions []mysql.Position + for tabletAlias, position := range validCandidates { + tablet, isFound := tabletMap[tabletAlias] + if !isFound { + return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", tabletAlias) + } + validTablets = append(validTablets, tablet.Tablet) + tabletPositions = append(tabletPositions, position) + } + + idealCell := "" + if prevPrimary != nil { + idealCell = prevPrimary.Alias.Cell + } + + // sort + err := sortTabletsForERS(validTablets, tabletPositions, idealCell) + if err != nil { + return nil, err + } + + winningPrimaryTablet := validTablets[0] + winningPosition := tabletPositions[0] + + // If we were requested to elect a particular primary, verify it's a valid + // candidate (non-zero position, no errant GTIDs) and is at least as + // advanced as the winning position. + if opts.newPrimaryAlias != nil { + requestedPrimaryAlias := topoproto.TabletAliasString(opts.newPrimaryAlias) + pos, ok := validCandidates[requestedPrimaryAlias] + if !ok { + return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "requested primary elect %v has errant GTIDs", requestedPrimaryAlias) + } + if pos.AtLeast(winningPosition) { + requestedPrimaryInfo, isFound := tabletMap[requestedPrimaryAlias] + if !isFound { + return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", requestedPrimaryAlias) + } + winningPrimaryTablet = requestedPrimaryInfo.Tablet + } + } + + return winningPrimaryTablet, nil +} From 1e02ff3ba110c9b254c32b1e2f085ac386836841 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 20 Sep 2021 20:03:31 +0530 Subject: [PATCH 105/176] moved vtorc functionality to promotedReplicaIsIdeal Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter.go | 5 +- .../vtctl/reparentutil/reparent_functions.go | 7 +- go/vt/vtctl/reparentutil/util.go | 97 +++++++++++++++++-- 3 files changed, 98 insertions(+), 11 deletions(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 8b432af11a6..50b8989f718 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -169,13 +169,14 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // find the primary candidate that we want to promote var newPrimary *topodatapb.Tablet - newPrimary, err = findPrimaryCandidate(erp.logger, prevPrimary, validCandidates, tabletMap, opts) + var validCandidateTablets []*topodatapb.Tablet + newPrimary, validCandidateTablets, err = findPrimaryCandidate(erp.logger, prevPrimary, validCandidates, tabletMap, opts) if err != nil { return err } // check weather the primary candidate selected is ideal or if it can be improved later - isIdeal := opts.PromotedReplicaIsIdeal(newPrimary, prevPrimary, tabletMap, validCandidates) + isIdeal := promotedReplicaIsIdeal(newPrimary, prevPrimary, validCandidateTablets, opts) // we now promote our primary candidate and also reparent all the other tablets to start replicating from this candidate validReplacementCandidates, err := promotePrimaryCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, opts.lockAction, tabletMap, statusMap, opts, isIdeal) diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index f0731e152a7..f7c66932f5d 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -39,9 +39,10 @@ type ( // EmergencyReparentShard operations. Options are passed by value, so it is safe // for callers to mutate and reuse options structs for multiple calls. EmergencyReparentOptions struct { - newPrimaryAlias *topodatapb.TabletAlias - ignoreReplicas sets.String - waitReplicasTimeout time.Duration + newPrimaryAlias *topodatapb.TabletAlias + ignoreReplicas sets.String + waitReplicasTimeout time.Duration + allowCrossCellPromotion bool lockAction string } diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 380fcb2ec64..36f9e95fad9 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -440,13 +440,13 @@ func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMa } // findPrimaryCandidate implements the ReparentFunctions interface -func findPrimaryCandidate(logger logutil.Logger, prevPrimary *topodatapb.Tablet, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (*topodatapb.Tablet, error) { +func findPrimaryCandidate(logger logutil.Logger, prevPrimary *topodatapb.Tablet, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (*topodatapb.Tablet, []*topodatapb.Tablet, error) { var validTablets []*topodatapb.Tablet var tabletPositions []mysql.Position for tabletAlias, position := range validCandidates { tablet, isFound := tabletMap[tabletAlias] if !isFound { - return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", tabletAlias) + return nil, nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", tabletAlias) } validTablets = append(validTablets, tablet.Tablet) tabletPositions = append(tabletPositions, position) @@ -460,7 +460,7 @@ func findPrimaryCandidate(logger logutil.Logger, prevPrimary *topodatapb.Tablet, // sort err := sortTabletsForERS(validTablets, tabletPositions, idealCell) if err != nil { - return nil, err + return nil, nil, err } winningPrimaryTablet := validTablets[0] @@ -473,16 +473,101 @@ func findPrimaryCandidate(logger logutil.Logger, prevPrimary *topodatapb.Tablet, requestedPrimaryAlias := topoproto.TabletAliasString(opts.newPrimaryAlias) pos, ok := validCandidates[requestedPrimaryAlias] if !ok { - return nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "requested primary elect %v has errant GTIDs", requestedPrimaryAlias) + return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "requested primary elect %v has errant GTIDs", requestedPrimaryAlias) } if pos.AtLeast(winningPosition) { requestedPrimaryInfo, isFound := tabletMap[requestedPrimaryAlias] if !isFound { - return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", requestedPrimaryAlias) + return nil, nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", requestedPrimaryAlias) } winningPrimaryTablet = requestedPrimaryInfo.Tablet } } - return winningPrimaryTablet, nil + return winningPrimaryTablet, validTablets, nil +} + +func promotedReplicaIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, opts EmergencyReparentOptions) bool { + if opts.newPrimaryAlias != nil { + // explicit request to promote a specific tablet + return topoproto.TabletAliasEqual(opts.newPrimaryAlias, newPrimary.Alias) + } + return getBetterCandidate(newPrimary, prevPrimary, validCandidates, opts) == newPrimary +} + +func getBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, opts EmergencyReparentOptions) *topodatapb.Tablet { + var preferredCandidates []*topodatapb.Tablet + var neutralReplicas []*topodatapb.Tablet + for _, candidate := range validCandidates { + promotionRule := PromotionRule(candidate) + if promotionRule == MustPromoteRule || promotionRule == PreferPromoteRule { + preferredCandidates = append(preferredCandidates, candidate) + } + if promotionRule == NeutralPromoteRule { + neutralReplicas = append(neutralReplicas, candidate) + } + } + + // So we've already promoted a replica. + // However, can we improve on our choice? Are there any replicas marked with "is_candidate"? + // Maybe we actually promoted such a replica. Does that mean we should keep it? + // Maybe we promoted a "neutral", and some "prefer" server is available. + // Maybe we promoted a "prefer_not" + // Maybe we promoted a server in a different DC than the primary + // There's many options. We may wish to replace the server we promoted with a better one. + candidate := findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, true, true) + if candidate != nil { + return candidate + } + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, false, true) + if candidate != nil { + return candidate + } + // do not have a preferred candidate in the same cell + + if opts.allowCrossCellPromotion { + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, true, false) + if candidate != nil { + return candidate + } + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, false, false) + if candidate != nil { + return candidate + } + } + + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, true, true) + if candidate != nil { + return candidate + } + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, false, true) + if candidate != nil { + return candidate + } + + if opts.allowCrossCellPromotion { + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, true, false) + if candidate != nil { + return candidate + } + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, false, false) + if candidate != nil { + return candidate + } + } + + return newPrimary +} + +func findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary *topodatapb.Tablet, possibleCandidates []*topodatapb.Tablet, checkEqualPrimary bool, checkSameCell bool) *topodatapb.Tablet { + for _, candidate := range possibleCandidates { + if checkEqualPrimary && !(topoproto.TabletAliasEqual(newPrimary.Alias, candidate.Alias)) { + continue + } + if checkSameCell && prevPrimary != nil && !(prevPrimary.Alias.Cell == candidate.Alias.Cell) { + continue + } + return candidate + } + return nil } From 79c3c887ae35b569a337d8dd027551581ed15edb Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 20 Sep 2021 20:24:58 +0530 Subject: [PATCH 106/176] fixed remaining ers to match newer implementation Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter.go | 52 ++++++----------- .../vtctl/reparentutil/reparent_functions.go | 8 +-- go/vt/vtctl/reparentutil/util.go | 58 ++++++++++--------- 3 files changed, 55 insertions(+), 63 deletions(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 50b8989f718..637e7c1503c 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -18,7 +18,6 @@ package reparentutil import ( "context" - "time" "vitess.io/vitess/go/stats" @@ -178,34 +177,33 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // check weather the primary candidate selected is ideal or if it can be improved later isIdeal := promotedReplicaIsIdeal(newPrimary, prevPrimary, validCandidateTablets, opts) - // we now promote our primary candidate and also reparent all the other tablets to start replicating from this candidate - validReplacementCandidates, err := promotePrimaryCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, opts.lockAction, tabletMap, statusMap, opts, isIdeal) - if err != nil { - return err - } - // Check (again) we still have the topology lock. if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } - // try to find a better candidate if we do not already have the most ideal one - betterCandidate := newPrimary if !isIdeal { - betterCandidate = opts.GetBetterCandidate(newPrimary, prevPrimary, validReplacementCandidates, tabletMap) - } - - // if our better candidate is different from our previous candidate, then we replace our primary - if !topoproto.TabletAliasEqual(betterCandidate.Alias, newPrimary.Alias) { - err = replaceWithBetterCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, betterCandidate, opts.lockAction, tabletMap, statusMap, opts) + // we now promote our primary candidate and also reparent all the other tablets to start replicating from this candidate + validReplacementCandidates, err := promoteIntermediatePrimary(ctx, erp.tmc, ev, erp.logger, newPrimary, opts.lockAction, tabletMap, statusMap, opts) if err != nil { return err } - newPrimary = betterCandidate + + // try to find a better candidate if we do not already have the most ideal one + betterCandidate := getBetterCandidate(newPrimary, prevPrimary, validReplacementCandidates, opts) + + // if our better candidate is different from our previous candidate, then we replace our primary + if !topoproto.TabletAliasEqual(betterCandidate.Alias, newPrimary.Alias) { + err = replaceWithBetterCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, betterCandidate, opts.lockAction, tabletMap, statusMap, opts) + if err != nil { + return err + } + newPrimary = betterCandidate + } } // now we check if there is a need to override the promotion of the newPrimary - errInPromotion := opts.CheckIfNeedToOverridePromotion(newPrimary) + errInPromotion := checkIfNeedToOverridePromotion(newPrimary, prevPrimary, opts) if errInPromotion != nil { erp.logger.Errorf("have to override promotion because of constraint failure - %v", errInPromotion) // we try and undo the promotion @@ -223,8 +221,10 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve if err != nil { return vterrors.Wrapf(err, "primary-elect tablet %v failed to be upgraded to primary: %v", newPrimary.Alias, err) } - // call the post tablet change hook on the new primary - opts.PostTabletChangeHook(newPrimary) + _, err = reparentReplicas(ctx, ev, erp.logger, erp.tmc, newPrimary, opts.lockAction, tabletMap, statusMap, opts, false, true) + if err != nil { + return err + } ev.NewPrimary = proto.Clone(newPrimary).(*topodatapb.Tablet) return errInPromotion @@ -232,7 +232,6 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve func (erp *EmergencyReparenter) undoPromotion(ctx context.Context, ts *topo.Server, ev *events.Reparent, keyspace, shard string, prevPrimary *topodatapb.Tablet, lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions) (*topodatapb.Tablet, error) { - var primaryAlias *topodatapb.TabletAlias var err error if prevPrimary != nil { // promote the original primary back @@ -241,20 +240,7 @@ func (erp *EmergencyReparenter) undoPromotion(ctx context.Context, ts *topo.Serv return prevPrimary, nil } erp.logger.Errorf("error in undoing promotion - %v", err) - primaryAlias = prevPrimary.Alias } - newTerm := time.Now() - _, errInUpdate := ts.UpdateShardFields(ctx, keyspace, shard, func(si *topo.ShardInfo) error { - if proto.Equal(si.PrimaryAlias, primaryAlias) { - return nil - } - si.PrimaryAlias = primaryAlias - si.PrimaryTermStartTime = logutil.TimeToProto(newTerm) - return nil - }) - if errInUpdate != nil { - return nil, errInUpdate - } return prevPrimary, err } diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index f7c66932f5d..2522048c617 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -39,10 +39,10 @@ type ( // EmergencyReparentShard operations. Options are passed by value, so it is safe // for callers to mutate and reuse options structs for multiple calls. EmergencyReparentOptions struct { - newPrimaryAlias *topodatapb.TabletAlias - ignoreReplicas sets.String - waitReplicasTimeout time.Duration - allowCrossCellPromotion bool + newPrimaryAlias *topodatapb.TabletAlias + ignoreReplicas sets.String + waitReplicasTimeout time.Duration + preventCrossCellPromotion bool lockAction string } diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 36f9e95fad9..5fc0aa767f0 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -99,12 +99,9 @@ func waitForAllRelayLogsToApply(ctx context.Context, logger logutil.Logger, tmc return nil } -// reparentReplicasAndPopulateJournal reparents all the replicas provided and populates the reparent journal on the primary. +// reparentReplicas reparents all the replicas provided and populates the reparent journal on the primary. // Also, it returns the replicas which started replicating only in the case where we wait for all the replicas -func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, - newPrimaryTablet *topodatapb.Tablet, lockAction string, tabletMap map[string]*topo.TabletInfo, - statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions, - waitForAllReplicas bool) ([]*topodatapb.Tablet, error) { +func reparentReplicas(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, newPrimaryTablet *topodatapb.Tablet, lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions, waitForAllReplicas bool, populateReparentJournal bool) ([]*topodatapb.Tablet, error) { var replicasStartedReplication []*topodatapb.Tablet var replicaMutex sync.Mutex @@ -136,8 +133,11 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent if err != nil { return err } - logger.Infof("populating reparent journal on new primary %v", alias) - return tmc.PopulateReparentJournal(replCtx, tablet, now, lockAction, newPrimaryTablet.Alias, position) + if populateReparentJournal { + logger.Infof("populating reparent journal on new primary %v", alias) + return tmc.PopulateReparentJournal(replCtx, tablet, now, lockAction, newPrimaryTablet.Alias, position) + } + return nil } handleReplica := func(alias string, ti *topo.TabletInfo) { @@ -192,7 +192,7 @@ func reparentReplicasAndPopulateJournal(ctx context.Context, ev *events.Reparent } // Spin up a background goroutine to wait until all replica goroutines - // finished. Polling this way allows us to have reparentReplicasAndPopulateJournal return + // finished. Polling this way allows us to have reparentReplicas return // success as soon as (a) the primary successfully populates its reparent // journal and (b) at least one replica successfully begins replicating. // @@ -313,7 +313,7 @@ func promotePrimaryCandidate(ctx context.Context, tmc tmclient.TabletManagerClie // now we reparent all the other tablets to start replication from our new primary // if the promoted primary is not ideal then we wait for all the replicas so that we choose a better candidate from them later - replicasStartedReplication, err := reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, opts, !isIdeal) + replicasStartedReplication, err := reparentReplicas(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, opts, !isIdeal, true) if err != nil { return nil, err } @@ -394,21 +394,6 @@ func replaceWithBetterCandidate(ctx context.Context, tmc tmclient.TabletManagerC if err != nil { return err } - - // Now change the type of the new primary tablet to PRIMARY - if err := changeTypeToPrimary(ctx, tmc, newPrimary); err != nil { - return err - } - - // We call PostTabletChangeHook every time there is an update to a tablet's replication or type - opts.PostTabletChangeHook(newPrimary) - - // we now reparent the other tablets, but this time we do not need to wait for all of them - _, err = reparentReplicasAndPopulateJournal(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, opts, false) - if err != nil { - return err - } - return nil } @@ -525,7 +510,7 @@ func getBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandida } // do not have a preferred candidate in the same cell - if opts.allowCrossCellPromotion { + if !opts.preventCrossCellPromotion { candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, true, false) if candidate != nil { return candidate @@ -545,7 +530,7 @@ func getBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandida return candidate } - if opts.allowCrossCellPromotion { + if !opts.preventCrossCellPromotion { candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, true, false) if candidate != nil { return candidate @@ -571,3 +556,24 @@ func findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary *topo } return nil } + +// promoteIntermediatePrimary promotes the primary candidate that we have, but it does not yet set to start accepting writes +func promoteIntermediatePrimary(ctx context.Context, tmc tmclient.TabletManagerClient, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, + lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions) ([]*topodatapb.Tablet, error) { + // now we reparent all the other tablets to start replication from our new primary + // if the promoted primary is not ideal then we wait for all the replicas so that we choose a better candidate from them later + replicasStartedReplication, err := reparentReplicas(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, opts, true, false) + if err != nil { + return nil, err + } + + replicasStartedReplication = append(replicasStartedReplication, newPrimary) + return replicasStartedReplication, nil +} + +func checkIfNeedToOverridePromotion(newPrimary, prevPrimary *topodatapb.Tablet, opts EmergencyReparentOptions) error { + if opts.preventCrossCellPromotion && prevPrimary != nil && newPrimary.Alias.Cell != prevPrimary.Alias.Cell { + return vterrors.Errorf(vtrpc.Code_ABORTED, "elected primary does not satisfy geographic constrains - %s", topoproto.TabletAliasString(newPrimary.Alias)) + } + return nil +} From d2e203316d2185c55a19a68d65e54d725e8756a7 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 20 Sep 2021 20:46:09 +0530 Subject: [PATCH 107/176] also override promotion for mustNotPromotionRules Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/util.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 5fc0aa767f0..73e9c0ee8f1 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -573,7 +573,10 @@ func promoteIntermediatePrimary(ctx context.Context, tmc tmclient.TabletManagerC func checkIfNeedToOverridePromotion(newPrimary, prevPrimary *topodatapb.Tablet, opts EmergencyReparentOptions) error { if opts.preventCrossCellPromotion && prevPrimary != nil && newPrimary.Alias.Cell != prevPrimary.Alias.Cell { - return vterrors.Errorf(vtrpc.Code_ABORTED, "elected primary does not satisfy geographic constrains - %s", topoproto.TabletAliasString(newPrimary.Alias)) + return vterrors.Errorf(vtrpc.Code_ABORTED, "elected primary does not satisfy geographic constraint - %s", topoproto.TabletAliasString(newPrimary.Alias)) + } + if PromotionRule(newPrimary) == MustNotPromoteRule { + return vterrors.Errorf(vtrpc.Code_ABORTED, "elected primary does not satisfy promotion rule constraint - %s", topoproto.TabletAliasString(newPrimary.Alias)) } return nil } From e9c047bd85767f406f742221c7ed61c2e2515f6f Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 22 Sep 2021 17:46:28 +0530 Subject: [PATCH 108/176] fixed bug in analysis occuring due to max connection limit to 1 Signed-off-by: Manan Gupta --- go/vt/orchestrator/inst/analysis_dao.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/vt/orchestrator/inst/analysis_dao.go b/go/vt/orchestrator/inst/analysis_dao.go index cf9e7c95301..318a0a51329 100644 --- a/go/vt/orchestrator/inst/analysis_dao.go +++ b/go/vt/orchestrator/inst/analysis_dao.go @@ -505,11 +505,11 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints) a.Analysis = PrimaryIsReadOnly a.Description = "Primary is read-only" // - } else if a.IsClusterPrimary && PrimarySemiSync(a.AnalyzedInstanceKey) != 0 && !a.SemiSyncPrimaryEnabled { + } else if a.IsClusterPrimary && reparentutil.PrimarySemiSyncFromTablet(tablet) != 0 && !a.SemiSyncPrimaryEnabled { a.Analysis = PrimarySemiSyncMustBeSet a.Description = "Primary semi-sync must be set" // - } else if a.IsClusterPrimary && PrimarySemiSync(a.AnalyzedInstanceKey) == 0 && a.SemiSyncPrimaryEnabled { + } else if a.IsClusterPrimary && reparentutil.PrimarySemiSyncFromTablet(tablet) == 0 && a.SemiSyncPrimaryEnabled { a.Analysis = PrimarySemiSyncMustNotBeSet a.Description = "Primary semi-sync must not be set" // From f3c54f2acba1856dc43d9a5e9f8c8b4adcfbdd4d Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 22 Sep 2021 18:14:39 +0530 Subject: [PATCH 109/176] added prevention for cross cell promotion as an argument Signed-off-by: Manan Gupta --- go/test/endtoend/cluster/cluster_process.go | 2 +- go/vt/orchestrator/logic/topology_recovery.go | 2 +- go/vt/vtctl/grpcvtctldserver/server.go | 2 +- .../reparentutil/emergency_reparenter_test.go | 46 +++++++++---------- .../vtctl/reparentutil/reparent_functions.go | 9 ++-- go/vt/wrangler/reparent.go | 2 +- 6 files changed, 30 insertions(+), 33 deletions(-) diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go index 4a7827d0152..a1fe74cc35b 100644 --- a/go/test/endtoend/cluster/cluster_process.go +++ b/go/test/endtoend/cluster/cluster_process.go @@ -42,7 +42,7 @@ const ( ) var ( - keepData = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders") + keepData = flag.Bool("keep-data", true, "don't delete the per-test VTDATAROOT subfolders") topoFlavor = flag.String("topo-flavor", "etcd2", "choose a topo server from etcd2, zk2 or consul") isCoverage = flag.Bool("is-coverage", false, "whether coverage is required") forceVTDATAROOT = flag.String("force-vtdataroot", "", "force path for VTDATAROOT, which may already be populated") diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 01f1bbf9f33..2861ef23098 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -656,7 +656,7 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat defer atomic.AddInt32(&shardsLockCounter, -1) // TODO: Fix durations - reparentFunctions := reparentutil.NewEmergencyReparentOptions(candidateTabletAlias, nil, 1*time.Second) + reparentFunctions := reparentutil.NewEmergencyReparentOptions(candidateTabletAlias, nil, 1*time.Second, config.Config.PreventCrossDataCenterPrimaryFailover) _, err = reparentutil.NewEmergencyReparenter(ts, tmclient.NewTabletManagerClient(), logutil.NewCallbackLogger(func(event *logutilpb.Event) { level := event.GetLevel() value := event.GetValue() diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 0bb1e6ed924..573493f1020 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -607,7 +607,7 @@ func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldat ev, err := reparentutil.NewEmergencyReparenter(s.ts, s.tmc, logger).ReparentShard(ctx, req.Keyspace, req.Shard, - reparentutil.NewEmergencyReparentOptions(req.NewPrimary, sets.NewString(ignoreReplicaAliases...), waitReplicasTimeout)) + reparentutil.NewEmergencyReparentOptions(req.NewPrimary, sets.NewString(ignoreReplicaAliases...), waitReplicasTimeout, false)) resp := &vtctldatapb.EmergencyReparentShardResponse{ Keyspace: req.Keyspace, diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 374f6c7d124..28c0ec154ad 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -125,7 +125,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }{ { name: "success", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000102": nil, @@ -241,7 +241,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { vtctlReparentFunctions: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ Cell: "zone1", Uid: 101, - }, nil, 0), + }, nil, 0, false), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000101": nil, @@ -351,7 +351,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "success with existing primary", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.PrimaryStatus @@ -464,7 +464,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "shard not found", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{}, unlockTopo: true, // we shouldn't try to lock the nonexistent shard shards: nil, @@ -476,7 +476,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "cannot stop replication", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -535,7 +535,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "lost topo lock", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -594,7 +594,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "cannot get reparent candidates", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -665,7 +665,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "zero valid reparent candidates", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{}, shards: []*vtctldatapb.Shard{ { @@ -682,7 +682,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { { name: "error waiting for relay logs to apply", // one replica is going to take a minute to apply relay logs - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, time.Millisecond*50), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, time.Millisecond*50, false), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -774,7 +774,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { vtctlReparentFunctions: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ Cell: "zone1", Uid: 200, - }, nil, 0), + }, nil, 0, false), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -861,7 +861,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { vtctlReparentFunctions: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ // we're requesting a tablet that's behind in replication Cell: "zone1", Uid: 102, - }, nil, 0), + }, nil, 0, false), keyspace: "testkeyspace", shard: "-", ts: memorytopo.NewServer("zone1"), @@ -946,14 +946,10 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "cannot promote new primary", - vtctlReparentFunctions: NewEmergencyReparentOptions( // We're explicitly requesting a primary-elect in this test case - // because we don't care about the correctness of the selection - // code (it's covered by other test cases), and it simplifies - // the error mocking. - &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, nil, 0, false), tmc: &testutil.TabletManagerClient{ PromoteReplicaResults: map[string]struct { Result string @@ -1130,7 +1126,7 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { }{ { name: "success", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, sets.NewString("zone1-0000000404"), 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, sets.NewString("zone1-0000000404"), 0, false), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1211,7 +1207,7 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { }, { name: "MasterPosition error", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string @@ -1253,7 +1249,7 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { }, { name: "cannot repopulate reparent journal on new primary", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": assert.AnError, @@ -1297,7 +1293,7 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { }, { name: "all replicas failing to SetMaster does fail the promotion", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1355,7 +1351,7 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { }, { name: "all replicas slow to SetMaster does fail the promotion", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, time.Millisecond*10), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, time.Millisecond*10, false), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1417,7 +1413,7 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { }, { name: "one replica failing to SetMaster does not fail the promotion", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0), + vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 2522048c617..4c675d63b13 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -49,11 +49,12 @@ type ( ) // NewEmergencyReparentOptions creates a new EmergencyReparentOptions which is used in ERS ans PRS -func NewEmergencyReparentOptions(newPrimaryAlias *topodatapb.TabletAlias, ignoreReplicas sets.String, waitReplicasTimeout time.Duration) EmergencyReparentOptions { +func NewEmergencyReparentOptions(newPrimaryAlias *topodatapb.TabletAlias, ignoreReplicas sets.String, waitReplicasTimeout time.Duration, preventCrossCellPromotion bool) EmergencyReparentOptions { return EmergencyReparentOptions{ - newPrimaryAlias: newPrimaryAlias, - ignoreReplicas: ignoreReplicas, - waitReplicasTimeout: waitReplicasTimeout, + newPrimaryAlias: newPrimaryAlias, + ignoreReplicas: ignoreReplicas, + waitReplicasTimeout: waitReplicasTimeout, + preventCrossCellPromotion: preventCrossCellPromotion, } } diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index d34b2e17936..e31331a9323 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -150,7 +150,7 @@ func (wr *Wrangler) EmergencyReparentShard(ctx context.Context, keyspace, shard ctx, keyspace, shard, - reparentutil.NewEmergencyReparentOptions(primaryElectTabletAlias, ignoredTablets, waitReplicasTimeout)) + reparentutil.NewEmergencyReparentOptions(primaryElectTabletAlias, ignoredTablets, waitReplicasTimeout, false)) return err } From c89a8e211d3b19b13b6ea597733dffc2192d7c7b Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 22 Sep 2021 18:23:48 +0530 Subject: [PATCH 110/176] skip tests that aren't supported yet Signed-off-by: Manan Gupta --- go/test/endtoend/cluster/cluster_process.go | 2 +- go/test/endtoend/vtorc/primary_failure_test.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go index a1fe74cc35b..4a7827d0152 100644 --- a/go/test/endtoend/cluster/cluster_process.go +++ b/go/test/endtoend/cluster/cluster_process.go @@ -42,7 +42,7 @@ const ( ) var ( - keepData = flag.Bool("keep-data", true, "don't delete the per-test VTDATAROOT subfolders") + keepData = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders") topoFlavor = flag.String("topo-flavor", "etcd2", "choose a topo server from etcd2, zk2 or consul") isCoverage = flag.Bool("is-coverage", false, "whether coverage is required") forceVTDATAROOT = flag.String("force-vtdataroot", "", "force path for VTDATAROOT, which may already be populated") diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index 49a3956acf7..3ba5ea18af0 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -154,6 +154,8 @@ func TestCrossDataCenterFailureError(t *testing.T) { // Failover will sometimes lead to a rdonly which can no longer replicate. // covers part of the test case master-failover-lost-replicas from orchestrator func TestLostRdonlyOnPrimaryFailure(t *testing.T) { + // new version of ERS does not check for lost replicas yet + t.Skip() defer cluster.PanicHandler(t) setupVttabletsAndVtorc(t, 2, 2, nil, "test_config.json") keyspace := &clusterInstance.Keyspaces[0] @@ -275,6 +277,8 @@ func TestPromotionLagSuccess(t *testing.T) { // This test checks that the promotion of a tablet succeeds if it passes the promotion lag test // covers the test case master-failover-fail-promotion-lag-minutes-failure from orchestrator func TestPromotionLagFailure(t *testing.T) { + // new version of ERS does not check for promotion lag yet + t.Skip() defer cluster.PanicHandler(t) setupVttabletsAndVtorc(t, 3, 1, nil, "test_config_promotion_failure.json") keyspace := &clusterInstance.Keyspaces[0] From afc178752e702520a2a5a58f9b39bc834e823ad7 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 22 Sep 2021 19:37:13 +0530 Subject: [PATCH 111/176] set durability policy from the vtctld and vtctl binaries Signed-off-by: Manan Gupta --- go/cmd/vtcombo/main.go | 5 ++++- go/cmd/vtctl/vtctl.go | 11 ++++++++++- go/cmd/vtctld/main.go | 6 +++++- go/test/endtoend/reparent/ers_test.go | 7 +++++++ go/test/endtoend/reparent/utils_test.go | 5 +++++ go/vt/vtctld/vtctld.go | 13 ++++++++++++- 6 files changed, 43 insertions(+), 4 deletions(-) diff --git a/go/cmd/vtcombo/main.go b/go/cmd/vtcombo/main.go index cfab3ff0172..52291078329 100644 --- a/go/cmd/vtcombo/main.go +++ b/go/cmd/vtcombo/main.go @@ -217,7 +217,10 @@ func main() { vtg := vtgate.Init(context.Background(), resilientServer, tpb.Cells[0], tabletTypesToWait) // vtctld configuration and init - vtctld.InitVtctld(ts) + err = vtctld.InitVtctld(ts) + if err != nil { + exit.Return(1) + } servenv.OnRun(func() { addStatusParts(vtg) diff --git a/go/cmd/vtctl/vtctl.go b/go/cmd/vtctl/vtctl.go index 844e8f8ec09..4e31b76dd79 100644 --- a/go/cmd/vtctl/vtctl.go +++ b/go/cmd/vtctl/vtctl.go @@ -26,6 +26,8 @@ import ( "syscall" "time" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/cmd" "context" @@ -45,6 +47,7 @@ import ( var ( waitTime = flag.Duration("wait-time", 24*time.Hour, "time to wait on an action") detachedMode = flag.Bool("detach", false, "detached mode - run vtcl detached from the terminal") + durability = flag.String("durability", "none", "type of durability to enforce. Default is none. Other values are dictated by registered plugins") ) func init() { @@ -90,6 +93,12 @@ func main() { log.Warningf("cannot connect to syslog: %v", err) } + err := reparentutil.SetDurabilityPolicy(*durability, nil) + if err != nil { + log.Errorf("error in setting durability policy: %v", err) + exit.Return(1) + } + closer := trace.StartTracing("vtctl") defer trace.LogErrorsWhenClosing(closer) @@ -104,7 +113,7 @@ func main() { wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) installSignalHandlers(cancel) - err := vtctl.RunCommand(ctx, wr, args) + err = vtctl.RunCommand(ctx, wr, args) cancel() switch err { case vtctl.ErrUnknownCommand: diff --git a/go/cmd/vtctld/main.go b/go/cmd/vtctld/main.go index 82c80a53dba..aa10badce33 100644 --- a/go/cmd/vtctld/main.go +++ b/go/cmd/vtctld/main.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "vitess.io/vitess/go/exit" "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/vtctld" @@ -40,7 +41,10 @@ func main() { defer ts.Close() // Init the vtctld core - vtctld.InitVtctld(ts) + err := vtctld.InitVtctld(ts) + if err != nil { + exit.Return(1) + } // Register http debug/health vtctld.RegisterDebugHealthHandler(ts) diff --git a/go/test/endtoend/reparent/ers_test.go b/go/test/endtoend/reparent/ers_test.go index dc5570f7bcc..f2f1481e7ee 100644 --- a/go/test/endtoend/reparent/ers_test.go +++ b/go/test/endtoend/reparent/ers_test.go @@ -42,6 +42,13 @@ func TestTrivialERS(t *testing.T) { require.NoError(t, err) time.Sleep(5 * time.Second) } + // We should do the same for vtctl binary + for i := 1; i <= 4; i++ { + out, err := ersWithVtctl() + log.Infof("ERS-vtctl loop %d. EmergencyReparentShard Output: %v", i, out) + require.NoError(t, err) + time.Sleep(5 * time.Second) + } } func TestReparentIgnoreReplicas(t *testing.T) { diff --git a/go/test/endtoend/reparent/utils_test.go b/go/test/endtoend/reparent/utils_test.go index a8039460061..5b34f7637fe 100644 --- a/go/test/endtoend/reparent/utils_test.go +++ b/go/test/endtoend/reparent/utils_test.go @@ -272,6 +272,11 @@ func ersIgnoreTablet(t *testing.T, tab *cluster.Vttablet, timeout string, tablet return clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput(args...) } +func ersWithVtctl() (string, error) { + args := []string{"EmergencyReparentShard", "-keyspace_shard", fmt.Sprintf("%s/%s", keyspaceName, shardName)} + return clusterInstance.VtctlProcess.ExecuteCommandWithOutput(args...) +} + func checkReparentFromOutside(t *testing.T, tablet *cluster.Vttablet, downPrimary bool, baseTime int64) { result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetShardReplication", cell1, keyspaceShard) require.Nil(t, err, "error should be Nil") diff --git a/go/vt/vtctld/vtctld.go b/go/vt/vtctld/vtctld.go index 36a7ae0a281..4c82faf37b4 100644 --- a/go/vt/vtctld/vtctld.go +++ b/go/vt/vtctld/vtctld.go @@ -24,6 +24,8 @@ import ( "strings" "time" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + rice "github.com/GeertJohan/go.rice" "context" @@ -39,6 +41,7 @@ import ( var ( enableRealtimeStats = flag.Bool("enable_realtime_stats", false, "Required for the Realtime Stats view. If set, vtctld will maintain a streaming RPC to each tablet (in all cells) to gather the realtime health stats.") + durability = flag.String("durability", "none", "type of durability to enforce. Default is none. Other values are dictated by registered plugins") _ = flag.String("web_dir", "", "NOT USED, here for backward compatibility") _ = flag.String("web_dir2", "", "NOT USED, here for backward compatibility") @@ -49,7 +52,13 @@ const ( ) // InitVtctld initializes all the vtctld functionality. -func InitVtctld(ts *topo.Server) { +func InitVtctld(ts *topo.Server) error { + err := reparentutil.SetDurabilityPolicy(*durability, nil) + if err != nil { + log.Errorf("error in setting durability policy: %v", err) + return err + } + actionRepo := NewActionRepository(ts) // keyspace actions @@ -185,4 +194,6 @@ func InitVtctld(ts *topo.Server) { // Setup reverse proxy for all vttablets through /vttablet/. initVTTabletRedirection(ts) + + return nil } From 64429693df683c3a2e117aace3f7790b0bb9f8c6 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 22 Sep 2021 20:47:14 +0530 Subject: [PATCH 112/176] remove unused code Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter.go | 4 +- .../vtctl/reparentutil/reparent_functions.go | 101 ------------------ go/vt/vtctl/reparentutil/util.go | 9 +- 3 files changed, 4 insertions(+), 110 deletions(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 637e7c1503c..eee31183ae6 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -137,7 +137,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // Stop replication on all the tablets and build their status map - statusMap, primaryStatusMap, err := StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, tabletMap, opts.GetWaitReplicasTimeout(), opts.GetIgnoreReplicas(), erp.logger) + statusMap, primaryStatusMap, err := StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, tabletMap, opts.waitReplicasTimeout, opts.ignoreReplicas, erp.logger) if err != nil { return vterrors.Wrapf(err, "failed to stop replication and build status maps: %v", err) } @@ -162,7 +162,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // Wait for all candidates to apply relay logs - if err := waitForAllRelayLogsToApply(ctx, erp.logger, erp.tmc, validCandidates, tabletMap, statusMap, opts.GetWaitForRelayLogsTimeout()); err != nil { + if err := waitForAllRelayLogsToApply(ctx, erp.logger, erp.tmc, validCandidates, tabletMap, statusMap, opts.waitReplicasTimeout); err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 4c675d63b13..09c335fd39b 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -20,16 +20,10 @@ import ( "context" "time" - "vitess.io/vitess/go/vt/proto/vtrpc" - "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/vterrors" - "k8s.io/apimachinery/pkg/util/sets" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/vt/logutil" topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topotools/events" "vitess.io/vitess/go/vt/vttablet/tmclient" ) @@ -58,101 +52,6 @@ func NewEmergencyReparentOptions(newPrimaryAlias *topodatapb.TabletAlias, ignore } } -// GetWaitReplicasTimeout implements the ReparentFunctions interface -func (opts *EmergencyReparentOptions) GetWaitReplicasTimeout() time.Duration { - return opts.waitReplicasTimeout -} - -// GetWaitForRelayLogsTimeout implements the ReparentFunctions interface -func (opts *EmergencyReparentOptions) GetWaitForRelayLogsTimeout() time.Duration { - return opts.waitReplicasTimeout -} - -// GetIgnoreReplicas implements the ReparentFunctions interface -func (opts *EmergencyReparentOptions) GetIgnoreReplicas() sets.String { - return opts.ignoreReplicas -} - -// FindPrimaryCandidate implements the ReparentFunctions interface -func (opts *EmergencyReparentOptions) FindPrimaryCandidate(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) { - // Elect the candidate with the most up-to-date position. - var winningPrimaryTabletAliasStr string - var winningPosition mysql.Position - for alias, position := range validCandidates { - if winningPosition.IsZero() || position.AtLeast(winningPosition) { - winningPosition = position - winningPrimaryTabletAliasStr = alias - } - } - - // If we were requested to elect a particular primary, verify it's a valid - // candidate (non-zero position, no errant GTIDs) and is at least as - // advanced as the winning position. - if opts.newPrimaryAlias != nil { - winningPrimaryTabletAliasStr = topoproto.TabletAliasString(opts.newPrimaryAlias) - pos, ok := validCandidates[winningPrimaryTabletAliasStr] - switch { - case !ok: - return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "primary elect %v has errant GTIDs", winningPrimaryTabletAliasStr) - case !pos.AtLeast(winningPosition): - return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "primary elect %v at position %v is not fully caught up. Winning position: %v", winningPrimaryTabletAliasStr, pos, winningPosition) - } - } - - newPrimaryAlias, isFound := tabletMap[winningPrimaryTabletAliasStr] - if !isFound { - return nil, nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", winningPrimaryTabletAliasStr) - } - return newPrimaryAlias.Tablet, tabletMap, nil -} - -// PostTabletChangeHook implements the ReparentFunctions interface -func (opts *EmergencyReparentOptions) PostTabletChangeHook(*topodatapb.Tablet) { -} - -// PromotedReplicaIsIdeal implements the ReparentFunctions interface -func (opts *EmergencyReparentOptions) PromotedReplicaIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, validCandidates map[string]mysql.Position) bool { - if opts.newPrimaryAlias != nil { - // explicit request to promote a specific tablet - return true - } - if prevPrimary != nil { - // check that the newPrimary has the same cell as the previous primary - if (newPrimary.Type == topodatapb.TabletType_PRIMARY || newPrimary.Type == topodatapb.TabletType_REPLICA) && newPrimary.Alias.Cell == prevPrimary.Alias.Cell { - return true - } - return false - } - - if newPrimary.Type == topodatapb.TabletType_PRIMARY || newPrimary.Type == topodatapb.TabletType_REPLICA { - return true - } - return false -} - -// GetBetterCandidate implements the ReparentFunctions interface -func (opts *EmergencyReparentOptions) GetBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo) *topodatapb.Tablet { - if prevPrimary != nil { - // find one which is of the correct type and matches the cell of the previous primary - for _, candidate := range validCandidates { - if (candidate.Type == topodatapb.TabletType_PRIMARY || candidate.Type == topodatapb.TabletType_REPLICA) && prevPrimary.Alias.Cell == candidate.Alias.Cell { - return candidate - } - } - } - for _, candidate := range validCandidates { - if candidate.Type == topodatapb.TabletType_PRIMARY || candidate.Type == topodatapb.TabletType_REPLICA { - return candidate - } - } - return newPrimary -} - -// CheckIfNeedToOverridePromotion implements the ReparentFunctions interface -func (opts *EmergencyReparentOptions) CheckIfNeedToOverridePromotion(newPrimary *topodatapb.Tablet) error { - return nil -} - // PostERSCompletionHook implements the ReparentFunctions interface func (opts *EmergencyReparentOptions) PostERSCompletionHook(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) { } diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 73e9c0ee8f1..7364ca2355c 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -106,7 +106,7 @@ func reparentReplicas(ctx context.Context, ev *events.Reparent, logger logutil.L var replicasStartedReplication []*topodatapb.Tablet var replicaMutex sync.Mutex - replCtx, replCancel := context.WithTimeout(ctx, opts.GetWaitReplicasTimeout()) + replCtx, replCancel := context.WithTimeout(ctx, opts.waitReplicasTimeout) defer replCancel() event.DispatchUpdate(ev, "reparenting all tablets") @@ -168,8 +168,6 @@ func reparentReplicas(ctx context.Context, ev *events.Reparent, logger logutil.L replicaMutex.Lock() replicasStartedReplication = append(replicasStartedReplication, ti.Tablet) replicaMutex.Unlock() - // We call PostTabletChangeHook every time there is an update to a tablet's replication or type - opts.PostTabletChangeHook(ti.Tablet) // Signal that at least one goroutine succeeded to SetReplicationSource. // We do this only when we do not want to wait for all the replicas @@ -184,7 +182,7 @@ func reparentReplicas(ctx context.Context, ev *events.Reparent, logger logutil.L switch { case alias == topoproto.TabletAliasString(newPrimaryTablet.Alias): continue - case !opts.GetIgnoreReplicas().Has(alias): + case !opts.ignoreReplicas.Has(alias): replWg.Add(1) numReplicas++ go handleReplica(alias, ti) @@ -308,9 +306,6 @@ func promotePrimaryCandidate(ctx context.Context, tmc tmclient.TabletManagerClie return nil, err } - // We call PostTabletChangeHook every time there is an update to a tablet's replication or type - opts.PostTabletChangeHook(newPrimary) - // now we reparent all the other tablets to start replication from our new primary // if the promoted primary is not ideal then we wait for all the replicas so that we choose a better candidate from them later replicasStartedReplication, err := reparentReplicas(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, opts, !isIdeal, true) From f1932dec0f9fcf5fca6e62fc3a32ee76dc7686f7 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 22 Sep 2021 21:23:02 +0530 Subject: [PATCH 113/176] remove duplicate function Signed-off-by: Manan Gupta --- go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go b/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go index 22c9f2e693d..703de1c1b51 100644 --- a/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go +++ b/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go @@ -491,14 +491,6 @@ func (fake *TabletManagerClient) SetMaster(ctx context.Context, tablet *topodata return assert.AnError } -// StartReplication is part of the tmclient.TabletManagerClient interface. -func (fake *TabletManagerClient) StartReplication(ctx context.Context, tablet *topodatapb.Tablet) error { - if tablet == nil { - return assert.AnError - } - return nil -} - // SetReplicationSource is part of the tmclient.TabletManagerClient interface. func (fake *TabletManagerClient) SetReplicationSource(ctx context.Context, tablet *topodatapb.Tablet, parent *topodatapb.TabletAlias, timeCreatedNS int64, waitPosition string, forceStartReplication bool) error { if fake.SetMasterResults == nil { From cba5ac955d78fdbbd6f8a77755abe7f614cee776 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 23 Sep 2021 11:09:57 +0530 Subject: [PATCH 114/176] revert changes to prs Signed-off-by: Manan Gupta --- .../vtctl/reparentutil/planned_reparenter.go | 4 --- .../reparentutil/planned_reparenter_test.go | 27 ------------------- .../testlib/planned_reparent_shard_test.go | 5 ---- 3 files changed, 36 deletions(-) diff --git a/go/vt/vtctl/reparentutil/planned_reparenter.go b/go/vt/vtctl/reparentutil/planned_reparenter.go index 8cf73634914..7c6f591900a 100644 --- a/go/vt/vtctl/reparentutil/planned_reparenter.go +++ b/go/vt/vtctl/reparentutil/planned_reparenter.go @@ -286,10 +286,6 @@ func (pr *PlannedReparenter) performGracefulPromotion( promoteCtx, promoteCancel := context.WithTimeout(ctx, opts.WaitReplicasTimeout) defer promoteCancel() - err = pr.tmc.ChangeType(promoteCtx, primaryElect, topodatapb.TabletType_PRIMARY) - if err != nil { - return "", vterrors.Wrapf(err, "primary-elect tablet %v failed to change type to primary; please try again", primaryElectAliasStr) - } rp, err := pr.tmc.PromoteReplica(promoteCtx, primaryElect) if err != nil { return "", vterrors.Wrapf(err, "primary-elect tablet %v failed to be promoted to primary; please try again", primaryElectAliasStr) diff --git a/go/vt/vtctl/reparentutil/planned_reparenter_test.go b/go/vt/vtctl/reparentutil/planned_reparenter_test.go index a9cc0f869a4..101346dce10 100644 --- a/go/vt/vtctl/reparentutil/planned_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/planned_reparenter_test.go @@ -721,9 +721,6 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { "position1": nil, }, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000200": nil, - }, }, ev: &events.Reparent{}, keyspace: "testkeyspace", @@ -1099,9 +1096,6 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { Error: nil, }, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000200": nil, - }, SetMasterResults: map[string]error{ "zone1-0000000200": nil, }, @@ -1297,9 +1291,6 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { Error: assert.AnError, }, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000200": nil, - }, SetMasterResults: map[string]error{ "zone1-0000000200": nil, }, @@ -1368,9 +1359,6 @@ func TestPlannedReparenter_performGracefulPromotion(t *testing.T) { Error: nil, }, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000200": nil, - }, SetMasterResults: map[string]error{ "zone1-0000000200": nil, }, @@ -1678,9 +1666,6 @@ func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { Error: nil, }, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000100": nil, - }, }, unlockTopo: false, keyspace: "testkeyspace", @@ -2015,9 +2000,6 @@ func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { Error: assert.AnError, }, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000100": nil, - }, }, unlockTopo: false, keyspace: "testkeyspace", @@ -2095,9 +2077,6 @@ func TestPlannedReparenter_performPotentialPromotion(t *testing.T) { Error: nil, }, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000100": nil, - }, }, timeout: time.Millisecond * 50, unlockTopo: false, @@ -2239,9 +2218,6 @@ func TestPlannedReparenter_reparentShardLocked(t *testing.T) { Error: nil, }, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000200": nil, - }, SetMasterResults: map[string]error{ "zone1-0000000100": nil, // zone1-100 gets reparented under zone1-200 }, @@ -2401,9 +2377,6 @@ func TestPlannedReparenter_reparentShardLocked(t *testing.T) { Error: nil, }, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000200": nil, - }, SetMasterResults: map[string]error{ "zone1-0000000100": nil, // called during reparentTablets to make oldPrimary a replica of newPrimary "zone1-0000000200": nil, // called during performGracefulPromotion to ensure newPrimary is caught up diff --git a/go/vt/wrangler/testlib/planned_reparent_shard_test.go b/go/vt/wrangler/testlib/planned_reparent_shard_test.go index 87e6bb2bd0c..da4035cdbfd 100644 --- a/go/vt/wrangler/testlib/planned_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/planned_reparent_shard_test.go @@ -753,9 +753,6 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { "STOP SLAVE", "FAKE SET MASTER", "START SLAVE", - "CREATE DATABASE IF NOT EXISTS _vt", - "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", - "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES", // extra commands because of retry "STOP SLAVE", "FAKE SET MASTER", @@ -765,8 +762,6 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES", } oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - "FAKE SET MASTER", - "START SLAVE", "FAKE SET MASTER", "START SLAVE", // extra commands because of retry From 58b6f378bd9facc087ef38bbb7769bac956e5577 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 23 Sep 2021 12:09:56 +0530 Subject: [PATCH 115/176] added flags for passing bool arguement preventing cross cell promotion Signed-off-by: Manan Gupta --- .../internal/command/reparents.go | 13 +- go/vt/proto/vtctldata/vtctldata.pb.go | 870 +++++++++--------- go/vt/proto/vtctldata/vtctldata_vtproto.pb.go | 33 + go/vt/vtctl/grpcvtctldserver/server.go | 3 +- go/vt/vtctl/reparent.go | 3 +- go/vt/wrangler/reparent.go | 4 +- .../testlib/emergency_reparent_shard_test.go | 2 +- proto/vtctldata.proto | 3 + 8 files changed, 493 insertions(+), 438 deletions(-) diff --git a/go/cmd/vtctldclient/internal/command/reparents.go b/go/cmd/vtctldclient/internal/command/reparents.go index 1469098fe66..a298c9afdf2 100644 --- a/go/cmd/vtctldclient/internal/command/reparents.go +++ b/go/cmd/vtctldclient/internal/command/reparents.go @@ -76,6 +76,7 @@ var emergencyReparentShardOptions = struct { WaitReplicasTimeout time.Duration NewPrimaryAliasStr string IgnoreReplicaAliasStrList []string + PreventCrossCellPromotion bool }{} func commandEmergencyReparentShard(cmd *cobra.Command, args []string) error { @@ -108,11 +109,12 @@ func commandEmergencyReparentShard(cmd *cobra.Command, args []string) error { cli.FinishedParsing(cmd) resp, err := client.EmergencyReparentShard(commandCtx, &vtctldatapb.EmergencyReparentShardRequest{ - Keyspace: keyspace, - Shard: shard, - NewPrimary: newPrimaryAlias, - IgnoreReplicas: ignoreReplicaAliases, - WaitReplicasTimeout: protoutil.DurationToProto(emergencyReparentShardOptions.WaitReplicasTimeout), + Keyspace: keyspace, + Shard: shard, + NewPrimary: newPrimaryAlias, + IgnoreReplicas: ignoreReplicaAliases, + WaitReplicasTimeout: protoutil.DurationToProto(emergencyReparentShardOptions.WaitReplicasTimeout), + PreventCrossCellPromotion: emergencyReparentShardOptions.PreventCrossCellPromotion, }) if err != nil { return err @@ -261,6 +263,7 @@ func commandTabletExternallyReparented(cmd *cobra.Command, args []string) error func init() { EmergencyReparentShard.Flags().DurationVar(&emergencyReparentShardOptions.WaitReplicasTimeout, "wait-replicas-timeout", *topo.RemoteOperationTimeout, "Time to wait for replicas to catch up in reparenting.") EmergencyReparentShard.Flags().StringVar(&emergencyReparentShardOptions.NewPrimaryAliasStr, "new-primary", "", "Alias of a tablet that should be the new primary. If not specified, the vtctld will select the best candidate to promote.") + EmergencyReparentShard.Flags().BoolVar(&emergencyReparentShardOptions.PreventCrossCellPromotion, "prevent-cross-cell-promotion", false, "Only promotes a new primary from the same cell as the previous primary") EmergencyReparentShard.Flags().StringSliceVarP(&emergencyReparentShardOptions.IgnoreReplicaAliasStrList, "ignore-replicas", "i", nil, "Comma-separated, repeated list of replica tablet aliases to ignore during the emergency reparent.") Root.AddCommand(EmergencyReparentShard) diff --git a/go/vt/proto/vtctldata/vtctldata.pb.go b/go/vt/proto/vtctldata/vtctldata.pb.go index ce6605202ef..5cf12df260e 100644 --- a/go/vt/proto/vtctldata/vtctldata.pb.go +++ b/go/vt/proto/vtctldata/vtctldata.pb.go @@ -2042,6 +2042,9 @@ type EmergencyReparentShardRequest struct { // WaitReplicasTimeout is the duration of time to wait for replicas to catch // up in reparenting. WaitReplicasTimeout *vttime.Duration `protobuf:"bytes,5,opt,name=wait_replicas_timeout,json=waitReplicasTimeout,proto3" json:"wait_replicas_timeout,omitempty"` + // PreventCrossCellPromotion is used to only promote the new primary from the same cell + // as the failed primary. + PreventCrossCellPromotion bool `protobuf:"varint,6,opt,name=prevent_cross_cell_promotion,json=preventCrossCellPromotion,proto3" json:"prevent_cross_cell_promotion,omitempty"` } func (x *EmergencyReparentShardRequest) Reset() { @@ -2111,6 +2114,13 @@ func (x *EmergencyReparentShardRequest) GetWaitReplicasTimeout() *vttime.Duratio return nil } +func (x *EmergencyReparentShardRequest) GetPreventCrossCellPromotion() bool { + if x != nil { + return x.PreventCrossCellPromotion + } + return false +} + type EmergencyReparentShardResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -6650,7 +6660,7 @@ var file_vtctldata_proto_rawDesc = []byte{ 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x22, 0x17, 0x0a, 0x15, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x8f, 0x02, 0x0a, 0x1d, 0x45, 0x6d, 0x65, 0x72, 0x67, 0x65, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xd0, 0x02, 0x0a, 0x1d, 0x45, 0x6d, 0x65, 0x72, 0x67, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x65, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, @@ -6667,452 +6677,456 @@ var file_vtctldata_proto_rawDesc = []byte{ 0x61, 0x73, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x74, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x77, 0x61, 0x69, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, - 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x22, 0xbc, 0x01, 0x0a, 0x1e, 0x45, 0x6d, 0x65, 0x72, - 0x67, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x65, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, - 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, - 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x40, 0x0a, 0x10, - 0x70, 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0f, 0x70, - 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x64, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x26, - 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, - 0x2e, 0x6c, 0x6f, 0x67, 0x75, 0x74, 0x69, 0x6c, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, - 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x3c, 0x0a, 0x1e, 0x46, 0x69, 0x6e, 0x64, 0x41, 0x6c, - 0x6c, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x49, 0x6e, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, - 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, - 0x70, 0x61, 0x63, 0x65, 0x22, 0xbe, 0x01, 0x0a, 0x1f, 0x46, 0x69, 0x6e, 0x64, 0x41, 0x6c, 0x6c, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x49, 0x6e, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, - 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x69, 0x6e, 0x64, 0x41, 0x6c, 0x6c, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x73, 0x49, 0x6e, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x06, 0x73, 0x68, 0x61, 0x72, 0x64, 0x73, 0x1a, 0x4b, 0x0a, 0x0b, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x9e, 0x01, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x42, 0x61, 0x63, - 0x6b, 0x75, 0x70, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, + 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x3f, 0x0a, 0x1c, 0x70, 0x72, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x5f, 0x63, 0x72, 0x6f, 0x73, 0x73, 0x5f, 0x63, 0x65, 0x6c, 0x6c, 0x5f, 0x70, 0x72, + 0x6f, 0x6d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x19, 0x70, + 0x72, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x43, 0x72, 0x6f, 0x73, 0x73, 0x43, 0x65, 0x6c, 0x6c, 0x50, + 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xbc, 0x01, 0x0a, 0x1e, 0x45, 0x6d, 0x65, + 0x72, 0x67, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x65, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x14, 0x0a, - 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6c, 0x69, - 0x6d, 0x69, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, - 0x25, 0x0a, 0x0e, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6c, 0x69, 0x6d, 0x69, - 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x65, - 0x64, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0x44, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x42, 0x61, 0x63, - 0x6b, 0x75, 0x70, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x07, - 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, - 0x6d, 0x79, 0x73, 0x71, 0x6c, 0x63, 0x74, 0x6c, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, - 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x73, 0x22, 0x28, 0x0a, 0x12, - 0x47, 0x65, 0x74, 0x43, 0x65, 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x22, 0x46, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x43, 0x65, 0x6c, - 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, - 0x09, 0x63, 0x65, 0x6c, 0x6c, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x65, 0x6c, 0x6c, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x63, 0x65, 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x19, - 0x0a, 0x17, 0x47, 0x65, 0x74, 0x43, 0x65, 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x4e, 0x61, 0x6d, - 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x30, 0x0a, 0x18, 0x47, 0x65, 0x74, - 0x43, 0x65, 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x22, 0x18, 0x0a, 0x16, 0x47, - 0x65, 0x74, 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xb6, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x43, 0x65, 0x6c, - 0x6c, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x49, 0x0a, 0x07, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x07, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x1a, 0x50, 0x0a, 0x0c, - 0x41, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, - 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x41, 0x6c, - 0x69, 0x61, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x15, - 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x49, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x73, - 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, - 0x09, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x13, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4b, 0x65, 0x79, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, 0x09, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, - 0x22, 0x30, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x40, 0x0a, + 0x10, 0x70, 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, + 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0f, + 0x70, 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x64, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, + 0x26, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x0e, 0x2e, 0x6c, 0x6f, 0x67, 0x75, 0x74, 0x69, 0x6c, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, + 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x3c, 0x0a, 0x1e, 0x46, 0x69, 0x6e, 0x64, 0x41, + 0x6c, 0x6c, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x49, 0x6e, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0xbe, 0x01, 0x0a, 0x1f, 0x46, 0x69, 0x6e, 0x64, 0x41, 0x6c, + 0x6c, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x49, 0x6e, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x06, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x76, 0x74, 0x63, 0x74, + 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x69, 0x6e, 0x64, 0x41, 0x6c, 0x6c, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x73, 0x49, 0x6e, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x06, 0x73, 0x68, 0x61, 0x72, 0x64, 0x73, 0x1a, 0x4b, 0x0a, 0x0b, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x74, 0x63, 0x74, + 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x9e, 0x01, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x42, 0x61, + 0x63, 0x6b, 0x75, 0x70, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, + 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x14, + 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6c, + 0x69, 0x6d, 0x69, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x65, 0x64, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x65, 0x64, + 0x12, 0x25, 0x0a, 0x0e, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6c, 0x69, 0x6d, + 0x69, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, + 0x65, 0x64, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0x44, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x42, 0x61, + 0x63, 0x6b, 0x75, 0x70, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, + 0x07, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, + 0x2e, 0x6d, 0x79, 0x73, 0x71, 0x6c, 0x63, 0x74, 0x6c, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x73, 0x22, 0x28, 0x0a, + 0x12, 0x47, 0x65, 0x74, 0x43, 0x65, 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x22, 0x46, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x43, 0x65, + 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, + 0x0a, 0x09, 0x63, 0x65, 0x6c, 0x6c, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x65, 0x6c, + 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x63, 0x65, 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x22, + 0x19, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x43, 0x65, 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x4e, 0x61, + 0x6d, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x30, 0x0a, 0x18, 0x47, 0x65, + 0x74, 0x43, 0x65, 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x22, 0x18, 0x0a, 0x16, + 0x47, 0x65, 0x74, 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xb6, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x43, 0x65, + 0x6c, 0x6c, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x49, 0x0a, 0x07, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x47, 0x65, 0x74, 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x1a, 0x50, 0x0a, + 0x0c, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x2a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, + 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x41, + 0x6c, 0x69, 0x61, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0x15, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x49, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x4b, 0x65, 0x79, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, + 0x0a, 0x09, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x13, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, 0x09, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x73, 0x22, 0x30, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x22, 0x46, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x08, 0x6b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x76, + 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x18, 0x0a, 0x16, 0x47, + 0x65, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x55, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x52, 0x6f, 0x75, 0x74, + 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x3a, 0x0a, 0x0d, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x72, 0x75, 0x6c, 0x65, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x52, 0x0c, + 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x22, 0x84, 0x02, 0x0a, + 0x10, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x38, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0b, + 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x74, + 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x62, + 0x6c, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x74, + 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x65, 0x78, 0x63, + 0x6c, 0x75, 0x64, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, + 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x76, 0x69, 0x65, 0x77, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x56, 0x69, 0x65, 0x77, 0x73, 0x12, + 0x28, 0x0a, 0x10, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x5f, 0x6f, + 0x6e, 0x6c, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x74, 0x61, 0x62, 0x6c, 0x65, + 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x61, 0x62, + 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x73, 0x4f, + 0x6e, 0x6c, 0x79, 0x22, 0x50, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, + 0x74, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x22, 0x4c, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x68, 0x61, 0x72, 0x64, 0x4e, + 0x61, 0x6d, 0x65, 0x22, 0x3a, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x22, + 0x32, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, + 0x05, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x63, 0x65, + 0x6c, 0x6c, 0x73, 0x22, 0xf3, 0x01, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x4b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, + 0x65, 0x74, 0x53, 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x1a, 0x69, 0x0a, 0x0a, + 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x45, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x76, 0x74, + 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x4b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x20, 0x0a, 0x08, 0x4e, 0x61, 0x6d, 0x65, 0x4c, + 0x69, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x22, 0x4a, 0x0a, 0x16, 0x47, 0x65, 0x74, + 0x53, 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, + 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x22, 0xcc, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, + 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x59, 0x0a, 0x0d, 0x73, 0x72, 0x76, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x72, 0x76, + 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, + 0x73, 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x1a, 0x56, 0x0a, 0x11, + 0x53, 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x2b, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x72, + 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0x2a, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x56, 0x53, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x63, 0x65, 0x6c, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, 0x6c, 0x6c, + 0x22, 0x4e, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x0c, 0x73, 0x72, 0x76, + 0x5f, 0x76, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x13, 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x52, 0x0a, 0x73, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x22, 0x2d, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x65, 0x6c, + 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x22, + 0xc5, 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x56, 0x0a, 0x0d, 0x73, 0x72, + 0x76, 0x5f, 0x76, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x32, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, + 0x74, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x73, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x73, 0x1a, 0x53, 0x0a, 0x10, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x2e, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4c, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x54, 0x61, + 0x62, 0x6c, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x0c, 0x74, + 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, + 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0b, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, + 0x41, 0x6c, 0x69, 0x61, 0x73, 0x22, 0x3d, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, + 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a, 0x06, 0x74, 0x61, + 0x62, 0x6c, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x74, 0x6f, 0x70, + 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, + 0x62, 0x6c, 0x65, 0x74, 0x22, 0xb1, 0x01, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, + 0x65, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x14, 0x0a, 0x05, + 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x63, 0x65, 0x6c, + 0x6c, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x69, 0x63, 0x74, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x06, 0x73, 0x74, 0x72, 0x69, 0x63, 0x74, 0x12, 0x3c, 0x0a, 0x0e, 0x74, 0x61, + 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, + 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0d, 0x74, 0x61, 0x62, 0x6c, 0x65, + 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x22, 0x40, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x54, + 0x61, 0x62, 0x6c, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, + 0x0a, 0x07, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x10, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, + 0x74, 0x52, 0x07, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x73, 0x22, 0x2f, 0x0a, 0x11, 0x47, 0x65, + 0x74, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x42, 0x0a, 0x12, 0x47, + 0x65, 0x74, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x2c, 0x0a, 0x08, 0x76, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x4b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, 0x07, 0x76, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x22, + 0x52, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x22, 0x46, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x08, 0x6b, 0x65, 0x79, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x76, 0x74, - 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x18, 0x0a, 0x16, 0x47, 0x65, - 0x74, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x22, 0x55, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x69, - 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x3a, 0x0a, 0x0d, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x52, 0x0c, 0x72, - 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x22, 0x84, 0x02, 0x0a, 0x10, - 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x38, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0b, 0x74, - 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, - 0x62, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, - 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x74, 0x61, - 0x62, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x65, 0x78, 0x63, 0x6c, - 0x75, 0x64, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x63, - 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x76, 0x69, 0x65, 0x77, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x56, 0x69, 0x65, 0x77, 0x73, 0x12, 0x28, - 0x0a, 0x10, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x5f, 0x6f, 0x6e, - 0x6c, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4e, - 0x61, 0x6d, 0x65, 0x73, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x61, 0x62, 0x6c, - 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x73, 0x4f, 0x6e, - 0x6c, 0x79, 0x22, 0x50, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x22, 0x4c, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x63, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6f, 0x6e, 0x6c, + 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4f, + 0x6e, 0x6c, 0x79, 0x22, 0x49, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x09, 0x77, + 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, + 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, + 0x6c, 0x6f, 0x77, 0x52, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x22, 0xfb, + 0x01, 0x0a, 0x17, 0x49, 0x6e, 0x69, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x50, 0x72, 0x69, 0x6d, + 0x61, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x52, 0x0a, 0x1a, + 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x5f, 0x74, 0x61, + 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, + 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x17, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, + 0x45, 0x6c, 0x65, 0x63, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, + 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x12, 0x44, 0x0a, 0x15, 0x77, 0x61, 0x69, 0x74, 0x5f, 0x72, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x74, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x44, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x77, 0x61, 0x69, 0x74, 0x52, 0x65, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x73, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x22, 0x42, 0x0a, 0x18, + 0x49, 0x6e, 0x69, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x6c, 0x6f, 0x67, 0x75, 0x74, + 0x69, 0x6c, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, + 0x22, 0x4d, 0x0a, 0x11, 0x50, 0x69, 0x6e, 0x67, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, + 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, + 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, + 0x61, 0x73, 0x52, 0x0b, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x22, + 0x14, 0x0a, 0x12, 0x50, 0x69, 0x6e, 0x67, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x89, 0x02, 0x0a, 0x1b, 0x50, 0x6c, 0x61, 0x6e, 0x6e, 0x65, + 0x64, 0x52, 0x65, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x36, 0x0a, 0x0b, 0x6e, 0x65, 0x77, 0x5f, 0x70, + 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, + 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, + 0x69, 0x61, 0x73, 0x52, 0x0a, 0x6e, 0x65, 0x77, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, + 0x3a, 0x0a, 0x0d, 0x61, 0x76, 0x6f, 0x69, 0x64, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0c, 0x61, + 0x76, 0x6f, 0x69, 0x64, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x44, 0x0a, 0x15, 0x77, + 0x61, 0x69, 0x74, 0x5f, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x74, 0x74, + 0x69, 0x6d, 0x65, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x77, 0x61, + 0x69, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, + 0x74, 0x22, 0xba, 0x01, 0x0a, 0x1c, 0x50, 0x6c, 0x61, 0x6e, 0x6e, 0x65, 0x64, 0x52, 0x65, 0x70, + 0x61, 0x72, 0x65, 0x6e, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x12, 0x40, 0x0a, 0x10, 0x70, 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x64, + 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, + 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, + 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0f, 0x70, 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x64, 0x50, + 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x26, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, + 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x6c, 0x6f, 0x67, 0x75, 0x74, 0x69, 0x6c, + 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x32, + 0x0a, 0x1a, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, + 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x63, 0x65, 0x6c, + 0x6c, 0x73, 0x22, 0x1d, 0x0a, 0x1b, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x56, 0x53, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x4f, 0x0a, 0x13, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, + 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, + 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, + 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0b, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, + 0x61, 0x73, 0x22, 0x16, 0x0a, 0x14, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x64, 0x0a, 0x1a, 0x52, 0x65, + 0x66, 0x72, 0x65, 0x73, 0x68, 0x53, 0x74, 0x61, 0x74, 0x65, 0x42, 0x79, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x65, + 0x6c, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x63, 0x65, 0x6c, 0x6c, 0x73, + 0x22, 0x4b, 0x0a, 0x1b, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x42, 0x79, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x2c, 0x0a, 0x12, 0x69, 0x73, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x72, 0x65, + 0x66, 0x72, 0x65, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x69, 0x73, 0x50, + 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x22, 0x7f, 0x0a, + 0x19, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x43, + 0x65, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, + 0x72, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, + 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x22, 0x1c, + 0x0a, 0x1a, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x43, 0x65, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x9b, 0x01, 0x0a, + 0x16, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x43, 0x65, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x68, 0x61, 0x72, 0x64, 0x4e, 0x61, - 0x6d, 0x65, 0x22, 0x3a, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x22, 0x32, - 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, - 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x63, 0x65, 0x6c, - 0x6c, 0x73, 0x22, 0xf3, 0x01, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x4b, 0x65, 0x79, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x47, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x31, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, - 0x74, 0x53, 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x1a, 0x69, 0x0a, 0x0a, 0x4e, - 0x61, 0x6d, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x45, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x76, 0x74, 0x63, - 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x4b, 0x65, 0x79, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x20, 0x0a, 0x08, 0x4e, 0x61, 0x6d, 0x65, 0x4c, 0x69, - 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x22, 0x4a, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x53, - 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, - 0x0a, 0x05, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x63, - 0x65, 0x6c, 0x6c, 0x73, 0x22, 0xcc, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x4b, - 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x59, 0x0a, 0x0d, 0x73, 0x72, 0x76, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x72, 0x76, 0x4b, - 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x73, - 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x1a, 0x56, 0x0a, 0x11, 0x53, - 0x72, 0x76, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x2b, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x72, 0x76, - 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0x2a, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x63, - 0x65, 0x6c, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x22, - 0x4e, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x0c, 0x73, 0x72, 0x76, 0x5f, - 0x76, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, - 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x52, 0x0a, 0x73, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x22, - 0x2d, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x65, 0x6c, 0x6c, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x22, 0xc5, - 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x56, 0x0a, 0x0d, 0x73, 0x72, 0x76, - 0x5f, 0x76, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x32, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, - 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x73, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x73, 0x1a, 0x53, 0x0a, 0x10, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x2e, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4c, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, - 0x6c, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x0c, 0x74, 0x61, - 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, - 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0b, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, - 0x6c, 0x69, 0x61, 0x73, 0x22, 0x3d, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, - 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a, 0x06, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x74, 0x6f, 0x70, 0x6f, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x74, 0x22, 0xb1, 0x01, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, - 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x63, - 0x65, 0x6c, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x63, 0x65, 0x6c, 0x6c, - 0x73, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x69, 0x63, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x06, 0x73, 0x74, 0x72, 0x69, 0x63, 0x74, 0x12, 0x3c, 0x0a, 0x0e, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, - 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0d, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, - 0x41, 0x6c, 0x69, 0x61, 0x73, 0x65, 0x73, 0x22, 0x40, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x54, 0x61, - 0x62, 0x6c, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, - 0x07, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, + 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, + 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x22, 0x19, 0x0a, 0x17, 0x52, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x43, 0x65, 0x6c, 0x6c, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x46, 0x0a, 0x15, 0x52, 0x65, 0x70, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, + 0x0a, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, - 0x52, 0x07, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x73, 0x22, 0x2f, 0x0a, 0x11, 0x47, 0x65, 0x74, - 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, - 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x42, 0x0a, 0x12, 0x47, 0x65, - 0x74, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x2c, 0x0a, 0x08, 0x76, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x4b, 0x65, 0x79, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, 0x07, 0x76, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x22, 0x52, - 0x0a, 0x13, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4f, 0x6e, - 0x6c, 0x79, 0x22, 0x49, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, - 0x77, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x09, 0x77, 0x6f, - 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, - 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, - 0x6f, 0x77, 0x52, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x22, 0xfb, 0x01, - 0x0a, 0x17, 0x49, 0x6e, 0x69, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x50, 0x72, 0x69, 0x6d, 0x61, - 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x52, 0x0a, 0x1a, 0x70, - 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x5f, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, - 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x17, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x45, - 0x6c, 0x65, 0x63, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, - 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, - 0x66, 0x6f, 0x72, 0x63, 0x65, 0x12, 0x44, 0x0a, 0x15, 0x77, 0x61, 0x69, 0x74, 0x5f, 0x72, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x74, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x44, 0x75, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x77, 0x61, 0x69, 0x74, 0x52, 0x65, 0x70, 0x6c, - 0x69, 0x63, 0x61, 0x73, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x22, 0x42, 0x0a, 0x18, 0x49, - 0x6e, 0x69, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x6c, 0x6f, 0x67, 0x75, 0x74, 0x69, - 0x6c, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, - 0x4d, 0x0a, 0x11, 0x50, 0x69, 0x6e, 0x67, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x61, - 0x6c, 0x69, 0x61, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, + 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x22, 0x7b, 0x0a, + 0x16, 0x52, 0x65, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x72, 0x69, + 0x6d, 0x61, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, - 0x73, 0x52, 0x0b, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x22, 0x14, - 0x0a, 0x12, 0x50, 0x69, 0x6e, 0x67, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x89, 0x02, 0x0a, 0x1b, 0x50, 0x6c, 0x61, 0x6e, 0x6e, 0x65, 0x64, - 0x52, 0x65, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x36, 0x0a, 0x0b, 0x6e, 0x65, 0x77, 0x5f, 0x70, 0x72, - 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, - 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, - 0x61, 0x73, 0x52, 0x0a, 0x6e, 0x65, 0x77, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x3a, - 0x0a, 0x0d, 0x61, 0x76, 0x6f, 0x69, 0x64, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0c, 0x61, 0x76, - 0x6f, 0x69, 0x64, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x44, 0x0a, 0x15, 0x77, 0x61, - 0x69, 0x74, 0x5f, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x5f, 0x74, 0x69, 0x6d, 0x65, - 0x6f, 0x75, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x74, 0x74, 0x69, - 0x6d, 0x65, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x77, 0x61, 0x69, - 0x74, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, - 0x22, 0xba, 0x01, 0x0a, 0x1c, 0x50, 0x6c, 0x61, 0x6e, 0x6e, 0x65, 0x64, 0x52, 0x65, 0x70, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, + 0x73, 0x52, 0x07, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x22, 0x6a, 0x0a, 0x12, 0x53, 0x65, + 0x74, 0x57, 0x72, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x38, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0b, 0x74, + 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x72, + 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x77, 0x72, + 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x65, 0x74, 0x57, 0x72, 0x69, + 0x74, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x54, 0x0a, + 0x20, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, - 0x61, 0x72, 0x64, 0x12, 0x40, 0x0a, 0x10, 0x70, 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x64, 0x5f, - 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, - 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, - 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0f, 0x70, 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x64, 0x50, 0x72, - 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x26, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x6c, 0x6f, 0x67, 0x75, 0x74, 0x69, 0x6c, 0x2e, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x32, 0x0a, - 0x1a, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x47, - 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, - 0x65, 0x6c, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x63, 0x65, 0x6c, 0x6c, - 0x73, 0x22, 0x1d, 0x0a, 0x1b, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x56, 0x53, 0x63, 0x68, - 0x65, 0x6d, 0x61, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x4f, 0x0a, 0x13, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, - 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, - 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, - 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0b, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, - 0x73, 0x22, 0x16, 0x0a, 0x14, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x64, 0x0a, 0x1a, 0x52, 0x65, 0x66, - 0x72, 0x65, 0x73, 0x68, 0x53, 0x74, 0x61, 0x74, 0x65, 0x42, 0x79, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x65, 0x6c, - 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x22, - 0x4b, 0x0a, 0x1b, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x53, 0x74, 0x61, 0x74, 0x65, 0x42, - 0x79, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, - 0x0a, 0x12, 0x69, 0x73, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x72, 0x65, 0x66, - 0x72, 0x65, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x69, 0x73, 0x50, 0x61, - 0x72, 0x74, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x22, 0x7f, 0x0a, 0x19, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x43, 0x65, - 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x72, - 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x12, - 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x22, 0x1c, 0x0a, - 0x1a, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x43, - 0x65, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x9b, 0x01, 0x0a, 0x16, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x43, 0x65, 0x6c, 0x6c, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x68, 0x61, 0x72, 0x64, 0x4e, 0x61, 0x6d, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x63, 0x65, 0x6c, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x72, - 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, - 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x22, 0x19, 0x0a, 0x17, 0x52, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x43, 0x65, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x46, 0x0a, 0x15, 0x52, 0x65, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, - 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, - 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, - 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, - 0x6c, 0x69, 0x61, 0x73, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x22, 0x7b, 0x0a, 0x16, - 0x52, 0x65, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x72, 0x69, 0x6d, - 0x61, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, - 0x52, 0x07, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x22, 0x6a, 0x0a, 0x12, 0x53, 0x65, 0x74, - 0x57, 0x72, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x38, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0b, 0x74, 0x61, - 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x72, 0x69, - 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x77, 0x72, 0x69, - 0x74, 0x61, 0x62, 0x6c, 0x65, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x65, 0x74, 0x57, 0x72, 0x69, 0x74, - 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x54, 0x0a, 0x20, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x61, 0x72, 0x64, 0x22, 0xaa, 0x03, 0x0a, 0x21, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x78, 0x0a, 0x14, 0x72, 0x65, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x13, + 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x65, 0x73, 0x12, 0x5a, 0x0a, 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x6d, 0x61, + 0x70, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x4d, 0x61, 0x70, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x4d, 0x61, 0x70, 0x1a, + 0x5f, 0x0a, 0x18, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2d, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x72, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x1a, 0x4e, 0x0a, 0x0e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, + 0x61, 0x62, 0x6c, 0x65, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0x7c, 0x0a, 0x12, 0x53, 0x6c, 0x65, 0x65, 0x70, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, + 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, + 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, + 0x69, 0x61, 0x73, 0x52, 0x0b, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, + 0x12, 0x2c, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x74, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x44, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x15, + 0x0a, 0x13, 0x53, 0x6c, 0x65, 0x65, 0x70, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x53, 0x0a, 0x17, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x38, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0b, 0x74, + 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x22, 0x1a, 0x0a, 0x18, 0x53, 0x74, + 0x61, 0x72, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x52, 0x0a, 0x16, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x38, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0b, 0x74, + 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x22, 0x19, 0x0a, 0x17, 0x53, 0x74, + 0x6f, 0x70, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x52, 0x0a, 0x21, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x45, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x52, 0x65, 0x70, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x06, 0x74, 0x61, + 0x62, 0x6c, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, + 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, + 0x73, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x22, 0xc6, 0x01, 0x0a, 0x22, 0x54, 0x61, + 0x62, 0x6c, 0x65, 0x74, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x52, 0x65, + 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, - 0x72, 0x64, 0x22, 0xaa, 0x03, 0x0a, 0x21, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x70, 0x6c, - 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x78, 0x0a, 0x14, 0x72, 0x65, 0x70, 0x6c, - 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x13, 0x72, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x65, 0x73, 0x12, 0x5a, 0x0a, 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x6d, 0x61, 0x70, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x4d, 0x61, 0x70, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x4d, 0x61, 0x70, 0x1a, 0x5f, - 0x0a, 0x18, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2d, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x72, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, - 0x4e, 0x0a, 0x0e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, - 0x62, 0x6c, 0x65, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0x7c, 0x0a, 0x12, 0x53, 0x6c, 0x65, 0x65, 0x70, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, - 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, - 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, - 0x61, 0x73, 0x52, 0x0b, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, - 0x2c, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x74, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x15, 0x0a, - 0x13, 0x53, 0x6c, 0x65, 0x65, 0x70, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x53, 0x0a, 0x17, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x38, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0b, 0x74, 0x61, - 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x22, 0x1a, 0x0a, 0x18, 0x53, 0x74, 0x61, - 0x72, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x52, 0x0a, 0x16, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x38, 0x0a, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0b, 0x74, 0x61, - 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x22, 0x19, 0x0a, 0x17, 0x53, 0x74, 0x6f, - 0x70, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x52, 0x0a, 0x21, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x45, 0x78, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x52, 0x65, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, - 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x06, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, - 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x22, 0xc6, 0x01, 0x0a, 0x22, 0x54, 0x61, 0x62, - 0x6c, 0x65, 0x74, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x6c, 0x79, 0x52, 0x65, 0x70, - 0x61, 0x72, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, - 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, - 0x64, 0x12, 0x36, 0x0a, 0x0b, 0x6e, 0x65, 0x77, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0a, 0x6e, - 0x65, 0x77, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x36, 0x0a, 0x0b, 0x6f, 0x6c, 0x64, - 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, - 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, - 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0a, 0x6f, 0x6c, 0x64, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, - 0x79, 0x22, 0x5c, 0x0a, 0x15, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x65, 0x6c, 0x6c, 0x49, - 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x72, 0x64, 0x12, 0x36, 0x0a, 0x0b, 0x6e, 0x65, 0x77, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, + 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0a, + 0x6e, 0x65, 0x77, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x36, 0x0a, 0x0b, 0x6f, 0x6c, + 0x64, 0x5f, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x15, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, + 0x74, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0a, 0x6f, 0x6c, 0x64, 0x50, 0x72, 0x69, 0x6d, 0x61, + 0x72, 0x79, 0x22, 0x5c, 0x0a, 0x15, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x65, 0x6c, 0x6c, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, + 0x2f, 0x0a, 0x09, 0x63, 0x65, 0x6c, 0x6c, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x65, + 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x63, 0x65, 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, + 0x22, 0x5d, 0x0a, 0x16, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x65, 0x6c, 0x6c, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2f, 0x0a, 0x09, 0x63, 0x65, 0x6c, 0x6c, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x65, 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x63, 0x65, 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x22, - 0x5d, 0x0a, 0x16, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x65, 0x6c, 0x6c, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2f, 0x0a, - 0x09, 0x63, 0x65, 0x6c, 0x6c, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x65, 0x6c, 0x6c, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x63, 0x65, 0x6c, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x64, - 0x0a, 0x17, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x41, 0x6c, 0x69, - 0x61, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x35, 0x0a, - 0x0b, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x65, - 0x6c, 0x6c, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0a, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x41, - 0x6c, 0x69, 0x61, 0x73, 0x22, 0x65, 0x0a, 0x18, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x65, - 0x6c, 0x6c, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x35, 0x0a, 0x0b, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x5f, 0x61, 0x6c, - 0x69, 0x61, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x74, 0x6f, 0x70, 0x6f, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, - 0x0a, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x2a, 0x4a, 0x0a, 0x15, 0x4d, - 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x55, 0x53, 0x54, 0x4f, 0x4d, 0x10, 0x00, - 0x12, 0x0e, 0x0a, 0x0a, 0x4d, 0x4f, 0x56, 0x45, 0x54, 0x41, 0x42, 0x4c, 0x45, 0x53, 0x10, 0x01, - 0x12, 0x15, 0x0a, 0x11, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x4c, 0x4f, 0x4f, 0x4b, 0x55, 0x50, - 0x49, 0x4e, 0x44, 0x45, 0x58, 0x10, 0x02, 0x42, 0x28, 0x5a, 0x26, 0x76, 0x69, 0x74, 0x65, 0x73, - 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, - 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, 0x74, - 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x64, 0x0a, 0x17, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x41, 0x6c, + 0x69, 0x61, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x35, + 0x0a, 0x0b, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, + 0x65, 0x6c, 0x6c, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x0a, 0x63, 0x65, 0x6c, 0x6c, 0x73, + 0x41, 0x6c, 0x69, 0x61, 0x73, 0x22, 0x65, 0x0a, 0x18, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, + 0x65, 0x6c, 0x6c, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x35, 0x0a, 0x0b, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x5f, 0x61, + 0x6c, 0x69, 0x61, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x74, 0x6f, 0x70, + 0x6f, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, + 0x52, 0x0a, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x2a, 0x4a, 0x0a, 0x15, + 0x4d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x55, 0x53, 0x54, 0x4f, 0x4d, 0x10, + 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x4d, 0x4f, 0x56, 0x45, 0x54, 0x41, 0x42, 0x4c, 0x45, 0x53, 0x10, + 0x01, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x4c, 0x4f, 0x4f, 0x4b, 0x55, + 0x50, 0x49, 0x4e, 0x44, 0x45, 0x58, 0x10, 0x02, 0x42, 0x28, 0x5a, 0x26, 0x76, 0x69, 0x74, 0x65, + 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, 0x2f, + 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x74, 0x63, 0x74, 0x6c, 0x64, 0x61, + 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/go/vt/proto/vtctldata/vtctldata_vtproto.pb.go b/go/vt/proto/vtctldata/vtctldata_vtproto.pb.go index db5f34bb87e..8ff60c502bc 100644 --- a/go/vt/proto/vtctldata/vtctldata_vtproto.pb.go +++ b/go/vt/proto/vtctldata/vtctldata_vtproto.pb.go @@ -2195,6 +2195,16 @@ func (m *EmergencyReparentShardRequest) MarshalToSizedBufferVT(dAtA []byte) (int i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.PreventCrossCellPromotion { + i-- + if m.PreventCrossCellPromotion { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x30 + } if m.WaitReplicasTimeout != nil { size, err := m.WaitReplicasTimeout.MarshalToSizedBufferVT(dAtA[:i]) if err != nil { @@ -6613,6 +6623,9 @@ func (m *EmergencyReparentShardRequest) SizeVT() (n int) { l = m.WaitReplicasTimeout.SizeVT() n += 1 + l + sov(uint64(l)) } + if m.PreventCrossCellPromotion { + n += 2 + } if m.unknownFields != nil { n += len(m.unknownFields) } @@ -13317,6 +13330,26 @@ func (m *EmergencyReparentShardRequest) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PreventCrossCellPromotion", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.PreventCrossCellPromotion = bool(v != 0) default: iNdEx = preIndex skippy, err := skip(dAtA[iNdEx:]) diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 617e294a8bc..359232b590b 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -594,6 +594,7 @@ func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldat } span.Annotate("wait_replicas_timeout_sec", waitReplicasTimeout.Seconds()) + span.Annotate("prevent_cross_cell_promotion", req.PreventCrossCellPromotion) m := sync.RWMutex{} logstream := []*logutilpb.Event{} @@ -607,7 +608,7 @@ func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldat ev, err := reparentutil.NewEmergencyReparenter(s.ts, s.tmc, logger).ReparentShard(ctx, req.Keyspace, req.Shard, - reparentutil.NewEmergencyReparentOptions(req.NewPrimary, sets.NewString(ignoreReplicaAliases...), waitReplicasTimeout, false)) + reparentutil.NewEmergencyReparentOptions(req.NewPrimary, sets.NewString(ignoreReplicaAliases...), waitReplicasTimeout, req.PreventCrossCellPromotion)) resp := &vtctldatapb.EmergencyReparentShardResponse{ Keyspace: req.Keyspace, diff --git a/go/vt/vtctl/reparent.go b/go/vt/vtctl/reparent.go index 6d07b1a1e04..ded144f7ba4 100644 --- a/go/vt/vtctl/reparent.go +++ b/go/vt/vtctl/reparent.go @@ -169,6 +169,7 @@ func commandEmergencyReparentShard(ctx context.Context, wr *wrangler.Wrangler, s waitReplicasTimeout := subFlags.Duration("wait_replicas_timeout", *topo.RemoteOperationTimeout, "time to wait for replicas to catch up in reparenting") keyspaceShard := subFlags.String("keyspace_shard", "", "keyspace/shard of the shard that needs to be reparented") newPrimary := subFlags.String("new_primary", "", "optional alias of a tablet that should be the new primary. If not specified, Vitess will select the best candidate") + preventCrossCellPromotion := subFlags.Bool("prevent_cross_cell_promotion", false, "only promotes a new primary from the same cell as the previous primary") ignoreReplicasList := subFlags.String("ignore_replicas", "", "comma-separated list of replica tablet aliases to ignore during emergency reparent") // handle deprecated flags @@ -205,7 +206,7 @@ func commandEmergencyReparentShard(ctx context.Context, wr *wrangler.Wrangler, s } } unreachableReplicas := topoproto.ParseTabletSet(*ignoreReplicasList) - return wr.EmergencyReparentShard(ctx, keyspace, shard, tabletAlias, *waitReplicasTimeout, unreachableReplicas) + return wr.EmergencyReparentShard(ctx, keyspace, shard, tabletAlias, *waitReplicasTimeout, unreachableReplicas, *preventCrossCellPromotion) } func commandTabletExternallyReparented(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error { diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index e31331a9323..3493aec6ead 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -145,12 +145,12 @@ func (wr *Wrangler) PlannedReparentShard(ctx context.Context, keyspace, shard st // EmergencyReparentShard will make the provided tablet the primary for // the shard, when the old primary is completely unreachable. -func (wr *Wrangler) EmergencyReparentShard(ctx context.Context, keyspace, shard string, primaryElectTabletAlias *topodatapb.TabletAlias, waitReplicasTimeout time.Duration, ignoredTablets sets.String) (err error) { +func (wr *Wrangler) EmergencyReparentShard(ctx context.Context, keyspace, shard string, primaryElectTabletAlias *topodatapb.TabletAlias, waitReplicasTimeout time.Duration, ignoredTablets sets.String, preventCrossCellPromotion bool) (err error) { _, err = reparentutil.NewEmergencyReparenter(wr.ts, wr.tmc, wr.logger).ReparentShard( ctx, keyspace, shard, - reparentutil.NewEmergencyReparentOptions(primaryElectTabletAlias, ignoredTablets, waitReplicasTimeout, false)) + reparentutil.NewEmergencyReparentOptions(primaryElectTabletAlias, ignoredTablets, waitReplicasTimeout, preventCrossCellPromotion)) return err } diff --git a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go index 71efac9286b..77668a7a5a1 100644 --- a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go @@ -273,7 +273,7 @@ func TestEmergencyReparentShardPrimaryElectNotBest(t *testing.T) { defer moreAdvancedReplica.StopActionLoop(t) // run EmergencyReparentShard - err := wr.EmergencyReparentShard(ctx, newPrimary.Tablet.Keyspace, newPrimary.Tablet.Shard, newPrimary.Tablet.Alias, 10*time.Second, sets.NewString()) + err := wr.EmergencyReparentShard(ctx, newPrimary.Tablet.Keyspace, newPrimary.Tablet.Shard, newPrimary.Tablet.Alias, 10*time.Second, sets.NewString(), false) cancel() assert.Error(t, err) diff --git a/proto/vtctldata.proto b/proto/vtctldata.proto index 624ed009117..c1ddc8f74fe 100644 --- a/proto/vtctldata.proto +++ b/proto/vtctldata.proto @@ -353,6 +353,9 @@ message EmergencyReparentShardRequest { // WaitReplicasTimeout is the duration of time to wait for replicas to catch // up in reparenting. vttime.Duration wait_replicas_timeout = 5; + // PreventCrossCellPromotion is used to only promote the new primary from the same cell + // as the failed primary. + bool prevent_cross_cell_promotion = 6; } message EmergencyReparentShardResponse { From 55493b26c471d482b15f5c2ea9f48496501f6d75 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 23 Sep 2021 17:28:35 +0530 Subject: [PATCH 116/176] refactor and addition of comments Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter.go | 104 +++++++++--------- .../vtctl/reparentutil/reparent_functions.go | 8 -- go/vt/vtctl/reparentutil/util.go | 52 +++++---- 3 files changed, 85 insertions(+), 79 deletions(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index eee31183ae6..a61457cf023 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -19,6 +19,8 @@ package reparentutil import ( "context" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/stats" replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" @@ -101,25 +103,26 @@ func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, sha // run ERS with shard already locked err = erp.reparentShardLocked(ctx, ev, keyspace, shard, opts) - opts.PostERSCompletionHook(ctx, ev, erp.logger, erp.tmc) - return ev, err } // reparentShardLocked performs Emergency Reparent Shard operation assuming that the shard is already locked -func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *events.Reparent, keyspace, shard string, opts EmergencyReparentOptions) error { +func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *events.Reparent, keyspace, shard string, opts EmergencyReparentOptions) (err error) { // log the starting of the operation and increment the counter erp.logger.Infof("will initiate emergency reparent shard in keyspace - %s, shard - %s", keyspace, shard) ersCounter.Add(1) // get the shard information from the topology server - shardInfo, err := erp.ts.GetShard(ctx, keyspace, shard) + var shardInfo *topo.ShardInfo + shardInfo, err = erp.ts.GetShard(ctx, keyspace, shard) if err != nil { return err } ev.ShardInfo = *shardInfo - // get the previous primary according to the topology server + // get the previous primary according to the topology server, + // we use this information to choose the best candidate in the same cell + // and to undo promotion in case of failure var prevPrimary *topodatapb.Tablet if shardInfo.PrimaryAlias != nil { prevPrimaryInfo, err := erp.ts.GetTablet(ctx, shardInfo.PrimaryAlias) @@ -131,13 +134,16 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // read all the tablets and there information event.DispatchUpdate(ev, "reading all tablets") - tabletMap, err := erp.ts.GetTabletMapForShard(ctx, keyspace, shard) + var tabletMap map[string]*topo.TabletInfo + tabletMap, err = erp.ts.GetTabletMapForShard(ctx, keyspace, shard) if err != nil { return vterrors.Wrapf(err, "failed to get tablet map for %v/%v: %v", keyspace, shard, err) } // Stop replication on all the tablets and build their status map - statusMap, primaryStatusMap, err := StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, tabletMap, opts.waitReplicasTimeout, opts.ignoreReplicas, erp.logger) + var statusMap map[string]*replicationdatapb.StopReplicationStatus + var primaryStatusMap map[string]*replicationdatapb.PrimaryStatus + statusMap, primaryStatusMap, err = StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, tabletMap, opts.waitReplicasTimeout, opts.ignoreReplicas, erp.logger) if err != nil { return vterrors.Wrapf(err, "failed to stop replication and build status maps: %v", err) } @@ -149,11 +155,12 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // find the valid candidates for becoming the primary // this is where we check for errant GTIDs and remove the tablets that have them from consideration - validCandidates, err := FindValidEmergencyReparentCandidates(statusMap, primaryStatusMap) + var validCandidates map[string]mysql.Position + validCandidates, err = FindValidEmergencyReparentCandidates(statusMap, primaryStatusMap) if err != nil { return err } - // Now, we restrict the valid candidates list according to the ReparentFunctions implementations + // Now, we restrict the valid candidates list, which removes some tablets from consideration validCandidates, err = restrictValidCandidates(validCandidates, tabletMap) if err != nil { return err @@ -162,39 +169,45 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // Wait for all candidates to apply relay logs - if err := waitForAllRelayLogsToApply(ctx, erp.logger, erp.tmc, validCandidates, tabletMap, statusMap, opts.waitReplicasTimeout); err != nil { + if err = waitForAllRelayLogsToApply(ctx, erp.logger, erp.tmc, validCandidates, tabletMap, statusMap, opts.waitReplicasTimeout); err != nil { return err } - // find the primary candidate that we want to promote - var newPrimary *topodatapb.Tablet + // find the intermediate primary candidate that we want to replicate from. This will always be the most advanced tablet that we have + // We let all the other tablets replicate from this primary. We will then try to choose a better candidate and let it catch up + var intermediatePrimary *topodatapb.Tablet var validCandidateTablets []*topodatapb.Tablet - newPrimary, validCandidateTablets, err = findPrimaryCandidate(erp.logger, prevPrimary, validCandidates, tabletMap, opts) + intermediatePrimary, validCandidateTablets, err = findIntermediatePrimaryCandidate(erp.logger, prevPrimary, validCandidates, tabletMap, opts) if err != nil { return err } // check weather the primary candidate selected is ideal or if it can be improved later - isIdeal := promotedReplicaIsIdeal(newPrimary, prevPrimary, validCandidateTablets, opts) + isIdeal := intermediateCandidateIsIdeal(intermediatePrimary, prevPrimary, validCandidateTablets, opts) // Check (again) we still have the topology lock. - if err := topo.CheckShardLocked(ctx, keyspace, shard); err != nil { + if err = topo.CheckShardLocked(ctx, keyspace, shard); err != nil { return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } + // initialize the newPrimary with the intermediate primary, override this value if it is not the ideal candidate + newPrimary := intermediatePrimary if !isIdeal { - // we now promote our primary candidate and also reparent all the other tablets to start replicating from this candidate - validReplacementCandidates, err := promoteIntermediatePrimary(ctx, erp.tmc, ev, erp.logger, newPrimary, opts.lockAction, tabletMap, statusMap, opts) + // we now promote our intermediate primary candidate and also reparent all the other tablets to start replicating from this candidate + // we do not promote the tablet or change the shard record. We only change the replication for all the other tablets + // it also returns the list of the tablets that started replication successfully including itself. These are the candidates that we can use to find a replacement + var validReplacementCandidates []*topodatapb.Tablet + validReplacementCandidates, err = promoteIntermediatePrimary(ctx, erp.tmc, ev, erp.logger, intermediatePrimary, opts.lockAction, tabletMap, statusMap, opts) if err != nil { return err } - // try to find a better candidate if we do not already have the most ideal one - betterCandidate := getBetterCandidate(newPrimary, prevPrimary, validReplacementCandidates, opts) + // try to find a better candidate using the list we got back + betterCandidate := getBetterCandidate(intermediatePrimary, prevPrimary, validReplacementCandidates, opts) - // if our better candidate is different from our previous candidate, then we replace our primary - if !topoproto.TabletAliasEqual(betterCandidate.Alias, newPrimary.Alias) { - err = replaceWithBetterCandidate(ctx, erp.tmc, erp.ts, ev, erp.logger, newPrimary, betterCandidate, opts.lockAction, tabletMap, statusMap, opts) + // if our better candidate is different from our previous candidate, then we wait for it to catch up to the intermediate primary + if !topoproto.TabletAliasEqual(betterCandidate.Alias, intermediatePrimary.Alias) { + err = waitForCatchingUp(ctx, erp.tmc, erp.ts, ev, erp.logger, intermediatePrimary, betterCandidate, opts.lockAction, tabletMap, statusMap, opts) if err != nil { return err } @@ -202,45 +215,36 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } } - // now we check if there is a need to override the promotion of the newPrimary - errInPromotion := checkIfNeedToOverridePromotion(newPrimary, prevPrimary, opts) - if errInPromotion != nil { - erp.logger.Errorf("have to override promotion because of constraint failure - %v", errInPromotion) - // we try and undo the promotion - newPrimary, err = erp.undoPromotion(ctx, erp.ts, ev, keyspace, shard, prevPrimary, opts.lockAction, tabletMap, statusMap, opts) - if err != nil { - return err - } - if newPrimary == nil { - return vterrors.Errorf(vtrpc.Code_ABORTED, "could not undo promotion") + // now we check if all the constraints are satisfied. If they are not, then we should abort + constraintFailure := checkIfConstraintsSatisfied(newPrimary, prevPrimary, opts) + if constraintFailure != nil { + erp.logger.Errorf("have to override promotion because of constraint failure - %v", constraintFailure) + // we want to send both the errors to the user, constraint failure and also any error encountered in undoing the promotion + defer func() { + if err != nil { + err = vterrors.Errorf(vtrpc.Code_ABORTED, "error in undoing promotion - %v, constraint failure - %v", err, constraintFailure) + } else { + err = constraintFailure + } + }() + // we now try to undo are changes. We can do so by promoting the previous primary instead of the new one we selected + if prevPrimary == nil { + return vterrors.Errorf(vtrpc.Code_ABORTED, "could not undo promotion, since shard record has no primary information") } + newPrimary = prevPrimary } - // Finally, we call PromoteReplica which fixes the semi-sync, set the primary to read-write and flushes the binlogs + // Finally, we call PromoteReplica which changes the tablet type, fixes the semi-sync, set the primary to read-write and flushes the binlogs _, err = erp.tmc.PromoteReplica(ctx, newPrimary) if err != nil { return vterrors.Wrapf(err, "primary-elect tablet %v failed to be upgraded to primary: %v", newPrimary.Alias, err) } + // we now reparent all the replicas to the new primary we have promoted _, err = reparentReplicas(ctx, ev, erp.logger, erp.tmc, newPrimary, opts.lockAction, tabletMap, statusMap, opts, false, true) if err != nil { return err } ev.NewPrimary = proto.Clone(newPrimary).(*topodatapb.Tablet) - return errInPromotion -} - -func (erp *EmergencyReparenter) undoPromotion(ctx context.Context, ts *topo.Server, ev *events.Reparent, keyspace, shard string, prevPrimary *topodatapb.Tablet, - lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions) (*topodatapb.Tablet, error) { - var err error - if prevPrimary != nil { - // promote the original primary back - _, err = promotePrimaryCandidate(ctx, erp.tmc, ts, ev, erp.logger, prevPrimary, lockAction, tabletMap, statusMap, opts, true) - if err == nil { - return prevPrimary, nil - } - erp.logger.Errorf("error in undoing promotion - %v", err) - } - - return prevPrimary, err + return err } diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go index 09c335fd39b..83dbb71d6ce 100644 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ b/go/vt/vtctl/reparentutil/reparent_functions.go @@ -17,15 +17,11 @@ limitations under the License. package reparentutil import ( - "context" "time" "k8s.io/apimachinery/pkg/util/sets" - "vitess.io/vitess/go/vt/logutil" topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/topotools/events" - "vitess.io/vitess/go/vt/vttablet/tmclient" ) type ( @@ -51,7 +47,3 @@ func NewEmergencyReparentOptions(newPrimaryAlias *topodatapb.TabletAlias, ignore preventCrossCellPromotion: preventCrossCellPromotion, } } - -// PostERSCompletionHook implements the ReparentFunctions interface -func (opts *EmergencyReparentOptions) PostERSCompletionHook(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) { -} diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 7364ca2355c..7eb4e3e2296 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -374,8 +374,8 @@ func FindCurrentPrimary(tabletMap map[string]*topo.TabletInfo, logger logutil.Lo return currentPrimary } -// replaceWithBetterCandidate promotes the newer candidate over the primary candidate that we have, but it does not set to start accepting writes -func replaceWithBetterCandidate(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, prevPrimary, newPrimary *topodatapb.Tablet, +// waitForCatchingUp promotes the newer candidate over the primary candidate that we have, but it does not set to start accepting writes +func waitForCatchingUp(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, prevPrimary, newPrimary *topodatapb.Tablet, lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions) error { // Find the primary position of the previous primary pos, err := tmc.PrimaryPosition(ctx, prevPrimary) @@ -410,8 +410,8 @@ func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMa if !ok { return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) } - // We only allow PRIMARY and REPLICA type of tablets to be considered for replication - if candidateInfo.Type != topodatapb.TabletType_PRIMARY && candidateInfo.Type != topodatapb.TabletType_REPLICA { + // We do not allow Experimental type of tablets to be considered for replication + if candidateInfo.Type == topodatapb.TabletType_EXPERIMENTAL { continue } restrictedValidCandidates[candidate] = position @@ -419,17 +419,12 @@ func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMa return restrictedValidCandidates, nil } -// findPrimaryCandidate implements the ReparentFunctions interface -func findPrimaryCandidate(logger logutil.Logger, prevPrimary *topodatapb.Tablet, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (*topodatapb.Tablet, []*topodatapb.Tablet, error) { - var validTablets []*topodatapb.Tablet - var tabletPositions []mysql.Position - for tabletAlias, position := range validCandidates { - tablet, isFound := tabletMap[tabletAlias] - if !isFound { - return nil, nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", tabletAlias) - } - validTablets = append(validTablets, tablet.Tablet) - tabletPositions = append(tabletPositions, position) +// findIntermediatePrimaryCandidate implements the ReparentFunctions interface +func findIntermediatePrimaryCandidate(logger logutil.Logger, prevPrimary *topodatapb.Tablet, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (*topodatapb.Tablet, []*topodatapb.Tablet, error) { + // convert the valid candidates into a list so that we can use it for sorting + validTablets, tabletPositions, err := getValidCandidatesAndPositionsAsList(validCandidates, tabletMap) + if err != nil { + return nil, nil, err } idealCell := "" @@ -437,18 +432,19 @@ func findPrimaryCandidate(logger logutil.Logger, prevPrimary *topodatapb.Tablet, idealCell = prevPrimary.Alias.Cell } - // sort - err := sortTabletsForERS(validTablets, tabletPositions, idealCell) + // sort the tablets for finding the best intermediate primary in ERS + err = sortTabletsForERS(validTablets, tabletPositions, idealCell) if err != nil { return nil, nil, err } + // The first tablet in the sorted list will be the most eligible candidate unless explicitly asked for some other tablet winningPrimaryTablet := validTablets[0] winningPosition := tabletPositions[0] // If we were requested to elect a particular primary, verify it's a valid - // candidate (non-zero position, no errant GTIDs) and is at least as - // advanced as the winning position. + // candidate (non-zero position, no errant GTIDs) + // Also, if the candidate is if opts.newPrimaryAlias != nil { requestedPrimaryAlias := topoproto.TabletAliasString(opts.newPrimaryAlias) pos, ok := validCandidates[requestedPrimaryAlias] @@ -467,7 +463,21 @@ func findPrimaryCandidate(logger logutil.Logger, prevPrimary *topodatapb.Tablet, return winningPrimaryTablet, validTablets, nil } -func promotedReplicaIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, opts EmergencyReparentOptions) bool { +func getValidCandidatesAndPositionsAsList(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) ([]*topodatapb.Tablet, []mysql.Position, error) { + var validTablets []*topodatapb.Tablet + var tabletPositions []mysql.Position + for tabletAlias, position := range validCandidates { + tablet, isFound := tabletMap[tabletAlias] + if !isFound { + return nil, nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", tabletAlias) + } + validTablets = append(validTablets, tablet.Tablet) + tabletPositions = append(tabletPositions, position) + } + return validTablets, tabletPositions, nil +} + +func intermediateCandidateIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, opts EmergencyReparentOptions) bool { if opts.newPrimaryAlias != nil { // explicit request to promote a specific tablet return topoproto.TabletAliasEqual(opts.newPrimaryAlias, newPrimary.Alias) @@ -566,7 +576,7 @@ func promoteIntermediatePrimary(ctx context.Context, tmc tmclient.TabletManagerC return replicasStartedReplication, nil } -func checkIfNeedToOverridePromotion(newPrimary, prevPrimary *topodatapb.Tablet, opts EmergencyReparentOptions) error { +func checkIfConstraintsSatisfied(newPrimary, prevPrimary *topodatapb.Tablet, opts EmergencyReparentOptions) error { if opts.preventCrossCellPromotion && prevPrimary != nil && newPrimary.Alias.Cell != prevPrimary.Alias.Cell { return vterrors.Errorf(vtrpc.Code_ABORTED, "elected primary does not satisfy geographic constraint - %s", topoproto.TabletAliasString(newPrimary.Alias)) } From 1f39f98da01e3e732dc2cd371e60abbbac84d37c Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 23 Sep 2021 17:30:37 +0530 Subject: [PATCH 117/176] removed reparent_functions file in reparentutil Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter.go | 27 ++++++++++ .../vtctl/reparentutil/reparent_functions.go | 49 ------------------- 2 files changed, 27 insertions(+), 49 deletions(-) delete mode 100644 go/vt/vtctl/reparentutil/reparent_functions.go diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index a61457cf023..bb21238ba4f 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -18,6 +18,9 @@ package reparentutil import ( "context" + "time" + + "k8s.io/apimachinery/pkg/util/sets" "vitess.io/vitess/go/mysql" @@ -51,6 +54,30 @@ type EmergencyReparenter struct { logger logutil.Logger } +type ( + // EmergencyReparentOptions provides optional parameters to + // EmergencyReparentShard operations. Options are passed by value, so it is safe + // for callers to mutate and reuse options structs for multiple calls. + EmergencyReparentOptions struct { + newPrimaryAlias *topodatapb.TabletAlias + ignoreReplicas sets.String + waitReplicasTimeout time.Duration + preventCrossCellPromotion bool + + lockAction string + } +) + +// NewEmergencyReparentOptions creates a new EmergencyReparentOptions which is used in ERS +func NewEmergencyReparentOptions(newPrimaryAlias *topodatapb.TabletAlias, ignoreReplicas sets.String, waitReplicasTimeout time.Duration, preventCrossCellPromotion bool) EmergencyReparentOptions { + return EmergencyReparentOptions{ + newPrimaryAlias: newPrimaryAlias, + ignoreReplicas: ignoreReplicas, + waitReplicasTimeout: waitReplicasTimeout, + preventCrossCellPromotion: preventCrossCellPromotion, + } +} + // counters for Emergency Reparent Shard var ( ersCounter = stats.NewGauge("ers_counter", "Number of time Emergency Reparent Shard has been run") diff --git a/go/vt/vtctl/reparentutil/reparent_functions.go b/go/vt/vtctl/reparentutil/reparent_functions.go deleted file mode 100644 index 83dbb71d6ce..00000000000 --- a/go/vt/vtctl/reparentutil/reparent_functions.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2021 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package reparentutil - -import ( - "time" - - "k8s.io/apimachinery/pkg/util/sets" - - topodatapb "vitess.io/vitess/go/vt/proto/topodata" -) - -type ( - // EmergencyReparentOptions provides optional parameters to - // EmergencyReparentShard operations. Options are passed by value, so it is safe - // for callers to mutate and reuse options structs for multiple calls. - EmergencyReparentOptions struct { - newPrimaryAlias *topodatapb.TabletAlias - ignoreReplicas sets.String - waitReplicasTimeout time.Duration - preventCrossCellPromotion bool - - lockAction string - } -) - -// NewEmergencyReparentOptions creates a new EmergencyReparentOptions which is used in ERS ans PRS -func NewEmergencyReparentOptions(newPrimaryAlias *topodatapb.TabletAlias, ignoreReplicas sets.String, waitReplicasTimeout time.Duration, preventCrossCellPromotion bool) EmergencyReparentOptions { - return EmergencyReparentOptions{ - newPrimaryAlias: newPrimaryAlias, - ignoreReplicas: ignoreReplicas, - waitReplicasTimeout: waitReplicasTimeout, - preventCrossCellPromotion: preventCrossCellPromotion, - } -} From f97051e3925bef6eaf7f97d656a8e6f61fa61571 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 23 Sep 2021 17:55:09 +0530 Subject: [PATCH 118/176] fix grpc tests Signed-off-by: Manan Gupta --- go/vt/vtctl/grpcvtctldserver/server_slow_test.go | 15 +++------------ go/vt/vtctl/grpcvtctldserver/server_test.go | 9 +++------ 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server_slow_test.go b/go/vt/vtctl/grpcvtctldserver/server_slow_test.go index 129652e84e4..be8e4a2f468 100644 --- a/go/vt/vtctl/grpcvtctldserver/server_slow_test.go +++ b/go/vt/vtctl/grpcvtctldserver/server_slow_test.go @@ -21,6 +21,8 @@ import ( "testing" "time" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -118,9 +120,6 @@ func TestEmergencyReparentShardSlow(t *testing.T) { }{ "zone1-0000000200": {}, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000200": nil, - }, MasterPositionResults: map[string]struct { Position string Error error @@ -240,9 +239,6 @@ func TestEmergencyReparentShardSlow(t *testing.T) { }{ "zone1-0000000200": {}, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000200": nil, - }, MasterPositionResults: map[string]struct { Position string Error error @@ -299,6 +295,7 @@ func TestEmergencyReparentShardSlow(t *testing.T) { } ctx := context.Background() + _ = reparentutil.SetDurabilityPolicy("none", nil) for _, tt := range tests { tt := tt @@ -434,9 +431,6 @@ func TestPlannedReparentShardSlow(t *testing.T) { Error: nil, }, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000200": nil, - }, SetMasterResults: map[string]error{ "zone1-0000000200": nil, // waiting for master-position during promotion // reparent SetMaster calls @@ -540,9 +534,6 @@ func TestPlannedReparentShardSlow(t *testing.T) { Error: nil, }, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000200": nil, - }, SetMasterResults: map[string]error{ "zone1-0000000200": nil, // waiting for master-position during promotion // reparent SetMaster calls diff --git a/go/vt/vtctl/grpcvtctldserver/server_test.go b/go/vt/vtctl/grpcvtctldserver/server_test.go index a65c249fd03..d67e2233b44 100644 --- a/go/vt/vtctl/grpcvtctldserver/server_test.go +++ b/go/vt/vtctl/grpcvtctldserver/server_test.go @@ -24,6 +24,8 @@ import ( "testing" "time" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" @@ -2658,9 +2660,6 @@ func TestEmergencyReparentShard(t *testing.T) { }{ "zone1-0000000200": {}, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000200": nil, - }, SetMasterResults: map[string]error{ "zone1-0000000100": nil, "zone1-0000000101": nil, @@ -2746,6 +2745,7 @@ func TestEmergencyReparentShard(t *testing.T) { } ctx := context.Background() + _ = reparentutil.SetDurabilityPolicy("none", nil) for _, tt := range tests { tt := tt @@ -4776,9 +4776,6 @@ func TestPlannedReparentShard(t *testing.T) { Error: nil, }, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000200": nil, - }, SetMasterResults: map[string]error{ "zone1-0000000200": nil, // waiting for master-position during promotion // reparent SetMaster calls From 5c2fa2b9b188d636a5b3ebc6023d2f105471fa9a Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 23 Sep 2021 20:31:52 +0530 Subject: [PATCH 119/176] fix emergency_reparent tests Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter.go | 23 +- .../reparentutil/emergency_reparenter_test.go | 225 ++++++++++++------ go/vt/vtctl/reparentutil/util.go | 16 +- 3 files changed, 174 insertions(+), 90 deletions(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index bb21238ba4f..e64f0ee7fdb 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -210,7 +210,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // check weather the primary candidate selected is ideal or if it can be improved later - isIdeal := intermediateCandidateIsIdeal(intermediatePrimary, prevPrimary, validCandidateTablets, opts) + isIdeal := intermediateCandidateIsIdeal(intermediatePrimary, prevPrimary, validCandidateTablets, tabletMap, opts) // Check (again) we still have the topology lock. if err = topo.CheckShardLocked(ctx, keyspace, shard); err != nil { @@ -230,7 +230,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // try to find a better candidate using the list we got back - betterCandidate := getBetterCandidate(intermediatePrimary, prevPrimary, validReplacementCandidates, opts) + betterCandidate := getBetterCandidate(intermediatePrimary, prevPrimary, validReplacementCandidates, tabletMap, opts) // if our better candidate is different from our previous candidate, then we wait for it to catch up to the intermediate primary if !topoproto.TabletAliasEqual(betterCandidate.Alias, intermediatePrimary.Alias) { @@ -261,8 +261,19 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve newPrimary = prevPrimary } - // Finally, we call PromoteReplica which changes the tablet type, fixes the semi-sync, set the primary to read-write and flushes the binlogs - _, err = erp.tmc.PromoteReplica(ctx, newPrimary) + // Final step is to promote our primary candidate + err = erp.promoteNewPrimary(ctx, ev, newPrimary, opts, tabletMap, statusMap) + if err != nil { + return err + } + + ev.NewPrimary = proto.Clone(newPrimary).(*topodatapb.Tablet) + return err +} + +func (erp *EmergencyReparenter) promoteNewPrimary(ctx context.Context, ev *events.Reparent, newPrimary *topodatapb.Tablet, opts EmergencyReparentOptions, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus) error { + // we call PromoteReplica which changes the tablet type, fixes the semi-sync, set the primary to read-write and flushes the binlogs + _, err := erp.tmc.PromoteReplica(ctx, newPrimary) if err != nil { return vterrors.Wrapf(err, "primary-elect tablet %v failed to be upgraded to primary: %v", newPrimary.Alias, err) } @@ -271,7 +282,5 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve if err != nil { return err } - - ev.NewPrimary = proto.Clone(newPrimary).(*topodatapb.Tablet) - return err + return nil } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 28c0ec154ad..5911920dba9 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -109,9 +109,9 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { t.Parallel() tests := []struct { - name string - vtctlReparentFunctions EmergencyReparentOptions - tmc *testutil.TabletManagerClient + name string + emergencyReparentOps EmergencyReparentOptions + tmc *testutil.TabletManagerClient // setup ts *topo.Server keyspace string @@ -124,8 +124,8 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { errShouldContain string }{ { - name: "success", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), + name: "success", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000102": nil, @@ -238,7 +238,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { // Here, all our tablets are tied, so we're going to explicitly pick // zone1-101. name: "success with requested primary-elect", - vtctlReparentFunctions: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ + emergencyReparentOps: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ Cell: "zone1", Uid: 101, }, nil, 0, false), @@ -350,8 +350,8 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { shouldErr: false, }, { - name: "success with existing primary", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), + name: "success with existing primary", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.PrimaryStatus @@ -463,20 +463,20 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { shouldErr: false, }, { - name: "shard not found", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), - tmc: &testutil.TabletManagerClient{}, - unlockTopo: true, // we shouldn't try to lock the nonexistent shard - shards: nil, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: true, - errShouldContain: "node doesn't exist: keyspaces/testkeyspace/shards/-/Shard", + name: "shard not found", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + tmc: &testutil.TabletManagerClient{}, + unlockTopo: true, // we shouldn't try to lock the nonexistent shard + shards: nil, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: "node doesn't exist: keyspaces/testkeyspace/shards/-/Shard", }, { - name: "cannot stop replication", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), + name: "cannot stop replication", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -534,8 +534,8 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { errShouldContain: "failed to stop replication and build status maps", }, { - name: "lost topo lock", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), + name: "lost topo lock", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -593,8 +593,8 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { errShouldContain: "lost topology lock, aborting", }, { - name: "cannot get reparent candidates", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), + name: "cannot get reparent candidates", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -664,9 +664,9 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { errShouldContain: "encountered tablet zone1-0000000102 with no relay log position", }, { - name: "zero valid reparent candidates", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), - tmc: &testutil.TabletManagerClient{}, + name: "zero valid reparent candidates", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + tmc: &testutil.TabletManagerClient{}, shards: []*vtctldatapb.Shard{ { Keyspace: "testkeyspace", @@ -682,7 +682,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { { name: "error waiting for relay logs to apply", // one replica is going to take a minute to apply relay logs - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, time.Millisecond*50, false), + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, time.Millisecond*50, false), tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -771,7 +771,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "requested primary-elect is not in tablet map", - vtctlReparentFunctions: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ + emergencyReparentOps: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ Cell: "zone1", Uid: 200, }, nil, 0, false), @@ -858,7 +858,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "requested primary-elect is not winning primary-elect", - vtctlReparentFunctions: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ // we're requesting a tablet that's behind in replication + emergencyReparentOps: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ // we're requesting a tablet that's behind in replication Cell: "zone1", Uid: 102, }, nil, 0, false), @@ -866,6 +866,35 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { shard: "-", ts: memorytopo.NewServer("zone1"), tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000102": nil, + }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000102": { + Result: "ok", + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000100": nil, + "zone1-0000000101": nil, + "zone1-0000000102": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", + Error: nil, + }, + "zone1-0000000102": { + Error: nil, + }, + }, StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status StopStatus *replicationdatapb.StopReplicationStatus @@ -873,6 +902,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }{ "zone1-0000000100": { StopStatus: &replicationdatapb.StopReplicationStatus{ + Before: &replicationdatapb.Status{}, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", @@ -881,14 +911,16 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, "zone1-0000000101": { StopStatus: &replicationdatapb.StopReplicationStatus{ + Before: &replicationdatapb.Status{}, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", - RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-20", }, }, }, "zone1-0000000102": { StopStatus: &replicationdatapb.StopReplicationStatus{ + Before: &replicationdatapb.Status{}, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-20", @@ -901,10 +933,11 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21": nil, }, "zone1-0000000101": { - "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21": nil, + "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-20": nil, }, "zone1-0000000102": { "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-20": nil, + "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21": nil, }, }, }, @@ -941,12 +974,11 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { Hostname: "not most up-to-date position", }, }, - shouldErr: true, - errShouldContain: "primary elect zone1-0000000102 at position 3e11fa47-71ca-11e1-9e33-c80aa9429562:1-20 is not fully caught up", + shouldErr: false, }, { name: "cannot promote new primary", - vtctlReparentFunctions: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ + emergencyReparentOps: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ Cell: "zone1", Uid: 102, }, nil, 0, false), @@ -1061,6 +1093,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, } + _ = SetDurabilityPolicy("none", nil) for _, tt := range tests { tt := tt @@ -1075,7 +1108,6 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { tablet.Type = topodatapb.TabletType_REPLICA tt.tablets[i] = tablet } - tt.tmc.TopoServer = tt.ts testutil.AddShards(ctx, t, tt.ts, tt.shards...) testutil.AddTablets(ctx, t, tt.ts, nil, tt.tablets...) @@ -1094,7 +1126,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { erp := NewEmergencyReparenter(tt.ts, tt.tmc, logger) - err := erp.reparentShardLocked(ctx, ev, tt.keyspace, tt.shard, tt.vtctlReparentFunctions) + err := erp.reparentShardLocked(ctx, ev, tt.keyspace, tt.shard, tt.emergencyReparentOps) if tt.shouldErr { assert.Error(t, err) assert.Contains(t, err.Error(), tt.errShouldContain) @@ -1106,27 +1138,27 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { } } -func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { +func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { t.Parallel() tests := []struct { - name string - vtctlReparentFunctions EmergencyReparentOptions - tmc *testutil.TabletManagerClient - unlockTopo bool - newPrimaryTabletAlias string - ts *topo.Server - keyspace string - shard string - tablets []*topodatapb.Tablet - tabletMap map[string]*topo.TabletInfo - statusMap map[string]*replicationdatapb.StopReplicationStatus - shouldErr bool - errShouldContain string + name string + emergencyReparentOps EmergencyReparentOptions + tmc *testutil.TabletManagerClient + unlockTopo bool + newPrimaryTabletAlias string + ts *topo.Server + keyspace string + shard string + tablets []*topodatapb.Tablet + tabletMap map[string]*topo.TabletInfo + statusMap map[string]*replicationdatapb.StopReplicationStatus + shouldErr bool + errShouldContain string }{ { - name: "success", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, sets.NewString("zone1-0000000404"), 0, false), + name: "success", + emergencyReparentOps: NewEmergencyReparentOptions(nil, sets.NewString("zone1-0000000404"), 0, false), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1139,14 +1171,19 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { Error: nil, }, }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, SetMasterResults: map[string]error{ "zone1-0000000101": nil, "zone1-0000000102": nil, "zone1-0000000404": assert.AnError, // okay, because we're ignoring it. }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000100": nil, - }, }, newPrimaryTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ @@ -1206,8 +1243,8 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { shouldErr: false, }, { - name: "MasterPosition error", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), + name: "MasterPosition error", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string @@ -1217,8 +1254,13 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { Error: fmt.Errorf("primary position error"), }, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000100": nil, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000100": { + Error: fmt.Errorf("primary position error"), + }, }, }, newPrimaryTabletAlias: "zone1-0000000100", @@ -1248,8 +1290,8 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { errShouldContain: "primary position error", }, { - name: "cannot repopulate reparent journal on new primary", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), + name: "cannot repopulate reparent journal on new primary", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": assert.AnError, @@ -1261,8 +1303,14 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { "zone1-0000000100": { Error: nil, }, - }, ChangeTabletTypeResult: map[string]error{ - "zone1-0000000100": nil, + }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, }, }, newPrimaryTabletAlias: "zone1-0000000100", @@ -1292,8 +1340,8 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { errShouldContain: "failed to PopulateReparentJournal on primary", }, { - name: "all replicas failing to SetMaster does fail the promotion", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), + name: "all replicas failing to SetMaster does fail the promotion", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1306,14 +1354,20 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { Error: nil, }, }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + SetMasterResults: map[string]error{ // everyone fails, we all fail "zone1-0000000101": assert.AnError, "zone1-0000000102": assert.AnError, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000100": nil, - }, }, newPrimaryTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ @@ -1350,8 +1404,8 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { errShouldContain: " replica(s) failed", }, { - name: "all replicas slow to SetMaster does fail the promotion", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, time.Millisecond*10, false), + name: "all replicas slow to SetMaster does fail the promotion", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, time.Millisecond*10, false), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1364,6 +1418,15 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { Error: nil, }, }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + SetMasterDelays: map[string]time.Duration{ // nothing is failing, we're just slow "zone1-0000000101": time.Millisecond * 100, @@ -1373,9 +1436,6 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { "zone1-0000000101": nil, "zone1-0000000102": nil, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000100": nil, - }, }, newPrimaryTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ @@ -1412,8 +1472,8 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { errShouldContain: "context deadline exceeded", }, { - name: "one replica failing to SetMaster does not fail the promotion", - vtctlReparentFunctions: NewEmergencyReparentOptions(nil, nil, 0, false), + name: "one replica failing to SetMaster does not fail the promotion", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1426,9 +1486,15 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { Error: nil, }, }, - ChangeTabletTypeResult: map[string]error{ - "zone1-0000000100": nil, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, }, + SetMasterResults: map[string]error{ "zone1-0000000101": nil, // this one succeeds, so we're good "zone1-0000000102": assert.AnError, @@ -1500,7 +1566,8 @@ func TestEmergencyReparenter_promotePrimaryCandidate(t *testing.T) { } tabletInfo := tt.tabletMap[tt.newPrimaryTabletAlias] - _, err := promotePrimaryCandidate(ctx, tt.tmc, tt.ts, ev, logger, tabletInfo.Tablet, "", tt.tabletMap, tt.statusMap, tt.vtctlReparentFunctions, true) + erp := NewEmergencyReparenter(tt.ts, tt.tmc, logger) + err := erp.promoteNewPrimary(ctx, ev, tabletInfo.Tablet, tt.emergencyReparentOps, tt.tabletMap, tt.statusMap) if tt.shouldErr { assert.Error(t, err) assert.Contains(t, err.Error(), tt.errShouldContain) diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 7eb4e3e2296..d2fef6de9d9 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -378,7 +378,7 @@ func FindCurrentPrimary(tabletMap map[string]*topo.TabletInfo, logger logutil.Lo func waitForCatchingUp(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, prevPrimary, newPrimary *topodatapb.Tablet, lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions) error { // Find the primary position of the previous primary - pos, err := tmc.PrimaryPosition(ctx, prevPrimary) + pos, err := tmc.MasterPosition(ctx, prevPrimary) if err != nil { return err } @@ -477,15 +477,23 @@ func getValidCandidatesAndPositionsAsList(validCandidates map[string]mysql.Posit return validTablets, tabletPositions, nil } -func intermediateCandidateIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, opts EmergencyReparentOptions) bool { +func intermediateCandidateIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) bool { if opts.newPrimaryAlias != nil { // explicit request to promote a specific tablet return topoproto.TabletAliasEqual(opts.newPrimaryAlias, newPrimary.Alias) } - return getBetterCandidate(newPrimary, prevPrimary, validCandidates, opts) == newPrimary + return getBetterCandidate(newPrimary, prevPrimary, validCandidates, tabletMap, opts) == newPrimary } -func getBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, opts EmergencyReparentOptions) *topodatapb.Tablet { +func getBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) *topodatapb.Tablet { + if opts.newPrimaryAlias != nil { + // explicit request to promote a specific tablet + requestedPrimaryAlias := topoproto.TabletAliasString(opts.newPrimaryAlias) + requestedPrimaryInfo, isFound := tabletMap[requestedPrimaryAlias] + if isFound { + return requestedPrimaryInfo.Tablet + } + } var preferredCandidates []*topodatapb.Tablet var neutralReplicas []*topodatapb.Tablet for _, candidate := range validCandidates { From fce72c8d93399613b45d29fca962effa047978f2 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 23 Sep 2021 21:21:14 +0530 Subject: [PATCH 120/176] added comments to util file Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/util.go | 48 +++++++++----------------------- 1 file changed, 13 insertions(+), 35 deletions(-) diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index d2fef6de9d9..fbbe4c230fa 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -298,30 +298,6 @@ func ChooseNewPrimary( return nil, nil } -// promotePrimaryCandidate promotes the primary candidate that we have, but it does not yet set to start accepting writes -func promotePrimaryCandidate(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, - lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions, isIdeal bool) ([]*topodatapb.Tablet, error) { - // first step is change the type of the newPrimary tablet to PRIMARY - if err := changeTypeToPrimary(ctx, tmc, newPrimary); err != nil { - return nil, err - } - - // now we reparent all the other tablets to start replication from our new primary - // if the promoted primary is not ideal then we wait for all the replicas so that we choose a better candidate from them later - replicasStartedReplication, err := reparentReplicas(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, opts, !isIdeal, true) - if err != nil { - return nil, err - } - - return replicasStartedReplication, nil -} - -// changeTypeToPrimary changes the type of the tablet to primary -// the tablet's shard sync will update the topo server for us. -func changeTypeToPrimary(ctx context.Context, tmc tmclient.TabletManagerClient, newPrimary *topodatapb.Tablet) error { - return tmc.ChangeType(ctx, newPrimary, topodatapb.TabletType_PRIMARY) -} - // FindCurrentPrimary returns the current primary tablet of a shard, if any. The // current primary is whichever tablet of type PRIMARY (if any) has the most // recent PrimaryTermStartTime, which is the same rule that vtgate uses to route @@ -402,7 +378,7 @@ func getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { return action } -// restrictValidCandidates implements the ReparentFunctions interface +// restrictValidCandidates is used to restrict some candidates from being considered eligible for becoming the intermediate primary func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { restrictedValidCandidates := make(map[string]mysql.Position) for candidate, position := range validCandidates { @@ -419,7 +395,7 @@ func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMa return restrictedValidCandidates, nil } -// findIntermediatePrimaryCandidate implements the ReparentFunctions interface +// findIntermediatePrimaryCandidate finds the intermediate primary candidate for ERS. We always choose the most advanced one from our valid candidates list func findIntermediatePrimaryCandidate(logger logutil.Logger, prevPrimary *topodatapb.Tablet, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (*topodatapb.Tablet, []*topodatapb.Tablet, error) { // convert the valid candidates into a list so that we can use it for sorting validTablets, tabletPositions, err := getValidCandidatesAndPositionsAsList(validCandidates, tabletMap) @@ -463,6 +439,7 @@ func findIntermediatePrimaryCandidate(logger logutil.Logger, prevPrimary *topoda return winningPrimaryTablet, validTablets, nil } +// getValidCandidatesAndPositionsAsList converts the valid candidates from a map to a list of tablets, making it easier to sort func getValidCandidatesAndPositionsAsList(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) ([]*topodatapb.Tablet, []mysql.Position, error) { var validTablets []*topodatapb.Tablet var tabletPositions []mysql.Position @@ -477,14 +454,13 @@ func getValidCandidatesAndPositionsAsList(validCandidates map[string]mysql.Posit return validTablets, tabletPositions, nil } +// intermediateCandidateIsIdeal is used to find whether the intermediate candidate that ERS chose is also the ideal one or not func intermediateCandidateIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) bool { - if opts.newPrimaryAlias != nil { - // explicit request to promote a specific tablet - return topoproto.TabletAliasEqual(opts.newPrimaryAlias, newPrimary.Alias) - } + // we try to find a better candidate with the current list of valid candidates, and if it matches our current primary candidate, then we return true return getBetterCandidate(newPrimary, prevPrimary, validCandidates, tabletMap, opts) == newPrimary } +// getBetterCandidate is used to find a better candidate for ERS promotion func getBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) *topodatapb.Tablet { if opts.newPrimaryAlias != nil { // explicit request to promote a specific tablet @@ -573,17 +549,19 @@ func findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary *topo // promoteIntermediatePrimary promotes the primary candidate that we have, but it does not yet set to start accepting writes func promoteIntermediatePrimary(ctx context.Context, tmc tmclient.TabletManagerClient, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions) ([]*topodatapb.Tablet, error) { - // now we reparent all the other tablets to start replication from our new primary - // if the promoted primary is not ideal then we wait for all the replicas so that we choose a better candidate from them later - replicasStartedReplication, err := reparentReplicas(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, opts, true, false) + // we reparent all the other tablets to start replication from our new primary + // we wait for all the replicas so that we can choose a better candidate from the ones that started replication later + validCandidatesForImprovement, err := reparentReplicas(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, opts, true, false) if err != nil { return nil, err } - replicasStartedReplication = append(replicasStartedReplication, newPrimary) - return replicasStartedReplication, nil + // also include the current tablet for being considered as part of valid candidates for ERS promotion + validCandidatesForImprovement = append(validCandidatesForImprovement, newPrimary) + return validCandidatesForImprovement, nil } +// checkIfConstraintsSatisfied is used to check whether the constraints for ERS are satisfied or not. func checkIfConstraintsSatisfied(newPrimary, prevPrimary *topodatapb.Tablet, opts EmergencyReparentOptions) error { if opts.preventCrossCellPromotion && prevPrimary != nil && newPrimary.Alias.Cell != prevPrimary.Alias.Cell { return vterrors.Errorf(vtrpc.Code_ABORTED, "elected primary does not satisfy geographic constraint - %s", topoproto.TabletAliasString(newPrimary.Alias)) From a5b106f82cc2524f35f8baee322316e426bbbb79 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 23 Sep 2021 22:08:44 +0530 Subject: [PATCH 121/176] added logging to ers functions Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 424 ------------------ go/vt/orchestrator/logic/topology_recovery.go | 3 +- .../reparentutil/emergency_reparenter.go | 7 +- go/vt/vtctl/reparentutil/util.go | 31 +- 4 files changed, 30 insertions(+), 435 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index 7aeb9e79c3d..da3a40284e6 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -19,18 +19,6 @@ package logic import ( "context" "fmt" - "time" - - "vitess.io/vitess/go/vt/vtctl/reparentutil" - - "vitess.io/vitess/go/vt/topo/topoproto" - - "vitess.io/vitess/go/vt/proto/vtrpc" - "vitess.io/vitess/go/vt/vterrors" - - "vitess.io/vitess/go/mysql" - - "k8s.io/apimachinery/pkg/util/sets" "vitess.io/vitess/go/vt/orchestrator/attributes" "vitess.io/vitess/go/vt/orchestrator/kv" @@ -41,11 +29,8 @@ import ( "vitess.io/vitess/go/vt/orchestrator/config" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/orchestrator/external/golib/log" "vitess.io/vitess/go/vt/orchestrator/inst" - "vitess.io/vitess/go/vt/topo" ) // VtOrcReparentFunctions is the VtOrc implementation for ReparentFunctions @@ -59,415 +44,6 @@ type VtOrcReparentFunctions struct { hasBestPromotionRule bool } -// NewVtorcReparentFunctions is used to create a new VtOrcReparentFunctions -func NewVtorcReparentFunctions(analysisEntry inst.ReplicationAnalysis, candidateInstanceKey *inst.InstanceKey, skipProcesses bool, topologyRecovery *TopologyRecovery) *VtOrcReparentFunctions { - return &VtOrcReparentFunctions{ - analysisEntry: analysisEntry, - candidateInstanceKey: candidateInstanceKey, - skipProcesses: skipProcesses, - topologyRecovery: topologyRecovery, - } -} - -// LockAction implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) LockAction() string { - return "Orc Recovery" -} - -// GetWaitReplicasTimeout implements the ReparentFunctions interface -// TODO : Discuss correct way -func (vtorcReparent *VtOrcReparentFunctions) GetWaitReplicasTimeout() time.Duration { - return 1 * time.Second -} - -// GetWaitForRelayLogsTimeout implements the ReparentFunctions interface -// TODO : Discuss correct way -func (vtorcReparent *VtOrcReparentFunctions) GetWaitForRelayLogsTimeout() time.Duration { - return 1 * time.Second -} - -// GetIgnoreReplicas implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) GetIgnoreReplicas() sets.String { - // vtorc does not ignore any replicas - return nil -} - -// RestrictValidCandidates implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) RestrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { - // we do not restrict the valid candidates for VtOrc for 2 reasons - - // any candidate that can no longer replicate from the new primary is detached later on - // we already restrict which candidate can become the primary via inst.IsBannedFromBeingCandidateReplica when we choose the candidate - return validCandidates, nil -} - -// FindPrimaryCandidate implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) FindPrimaryCandidate(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (*topodatapb.Tablet, map[string]*topo.TabletInfo, error) { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, "RecoverDeadPrimary: regrouping replicas via GTID") - lostReplicas, promotedReplica, hasBestPromotionRule, err := ChooseCandidate(&vtorcReparent.analysisEntry.AnalyzedInstanceKey, validCandidates, tabletMap, logger) - vtorcReparent.topologyRecovery.AddError(err) - vtorcReparent.hasBestPromotionRule = hasBestPromotionRule - if err != nil { - return nil, nil, err - } - newPrimary, err := inst.ReadTablet(promotedReplica.Key) - if err != nil { - return nil, nil, err - } - - for _, replica := range lostReplicas { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: - lost replica: %+v", replica.Key)) - } - - // detach lost replicas if the configuration specifies it - if promotedReplica != nil && len(lostReplicas) > 0 && config.Config.DetachLostReplicasAfterPrimaryFailover { - postponedFunction := func() error { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: lost %+v replicas during recovery process; detaching them", len(lostReplicas))) - for _, replica := range lostReplicas { - replica := replica - inst.DetachReplicaPrimaryHost(&replica.Key) - } - return nil - } - vtorcReparent.topologyRecovery.AddPostponedFunction(postponedFunction, fmt.Sprintf("RecoverDeadPrimary, detach %+v lost replicas", len(lostReplicas))) - } - - func() error { - // TODO(sougou): Commented out: this downtime feels a little aggressive. - //inst.BeginDowntime(inst.NewDowntime(failedInstanceKey, inst.GetMaintenanceOwner(), inst.DowntimeLostInRecoveryMessage, time.Duration(config.LostInRecoveryDowntimeSeconds)*time.Second)) - acknowledgeInstanceFailureDetection(&vtorcReparent.analysisEntry.AnalyzedInstanceKey) - for _, replica := range lostReplicas { - replica := replica - inst.BeginDowntime(inst.NewDowntime(&replica.Key, inst.GetMaintenanceOwner(), inst.DowntimeLostInRecoveryMessage, time.Duration(config.LostInRecoveryDowntimeSeconds)*time.Second)) - } - return nil - }() - - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: %d postponed functions", vtorcReparent.topologyRecovery.PostponedFunctionsContainer.Len())) - - vtorcReparent.promotedReplica = promotedReplica - vtorcReparent.topologyRecovery.LostReplicas.AddInstances(lostReplicas) - vtorcReparent.recoveryAttempted = true - - // we now remove the lost replicas from the tabletMap so that we do not try to move them below the newly elected candidate now - tabletMapWithoutLostReplicas := map[string]*topo.TabletInfo{} - - for alias, info := range tabletMap { - instance := getInstanceFromTablet(info.Tablet) - isLost := false - for _, replica := range lostReplicas { - if instance.Key.Equals(&replica.Key) { - isLost = true - break - } - } - if !isLost { - tabletMapWithoutLostReplicas[alias] = info - } - } - - return newPrimary, tabletMapWithoutLostReplicas, nil -} - -// ChooseCandidate will choose a candidate replica of a given instance and also returns whether the chosen candidate has the best promotion rule or not -func ChooseCandidate( - primaryKey *inst.InstanceKey, - validCandidates map[string]mysql.Position, - tabletMap map[string]*topo.TabletInfo, - logger logutil.Logger, -) ( - lostReplicas [](*inst.Instance), - candidateReplica *inst.Instance, - hasBestPromotionRule bool, - err error, -) { - var emptyReplicas [](*inst.Instance) - - dataCenterHint := "" - if primary, _, _ := inst.ReadInstance(primaryKey); primary != nil { - dataCenterHint = primary.DataCenter - } - - var replicas [](*inst.Instance) - - for candidate := range validCandidates { - candidateInfo, ok := tabletMap[candidate] - if !ok { - return emptyReplicas, candidateReplica, false, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) - } - candidateInstance, _, err := inst.ReadInstance(&inst.InstanceKey{ - Hostname: candidateInfo.MysqlHostname, - Port: int(candidateInfo.MysqlPort), - }) - if err != nil { - logger.Errorf("%v", err) - return emptyReplicas, candidateReplica, false, err - } - replicas = append(replicas, candidateInstance) - } - - inst.SortInstancesDataCenterHint(replicas, dataCenterHint) - for _, replica := range replicas { - logger.Infof("ChooseCandidate - sorted replica: %+v %+v", replica.Key, replica.ExecBinlogCoordinates) - } - - candidateReplica, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := inst.ChooseCandidateReplica(replicas) - if err != nil { - return emptyReplicas, candidateReplica, false, err - } - if candidateReplica == nil { - return emptyReplicas, candidateReplica, false, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "could not find a candidate replica for ERS") - } - mostUpToDateReplica := replicas[0] - if candidateReplica.ExecBinlogCoordinates.SmallerThan(&mostUpToDateReplica.ExecBinlogCoordinates) { - logger.Warningf("ChooseCandidate: chosen replica: %+v is behind most-up-to-date replica: %+v", candidateReplica.Key, mostUpToDateReplica.Key) - } - - logger.Infof("ChooseCandidate: candidate: %+v, ahead: %d, equal: %d, late: %d, break: %d", candidateReplica.Key, len(aheadReplicas), len(equalReplicas), len(laterReplicas), len(cannotReplicateReplicas)) - - replicasToMove := append(equalReplicas, laterReplicas...) - hasBestPromotionRule = true - for _, replica := range replicasToMove { - if replica.PromotionRule.BetterThan(candidateReplica.PromotionRule) { - hasBestPromotionRule = false - } - } - - lostReplicas = append(lostReplicas, aheadReplicas...) - lostReplicas = append(lostReplicas, cannotReplicateReplicas...) - return lostReplicas, candidateReplica, hasBestPromotionRule, nil -} - -// PromotedReplicaIsIdeal implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) PromotedReplicaIsIdeal(newPrimary, oldPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, validCandidates map[string]mysql.Position) bool { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: promotedReplicaIsIdeal(%+v)", newPrimary.Alias)) - newPrimaryKey := &inst.InstanceKey{ - Hostname: newPrimary.MysqlHostname, - Port: int(newPrimary.MysqlPort), - } - newPrimaryInst, _, _ := inst.ReadInstance(newPrimaryKey) - if vtorcReparent.candidateInstanceKey != nil { - // explicit request to promote a specific server - return newPrimaryKey.Equals(vtorcReparent.candidateInstanceKey) - } - if newPrimaryInst.DataCenter == vtorcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstanceDataCenter && - newPrimaryInst.PhysicalEnvironment == vtorcReparent.topologyRecovery.AnalysisEntry.AnalyzedInstancePhysicalEnvironment { - if newPrimaryInst.PromotionRule == reparentutil.MustPromoteRule || newPrimaryInst.PromotionRule == reparentutil.PreferPromoteRule || - (vtorcReparent.hasBestPromotionRule && newPrimaryInst.PromotionRule != reparentutil.MustNotPromoteRule) { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: found %+v to be ideal candidate; will optimize recovery", newPrimaryInst.Key)) - return true - } - } - return false -} - -// PostTabletChangeHook implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) PostTabletChangeHook(tablet *topodatapb.Tablet) { - instanceKey := &inst.InstanceKey{ - Hostname: tablet.MysqlHostname, - Port: int(tablet.MysqlPort), - } - inst.ReadTopologyInstance(instanceKey) - TabletRefresh(*instanceKey) -} - -// GetBetterCandidate implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) GetBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo) *topodatapb.Tablet { - if vtorcReparent.candidateInstanceKey != nil { - candidateTablet, _ := inst.ReadTablet(*vtorcReparent.candidateInstanceKey) - // return the requested candidate as long as it is valid - for _, validCandidate := range validCandidates { - if topoproto.TabletAliasEqual(validCandidate.Alias, candidateTablet.Alias) { - return validCandidate - } - } - } - replacementCandidate := getReplacementForPromotedReplica(vtorcReparent.topologyRecovery, newPrimary, prevPrimary, validCandidates) - vtorcReparent.promotedReplica = getInstanceFromTablet(replacementCandidate) - - return replacementCandidate -} - -// TODO: make simpler -func getReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, newPrimary, oldPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet) *topodatapb.Tablet { - var preferredCandidates []*topodatapb.Tablet - var neutralReplicas []*topodatapb.Tablet - for _, candidate := range validCandidates { - promotionRule := reparentutil.PromotionRule(candidate) - if promotionRule == reparentutil.MustPromoteRule || promotionRule == reparentutil.PreferPromoteRule { - preferredCandidates = append(preferredCandidates, candidate) - } - if promotionRule == reparentutil.NeutralPromoteRule { - neutralReplicas = append(neutralReplicas, candidate) - } - } - - // So we've already promoted a replica. - // However, can we improve on our choice? Are there any replicas marked with "is_candidate"? - // Maybe we actually promoted such a replica. Does that mean we should keep it? - // Maybe we promoted a "neutral", and some "prefer" server is available. - // Maybe we promoted a "prefer_not" - // Maybe we promoted a server in a different DC than the primary - // There's many options. We may wish to replace the server we promoted with a better one. - AuditTopologyRecovery(topologyRecovery, "checking if should replace promoted replica with a better candidate") - AuditTopologyRecovery(topologyRecovery, "+ checking if promoted replica is the ideal candidate") - if oldPrimary != nil { - for _, candidateReplica := range preferredCandidates { - if topoproto.TabletAliasEqual(newPrimary.Alias, candidateReplica.Alias) && - newPrimary.Alias.Cell == oldPrimary.Alias.Cell { - // Seems like we promoted a candidate in the same cell as dead IM! Ideal! We're happy! - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("promoted replica %+v is the ideal candidate", newPrimary.Alias)) - return newPrimary - } - } - } - // We didn't pick the ideal candidate; let's see if we can replace with a candidate from same DC and ENV - - // Try a candidate replica that is in same DC & env as the dead instance - AuditTopologyRecovery(topologyRecovery, "+ searching for an ideal candidate") - if oldPrimary != nil { - for _, candidateReplica := range preferredCandidates { - if canTakeOverPromotedServerAsPrimary(getInstanceFromTablet(candidateReplica), getInstanceFromTablet(newPrimary)) && - candidateReplica.Alias.Cell == oldPrimary.Alias.Cell { - // This would make a great candidate - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("orchestrator picks %+v as candidate replacement, based on being in same cell as failed instance", candidateReplica.Alias)) - return candidateReplica - } - } - } - - // We cannot find a candidate in same DC and ENV as dead primary - AuditTopologyRecovery(topologyRecovery, "+ checking if promoted replica is an OK candidate") - for _, candidateReplica := range preferredCandidates { - if topoproto.TabletAliasEqual(newPrimary.Alias, candidateReplica.Alias) { - // Seems like we promoted a candidate replica (though not in same DC and ENV as dead primary) - if satisfied, reason := PrimaryFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(candidateReplica)); satisfied { - // Good enough. No further action required. - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("promoted replica %+v is a good candidate", newPrimary.Alias)) - return newPrimary - } else { - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("skipping %+v; %s", candidateReplica.Alias, reason)) - } - } - } - - // Still nothing? - // Try a candidate replica that is in same DC & env as the promoted replica (our promoted replica is not an "is_candidate") - AuditTopologyRecovery(topologyRecovery, "+ searching for a candidate") - for _, candidateReplica := range preferredCandidates { - if canTakeOverPromotedServerAsPrimary(getInstanceFromTablet(candidateReplica), getInstanceFromTablet(newPrimary)) && - newPrimary.Alias.Cell == candidateReplica.Alias.Cell { - // OK, better than nothing - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("no candidate was offered for %+v but orchestrator picks %+v as candidate replacement, based on being in same DC & env as promoted instance", newPrimary.Alias, candidateReplica.Alias)) - return candidateReplica - } - } - - // Still nothing? - // Try a candidate replica (our promoted replica is not an "is_candidate") - AuditTopologyRecovery(topologyRecovery, "+ searching for a candidate") - for _, candidateReplica := range preferredCandidates { - if canTakeOverPromotedServerAsPrimary(getInstanceFromTablet(candidateReplica), getInstanceFromTablet(newPrimary)) { - if satisfied, reason := PrimaryFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(candidateReplica)); satisfied { - // OK, better than nothing - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("no candidate was offered for %+v but orchestrator picks %+v as candidate replacement", newPrimary.Alias, candidateReplica.Alias)) - return candidateReplica - } else { - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("skipping %+v; %s", candidateReplica.Alias, reason)) - } - } - } - - keepSearchingHint := "" - if satisfied, reason := PrimaryFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(newPrimary)); !satisfied { - keepSearchingHint = fmt.Sprintf("Will keep searching; %s", reason) - } else if reparentutil.PromotionRule(newPrimary) == reparentutil.PreferNotPromoteRule { - keepSearchingHint = fmt.Sprintf("Will keep searching because we have promoted a server with prefer_not rule: %+v", newPrimary.Alias) - } - if keepSearchingHint != "" { - AuditTopologyRecovery(topologyRecovery, keepSearchingHint) - // Still nothing? Then we didn't find a replica marked as "candidate". OK, further down the stream we have: - // find neutral instance in same dv&env as dead primary - if oldPrimary != nil { - AuditTopologyRecovery(topologyRecovery, "+ searching for a neutral server to replace promoted server, in same DC and env as dead master") - for _, neutralReplica := range neutralReplicas { - if canTakeOverPromotedServerAsPrimary(getInstanceFromTablet(neutralReplica), getInstanceFromTablet(newPrimary)) && - oldPrimary.Alias.Cell == neutralReplica.Alias.Cell { - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("no candidate was offered for %+v but orchestrator picks %+v as candidate replacement, based on being in same DC & env as dead master", newPrimary.Alias, neutralReplica.Alias)) - return neutralReplica - } - } - } - - // find neutral instance in same dv&env as promoted replica - AuditTopologyRecovery(topologyRecovery, "+ searching for a neutral server to replace promoted server, in same DC and env as promoted replica") - for _, neutralReplica := range neutralReplicas { - if canTakeOverPromotedServerAsPrimary(getInstanceFromTablet(neutralReplica), getInstanceFromTablet(newPrimary)) && - newPrimary.Alias.Cell == neutralReplica.Alias.Cell { - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("no candidate was offered for %+v but orchestrator picks %+v as candidate replacement, based on being in same DC & env as promoted instance", newPrimary.Alias, neutralReplica.Alias)) - return neutralReplica - } - } - - AuditTopologyRecovery(topologyRecovery, "+ searching for a neutral server to replace a prefer_not") - for _, neutralReplica := range neutralReplicas { - if canTakeOverPromotedServerAsPrimary(getInstanceFromTablet(neutralReplica), getInstanceFromTablet(newPrimary)) { - if satisfied, reason := PrimaryFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, getInstanceFromTablet(neutralReplica)); satisfied { - // OK, better than nothing - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("no candidate was offered for %+v but orchestrator picks %+v as candidate replacement, based on promoted instance having prefer_not promotion rule", newPrimary.Alias, neutralReplica.Alias)) - return neutralReplica - } else { - AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("skipping %+v; %s", neutralReplica.Alias, reason)) - } - } - } - } - - return newPrimary -} - -// TODO: handle error from this -func getInstanceFromTablet(tablet *topodatapb.Tablet) *inst.Instance { - instance, _, _ := inst.ReadInstance(&inst.InstanceKey{ - Hostname: tablet.MysqlHostname, - Port: int(tablet.MysqlPort), - }) - return instance -} - -// CheckIfNeedToOverridePromotion implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) CheckIfNeedToOverridePromotion(newPrimary *topodatapb.Tablet) error { - newPrimaryInstance := getInstanceFromTablet(newPrimary) - overridePrimaryPromotion := func() error { - // Scenarios where we might cancel the promotion. - if satisfied, reason := PrimaryFailoverGeographicConstraintSatisfied(&vtorcReparent.analysisEntry, newPrimaryInstance); !satisfied { - return fmt.Errorf("RecoverDeadPrimary: failed %+v promotion; %s", newPrimaryInstance.Key, reason) - } - if config.Config.FailPrimaryPromotionOnLagMinutes > 0 && - time.Duration(newPrimaryInstance.ReplicationLagSeconds.Int64)*time.Second >= time.Duration(config.Config.FailPrimaryPromotionOnLagMinutes)*time.Minute { - // candidate replica lags too much - return fmt.Errorf("RecoverDeadPrimary: failed promotion. FailPrimaryPromotionOnLagMinutes is set to %d (minutes) and promoted replica %+v 's lag is %d (seconds)", config.Config.FailPrimaryPromotionOnLagMinutes, newPrimaryInstance.Key, newPrimaryInstance.ReplicationLagSeconds.Int64) - } - if config.Config.FailPrimaryPromotionIfSQLThreadNotUpToDate && !newPrimaryInstance.SQLThreadUpToDate() { - return fmt.Errorf("RecoverDeadPrimary: failed promotion. FailPrimaryPromotionIfSQLThreadNotUpToDate is set and promoted replica %+v 's sql thread is not up to date (relay logs still unapplied). Aborting promotion", newPrimaryInstance.Key) - } - if config.Config.DelayPrimaryPromotionIfSQLThreadNotUpToDate && !newPrimaryInstance.SQLThreadUpToDate() { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("DelayPrimaryPromotionIfSQLThreadNotUpToDate: waiting for SQL thread on %+v", newPrimaryInstance.Key)) - if _, err := inst.WaitForSQLThreadUpToDate(&newPrimaryInstance.Key, 0, 0); err != nil { - return fmt.Errorf("DelayPrimaryPromotionIfSQLThreadNotUpToDate error: %+v", err) - } - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("DelayPrimaryPromotionIfSQLThreadNotUpToDate: SQL thread caught up on %+v", newPrimaryInstance.Key)) - } - // All seems well. No override done. - return nil - } - if err := overridePrimaryPromotion(); err != nil { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, err.Error()) - vtorcReparent.promotedReplica = nil - return err - } - return nil -} - // PostERSCompletionHook implements the ReparentFunctions interface func (vtorcReparent *VtOrcReparentFunctions) PostERSCompletionHook(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) { if vtorcReparent.promotedReplica != nil { diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 2861ef23098..e6420754b0b 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -660,13 +660,12 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat _, err = reparentutil.NewEmergencyReparenter(ts, tmclient.NewTabletManagerClient(), logutil.NewCallbackLogger(func(event *logutilpb.Event) { level := event.GetLevel() value := event.GetValue() + // we only log the warnings and errors explicitly, everything gets logged as an information message anyways in auditing topology recovery switch level { case logutilpb.Level_WARNING: log.Warningf("ERP - %s", value) case logutilpb.Level_ERROR: log.Errorf("ERP - %s", value) - default: - log.Infof("ERP - %s", value) } AuditTopologyRecovery(topologyRecovery, value) })).ReparentShard(context.Background(), tablet.Keyspace, tablet.Shard, reparentFunctions) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index e64f0ee7fdb..97d6880607e 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -208,9 +208,11 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve if err != nil { return err } + erp.logger.Infof("intermediate primary selected - %v", intermediatePrimary.Alias) // check weather the primary candidate selected is ideal or if it can be improved later - isIdeal := intermediateCandidateIsIdeal(intermediatePrimary, prevPrimary, validCandidateTablets, tabletMap, opts) + isIdeal := intermediateCandidateIsIdeal(erp.logger, intermediatePrimary, prevPrimary, validCandidateTablets, tabletMap, opts) + erp.logger.Infof("intermediate primary is ideal - %v", isIdeal) // Check (again) we still have the topology lock. if err = topo.CheckShardLocked(ctx, keyspace, shard); err != nil { @@ -230,7 +232,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // try to find a better candidate using the list we got back - betterCandidate := getBetterCandidate(intermediatePrimary, prevPrimary, validReplacementCandidates, tabletMap, opts) + betterCandidate := getBetterCandidate(erp.logger, intermediatePrimary, prevPrimary, validReplacementCandidates, tabletMap, opts) // if our better candidate is different from our previous candidate, then we wait for it to catch up to the intermediate primary if !topoproto.TabletAliasEqual(betterCandidate.Alias, intermediatePrimary.Alias) { @@ -272,6 +274,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } func (erp *EmergencyReparenter) promoteNewPrimary(ctx context.Context, ev *events.Reparent, newPrimary *topodatapb.Tablet, opts EmergencyReparentOptions, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus) error { + erp.logger.Infof("starting promotion for the new primary - %v", newPrimary.Alias) // we call PromoteReplica which changes the tablet type, fixes the semi-sync, set the primary to read-write and flushes the binlogs _, err := erp.tmc.PromoteReplica(ctx, newPrimary) if err != nil { diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index fbbe4c230fa..18946a14bf8 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -397,6 +397,7 @@ func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMa // findIntermediatePrimaryCandidate finds the intermediate primary candidate for ERS. We always choose the most advanced one from our valid candidates list func findIntermediatePrimaryCandidate(logger logutil.Logger, prevPrimary *topodatapb.Tablet, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (*topodatapb.Tablet, []*topodatapb.Tablet, error) { + logger.Infof("started finding the intermediate primary candidate") // convert the valid candidates into a list so that we can use it for sorting validTablets, tabletPositions, err := getValidCandidatesAndPositionsAsList(validCandidates, tabletMap) if err != nil { @@ -413,6 +414,9 @@ func findIntermediatePrimaryCandidate(logger logutil.Logger, prevPrimary *topoda if err != nil { return nil, nil, err } + for _, tablet := range validTablets { + logger.Infof("finding intermediate primary - sorted replica: %v", tablet.Alias) + } // The first tablet in the sorted list will be the most eligible candidate unless explicitly asked for some other tablet winningPrimaryTablet := validTablets[0] @@ -427,6 +431,8 @@ func findIntermediatePrimaryCandidate(logger logutil.Logger, prevPrimary *topoda if !ok { return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "requested primary elect %v has errant GTIDs", requestedPrimaryAlias) } + // if the requested tablet is as advanced as the most advanced tablet, then we can just use it for promotion. + // otherwise, we should let it catchup to the most advanced tablet and let it be the intermediate primary if pos.AtLeast(winningPosition) { requestedPrimaryInfo, isFound := tabletMap[requestedPrimaryAlias] if !isFound { @@ -455,13 +461,17 @@ func getValidCandidatesAndPositionsAsList(validCandidates map[string]mysql.Posit } // intermediateCandidateIsIdeal is used to find whether the intermediate candidate that ERS chose is also the ideal one or not -func intermediateCandidateIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) bool { +func intermediateCandidateIsIdeal(logger logutil.Logger, newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) bool { // we try to find a better candidate with the current list of valid candidates, and if it matches our current primary candidate, then we return true - return getBetterCandidate(newPrimary, prevPrimary, validCandidates, tabletMap, opts) == newPrimary + return getBetterCandidate(logger, newPrimary, prevPrimary, validCandidates, tabletMap, opts) == newPrimary } // getBetterCandidate is used to find a better candidate for ERS promotion -func getBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) *topodatapb.Tablet { +func getBetterCandidate(logger logutil.Logger, newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (candidate *topodatapb.Tablet) { + defer func() { + logger.Infof("found better candidate - %v", candidate.Alias) + }() + if opts.newPrimaryAlias != nil { // explicit request to promote a specific tablet requestedPrimaryAlias := topoproto.TabletAliasString(opts.newPrimaryAlias) @@ -483,33 +493,39 @@ func getBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandida } // So we've already promoted a replica. - // However, can we improve on our choice? Are there any replicas marked with "is_candidate"? + // However, can we improve on our choice? Are there any replicas with better promotion rules? // Maybe we actually promoted such a replica. Does that mean we should keep it? // Maybe we promoted a "neutral", and some "prefer" server is available. // Maybe we promoted a "prefer_not" - // Maybe we promoted a server in a different DC than the primary + // Maybe we promoted a server in a different cell than the primary // There's many options. We may wish to replace the server we promoted with a better one. - candidate := findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, true, true) + + // check whether the one we promoted is in the same cell and belongs to the preferred candidates list + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, true, true) if candidate != nil { return candidate } + // check whether there is some other tablet in the same cell belonging to the preferred candidates list candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, false, true) if candidate != nil { return candidate } - // do not have a preferred candidate in the same cell + // we do not have a preferred candidate in the same cell if !opts.preventCrossCellPromotion { + // check whether the one we promoted belongs to the preferred candidates list candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, true, false) if candidate != nil { return candidate } + // check whether there is some other tablet belonging to the preferred candidates list candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, false, false) if candidate != nil { return candidate } } + // repeat the same process for the neutral candidates list candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, true, true) if candidate != nil { return candidate @@ -530,6 +546,7 @@ func getBetterCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandida } } + // return the one that we have if nothing found return newPrimary } From ea56879fde75422f80c9d49bce0817976c77678a Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 23 Sep 2021 22:31:08 +0530 Subject: [PATCH 122/176] added post ers code to vtorc Signed-off-by: Manan Gupta --- .../orchestrator/logic/reparent_functions.go | 87 ------------------- go/vt/orchestrator/logic/topology_recovery.go | 73 ++++++++++++++-- .../reparentutil/emergency_reparenter.go | 6 +- 3 files changed, 73 insertions(+), 93 deletions(-) diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go index da3a40284e6..8afa16d407f 100644 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ b/go/vt/orchestrator/logic/reparent_functions.go @@ -15,90 +15,3 @@ limitations under the License. */ package logic - -import ( - "context" - "fmt" - - "vitess.io/vitess/go/vt/orchestrator/attributes" - "vitess.io/vitess/go/vt/orchestrator/kv" - "vitess.io/vitess/go/vt/vttablet/tmclient" - - "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/topotools/events" - - "vitess.io/vitess/go/vt/orchestrator/config" - - "vitess.io/vitess/go/vt/orchestrator/external/golib/log" - "vitess.io/vitess/go/vt/orchestrator/inst" -) - -// VtOrcReparentFunctions is the VtOrc implementation for ReparentFunctions -type VtOrcReparentFunctions struct { - analysisEntry inst.ReplicationAnalysis - candidateInstanceKey *inst.InstanceKey - skipProcesses bool - topologyRecovery *TopologyRecovery - promotedReplica *inst.Instance - recoveryAttempted bool - hasBestPromotionRule bool -} - -// PostERSCompletionHook implements the ReparentFunctions interface -func (vtorcReparent *VtOrcReparentFunctions) PostERSCompletionHook(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient) { - if vtorcReparent.promotedReplica != nil { - message := fmt.Sprintf("promoted replica: %+v", vtorcReparent.promotedReplica.Key) - AuditTopologyRecovery(vtorcReparent.topologyRecovery, message) - inst.AuditOperation("recover-dead-master", &vtorcReparent.analysisEntry.AnalyzedInstanceKey, message) - } - // And this is the end; whether successful or not, we're done. - resolveRecovery(vtorcReparent.topologyRecovery, vtorcReparent.promotedReplica) - // Now, see whether we are successful or not. From this point there's no going back. - if vtorcReparent.promotedReplica != nil { - // Success! - recoverDeadPrimarySuccessCounter.Inc(1) - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: successfully promoted %+v", vtorcReparent.promotedReplica.Key)) - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadPrimary: promoted server coordinates: %+v", vtorcReparent.promotedReplica.SelfBinlogCoordinates)) - - kvPairs := inst.GetClusterPrimaryKVPairs(vtorcReparent.analysisEntry.ClusterDetails.ClusterAlias, &vtorcReparent.promotedReplica.Key) - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("Writing KV %+v", kvPairs)) - for _, kvPair := range kvPairs { - err := kv.PutKVPair(kvPair) - log.Errore(err) - } - { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("Distributing KV %+v", kvPairs)) - err := kv.DistributePairs(kvPairs) - log.Errore(err) - } - if config.Config.PrimaryFailoverDetachReplicaPrimaryHost { - postponedFunction := func() error { - AuditTopologyRecovery(vtorcReparent.topologyRecovery, "- RecoverDeadPrimary: detaching master host on promoted master") - inst.DetachReplicaPrimaryHost(&vtorcReparent.promotedReplica.Key) - return nil - } - vtorcReparent.topologyRecovery.AddPostponedFunction(postponedFunction, fmt.Sprintf("RecoverDeadPrimary, detaching promoted master host %+v", vtorcReparent.promotedReplica.Key)) - } - func() error { - before := vtorcReparent.analysisEntry.AnalyzedInstanceKey.StringCode() - after := vtorcReparent.promotedReplica.Key.StringCode() - AuditTopologyRecovery(vtorcReparent.topologyRecovery, fmt.Sprintf("- RecoverDeadPrimary: updating cluster_alias: %v -> %v", before, after)) - //~~~inst.ReplaceClusterName(before, after) - if alias := vtorcReparent.analysisEntry.ClusterDetails.ClusterAlias; alias != "" { - inst.SetClusterAlias(vtorcReparent.promotedReplica.Key.StringCode(), alias) - } else { - inst.ReplaceAliasClusterName(before, after) - } - return nil - }() - - attributes.SetGeneralAttribute(vtorcReparent.analysisEntry.ClusterDetails.ClusterDomain, vtorcReparent.promotedReplica.Key.StringCode()) - - if !vtorcReparent.skipProcesses { - // Execute post master-failover processes - executeProcesses(config.Config.PostPrimaryFailoverProcesses, "PostPrimaryFailoverProcesses", vtorcReparent.topologyRecovery, false) - } - } else { - recoverDeadPrimaryFailureCounter.Inc(1) - } -} diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index e6420754b0b..2ac13d7aeda 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -28,6 +28,9 @@ import ( "sync/atomic" "time" + "vitess.io/vitess/go/vt/orchestrator/attributes" + "vitess.io/vitess/go/vt/orchestrator/kv" + "vitess.io/vitess/go/vt/logutil" logutilpb "vitess.io/vitess/go/vt/proto/logutil" @@ -187,8 +190,6 @@ func (this InstancesByCountReplicas) Less(i, j int) bool { return len(this[i].Replicas) < len(this[j].Replicas) } -var recoverDeadPrimarySuccessCounter = metrics.NewCounter() -var recoverDeadPrimaryFailureCounter = metrics.NewCounter() var recoverDeadIntermediatePrimaryCounter = metrics.NewCounter() var recoverDeadIntermediatePrimarySuccessCounter = metrics.NewCounter() var recoverDeadIntermediatePrimaryFailureCounter = metrics.NewCounter() @@ -198,8 +199,6 @@ var recoverDeadCoPrimaryFailureCounter = metrics.NewCounter() var countPendingRecoveriesGauge = metrics.NewGauge() func init() { - metrics.Register("recover.dead_primary.success", recoverDeadPrimarySuccessCounter) - metrics.Register("recover.dead_primary.fail", recoverDeadPrimaryFailureCounter) metrics.Register("recover.dead_intermediate_primary.start", recoverDeadIntermediatePrimaryCounter) metrics.Register("recover.dead_intermediate_primary.success", recoverDeadIntermediatePrimarySuccessCounter) metrics.Register("recover.dead_intermediate_primary.fail", recoverDeadIntermediatePrimaryFailureCounter) @@ -657,7 +656,7 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat // TODO: Fix durations reparentFunctions := reparentutil.NewEmergencyReparentOptions(candidateTabletAlias, nil, 1*time.Second, config.Config.PreventCrossDataCenterPrimaryFailover) - _, err = reparentutil.NewEmergencyReparenter(ts, tmclient.NewTabletManagerClient(), logutil.NewCallbackLogger(func(event *logutilpb.Event) { + ev, err := reparentutil.NewEmergencyReparenter(ts, tmclient.NewTabletManagerClient(), logutil.NewCallbackLogger(func(event *logutilpb.Event) { level := event.GetLevel() value := event.GetValue() // we only log the warnings and errors explicitly, everything gets logged as an information message anyways in auditing topology recovery @@ -670,10 +669,74 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat AuditTopologyRecovery(topologyRecovery, value) })).ReparentShard(context.Background(), tablet.Keyspace, tablet.Shard, reparentFunctions) + RefreshTablets() + var promotedReplica *inst.Instance + if ev.NewPrimary != nil { + promotedReplica, _, _ = inst.ReadInstance(&inst.InstanceKey{ + Hostname: ev.NewPrimary.MysqlHostname, + Port: int(ev.NewPrimary.MysqlPort), + }) + } + postErsCompletion(topologyRecovery, analysisEntry, skipProcesses, promotedReplica) + // TODO: fix recovery attempted return true, topologyRecovery, err } +func postErsCompletion(topologyRecovery *TopologyRecovery, analysisEntry inst.ReplicationAnalysis, skipProcesses bool, promotedReplica *inst.Instance) { + if promotedReplica != nil { + message := fmt.Sprintf("promoted replica: %+v", promotedReplica.Key) + AuditTopologyRecovery(topologyRecovery, message) + inst.AuditOperation("recover-dead-primary", &analysisEntry.AnalyzedInstanceKey, message) + } + // And this is the end; whether successful or not, we're done. + resolveRecovery(topologyRecovery, promotedReplica) + // Now, see whether we are successful or not. From this point there's no going back. + if promotedReplica != nil { + // Success! + AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("RecoverDeadPrimary: successfully promoted %+v", promotedReplica.Key)) + + kvPairs := inst.GetClusterPrimaryKVPairs(analysisEntry.ClusterDetails.ClusterAlias, &promotedReplica.Key) + AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("Writing KV %+v", kvPairs)) + for _, kvPair := range kvPairs { + err := kv.PutKVPair(kvPair) + log.Errore(err) + } + { + AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("Distributing KV %+v", kvPairs)) + err := kv.DistributePairs(kvPairs) + log.Errore(err) + } + if config.Config.PrimaryFailoverDetachReplicaPrimaryHost { + postponedFunction := func() error { + AuditTopologyRecovery(topologyRecovery, "- RecoverDeadPrimary: detaching primary host on promoted primary") + inst.DetachReplicaPrimaryHost(&promotedReplica.Key) + return nil + } + topologyRecovery.AddPostponedFunction(postponedFunction, fmt.Sprintf("RecoverDeadPrimary, detaching promoted primary host %+v", promotedReplica.Key)) + } + func() error { + before := analysisEntry.AnalyzedInstanceKey.StringCode() + after := promotedReplica.Key.StringCode() + AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("- RecoverDeadPrimary: updating cluster_alias: %v -> %v", before, after)) + //~~~inst.ReplaceClusterName(before, after) + if alias := analysisEntry.ClusterDetails.ClusterAlias; alias != "" { + inst.SetClusterAlias(promotedReplica.Key.StringCode(), alias) + } else { + inst.ReplaceAliasClusterName(before, after) + } + return nil + }() + + attributes.SetGeneralAttribute(analysisEntry.ClusterDetails.ClusterDomain, promotedReplica.Key.StringCode()) + + if !skipProcesses { + // Execute post primary-failover processes + executeProcesses(config.Config.PostPrimaryFailoverProcesses, "PostPrimaryFailoverProcesses", topologyRecovery, false) + } + } +} + // isGenerallyValidAsCandidateSiblingOfIntermediatePrimary sees that basic server configuration and state are valid func isGenerallyValidAsCandidateSiblingOfIntermediatePrimary(sibling *inst.Instance) bool { if !sibling.LogBinEnabled { diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 97d6880607e..0a4d5dfedfb 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -80,7 +80,9 @@ func NewEmergencyReparentOptions(newPrimaryAlias *topodatapb.TabletAlias, ignore // counters for Emergency Reparent Shard var ( - ersCounter = stats.NewGauge("ers_counter", "Number of time Emergency Reparent Shard has been run") + ersCounter = stats.NewGauge("ers_counter", "Number of times Emergency Reparent Shard has been run") + ersSuccessCounter = stats.NewGauge("ers_success_counter", "Number of times Emergency Reparent Shard has succeeded") + ersFailureCounter = stats.NewGauge("ers_failure_counter", "Number of times Emergency Reparent Shard has failed") ) // NewEmergencyReparenter returns a new EmergencyReparenter object, ready to @@ -121,8 +123,10 @@ func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, sha defer func() { switch err { case nil: + ersSuccessCounter.Add(1) event.DispatchUpdate(ev, "finished EmergencyReparentShard") default: + ersFailureCounter.Add(1) event.DispatchUpdate(ev, "failed EmergencyReparentShard: "+err.Error()) } }() From bd484e9a6b28b4dfa9e1ef85076dee728cde1ece Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 23 Sep 2021 22:31:44 +0530 Subject: [PATCH 123/176] remove reparent_functions file from login in vtorc Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/reparent_functions.go | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 go/vt/orchestrator/logic/reparent_functions.go diff --git a/go/vt/orchestrator/logic/reparent_functions.go b/go/vt/orchestrator/logic/reparent_functions.go deleted file mode 100644 index 8afa16d407f..00000000000 --- a/go/vt/orchestrator/logic/reparent_functions.go +++ /dev/null @@ -1,17 +0,0 @@ -/* -Copyright 2021 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package logic From 9e6cd966b1cccc0556ad2f49633932dc31333d03 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 23 Sep 2021 22:47:24 +0530 Subject: [PATCH 124/176] add test for ers counters Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter_test.go | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 5911920dba9..e174aebcf57 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -1823,3 +1823,155 @@ func TestEmergencyReparenter_waitForAllRelayLogsToApply(t *testing.T) { }) } } + +func TestEmergencyReparenterCounters(t *testing.T) { + ersCounter.Set(0) + ersSuccessCounter.Set(0) + ersFailureCounter.Set(0) + _ = SetDurabilityPolicy("none", nil) + + emergencyReparentOps := NewEmergencyReparentOptions(nil, nil, 0, false) + tmc := &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000102": nil, + }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000102": { + Result: "ok", + Error: nil, + }, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000102": { + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000100": nil, + "zone1-0000000101": nil, + }, + StopReplicationAndGetStatusResults: map[string]struct { + Status *replicationdatapb.Status + StopStatus *replicationdatapb.StopReplicationStatus + Error error + }{ + "zone1-0000000100": { + StopStatus: &replicationdatapb.StopReplicationStatus{ + Before: &replicationdatapb.Status{}, + After: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", + }, + }, + }, + "zone1-0000000101": { + StopStatus: &replicationdatapb.StopReplicationStatus{ + Before: &replicationdatapb.Status{}, + After: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", + }, + }, + }, + "zone1-0000000102": { + StopStatus: &replicationdatapb.StopReplicationStatus{ + Before: &replicationdatapb.Status{}, + After: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-26", + }, + }, + }, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000100": { + "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21": nil, + }, + "zone1-0000000101": { + "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21": nil, + }, + "zone1-0000000102": { + "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-26": nil, + }, + }, + } + shards := []*vtctldatapb.Shard{ + { + Keyspace: "testkeyspace", + Name: "-", + }, + } + tablets := []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Keyspace: "testkeyspace", + Shard: "-", + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + Keyspace: "testkeyspace", + Shard: "-", + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Keyspace: "testkeyspace", + Shard: "-", + Hostname: "most up-to-date position, wins election", + }, + } + keyspace := "testkeyspace" + shard := "-" + ts := memorytopo.NewServer("zone1") + + ctx := context.Background() + logger := logutil.NewMemoryLogger() + + for i, tablet := range tablets { + tablet.Type = topodatapb.TabletType_REPLICA + tablets[i] = tablet + } + + testutil.AddShards(ctx, t, ts, shards...) + testutil.AddTablets(ctx, t, ts, nil, tablets...) + + erp := NewEmergencyReparenter(ts, tmc, logger) + + // run a successful ers + _, err := erp.ReparentShard(ctx, keyspace, shard, emergencyReparentOps) + require.NoError(t, err) + + // check the counter values + require.EqualValues(t, 1, ersCounter.Get()) + require.EqualValues(t, 1, ersSuccessCounter.Get()) + require.EqualValues(t, 0, ersFailureCounter.Get()) + + // set emergencyReparentOps to request a non existent tablet + emergencyReparentOps.newPrimaryAlias = &topodatapb.TabletAlias{ + Cell: "bogus", + Uid: 100, + } + + // run a failing ers + _, err = erp.ReparentShard(ctx, keyspace, shard, emergencyReparentOps) + require.Error(t, err) + + // check the counter values + require.EqualValues(t, 2, ersCounter.Get()) + require.EqualValues(t, 1, ersSuccessCounter.Get()) + require.EqualValues(t, 1, ersFailureCounter.Get()) +} From 9416f392c74c83b7b0f35feb7227e3dffe087c99 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 23 Sep 2021 22:58:32 +0530 Subject: [PATCH 125/176] add test for checking constraint satisfaction Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/util_test.go | 88 +++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index 5cff5044423..7135bf570a1 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -21,6 +21,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/test/utils" "github.com/stretchr/testify/assert" @@ -513,3 +515,89 @@ func TestFindCurrentPrimary(t *testing.T) { }) } } + +func TestCheckIfConstraintsSatisfied(t *testing.T) { + testcases := []struct { + name string + newPrimary, prevPrimary *topodatapb.Tablet + opts EmergencyReparentOptions + err string + }{ + { + name: "no constraint failure", + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + }, + Type: topodatapb.TabletType_REPLICA, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + }, + }, + opts: EmergencyReparentOptions{preventCrossCellPromotion: true}, + err: "", + }, { + name: "promotion rule constraint failure", + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + }, + }, + opts: EmergencyReparentOptions{preventCrossCellPromotion: true}, + err: "elected primary does not satisfy promotion rule constraint - cell1-0000000100", + }, { + name: "cross cell constraint failure", + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + Uid: 100, + }, + Type: topodatapb.TabletType_REPLICA, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell2", + }, + }, + opts: EmergencyReparentOptions{preventCrossCellPromotion: true}, + err: "elected primary does not satisfy geographic constraint - cell1-0000000100", + }, { + name: "cross cell but no constraint failure", + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + Uid: 100, + }, + Type: topodatapb.TabletType_REPLICA, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell2", + }, + }, + opts: EmergencyReparentOptions{preventCrossCellPromotion: false}, + err: "", + }, + } + + _ = SetDurabilityPolicy("none", nil) + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + err := checkIfConstraintsSatisfied(testcase.newPrimary, testcase.prevPrimary, testcase.opts) + if testcase.err == "" { + require.NoError(t, err) + } else { + require.EqualError(t, err, testcase.err) + } + }) + } +} From 5da378b30f40d8447d7202c5048c88b78cda3c62 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 00:01:51 +0530 Subject: [PATCH 126/176] fix wrangler test Signed-off-by: Manan Gupta --- .../fakemysqldaemon/fakemysqldaemon.go | 24 +++--- .../endtoend/init_shard_primary_test.go | 8 +- go/vt/wrangler/testlib/backup_test.go | 8 +- .../testlib/emergency_reparent_shard_test.go | 35 ++++++--- .../testlib/external_reparent_test.go | 12 +-- .../testlib/planned_reparent_shard_test.go | 74 +++++++++---------- go/vt/wrangler/testlib/reparent_utils_test.go | 2 +- 7 files changed, 91 insertions(+), 72 deletions(-) diff --git a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go index 7f5701c1adf..bae45a83e18 100644 --- a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go +++ b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go @@ -109,16 +109,16 @@ type FakeMysqlDaemon struct { // StartReplicationUntilAfterPos is matched against the input StartReplicationUntilAfterPos mysql.Position - // SetReplicationSourceInput is matched against the input of SetReplicationSource - // (as "%v:%v"). If it doesn't match, SetReplicationSource will return an error. - SetReplicationSourceInput string + // SetReplicationSourceInputs are matched against the input of SetReplicationSource + // (as "%v:%v"). If all of them don't match, SetReplicationSource will return an error. + SetReplicationSourceInputs []string // SetReplicationSourceError is used by SetReplicationSource SetReplicationSourceError error - // WaitPrimaryPosition is checked by WaitSourcePos, if the + // WaitPrimaryPositions is checked by WaitSourcePos, if the // same it returns nil, if different it returns an error - WaitPrimaryPosition mysql.Position + WaitPrimaryPositions []mysql.Position // PromoteResult is returned by Promote PromoteResult mysql.Position @@ -376,8 +376,10 @@ func (fmd *FakeMysqlDaemon) SetReplicationPosition(ctx context.Context, pos mysq // SetReplicationSource is part of the MysqlDaemon interface. func (fmd *FakeMysqlDaemon) SetReplicationSource(ctx context.Context, host string, port int, stopReplicationBefore bool, startReplicationAfter bool) error { input := fmt.Sprintf("%v:%v", host, port) - if fmd.SetReplicationSourceInput != input { - return fmt.Errorf("wrong input for SetReplicationSourceCommands: expected %v got %v", fmd.SetReplicationSourceInput, input) + for _, sourceInput := range fmd.SetReplicationSourceInputs { + if sourceInput != input { + return fmt.Errorf("wrong input for SetReplicationSourceCommands: expected a value in %v got %v", fmd.SetReplicationSourceInputs, input) + } } if fmd.SetReplicationSourceError != nil { return fmd.SetReplicationSourceError @@ -408,10 +410,12 @@ func (fmd *FakeMysqlDaemon) WaitSourcePos(_ context.Context, pos mysql.Position) if fmd.TimeoutHook != nil { return fmd.TimeoutHook() } - if reflect.DeepEqual(fmd.WaitPrimaryPosition, pos) { - return nil + for _, position := range fmd.WaitPrimaryPositions { + if reflect.DeepEqual(position, pos) { + return nil + } } - return fmt.Errorf("wrong input for WaitSourcePos: expected %v got %v", fmd.WaitPrimaryPosition, pos) + return fmt.Errorf("wrong input for WaitSourcePos: expected a value in %v got %v", fmd.WaitPrimaryPositions, pos) } // Promote is part of the MysqlDaemon interface diff --git a/go/vt/vtctl/grpcvtctldserver/endtoend/init_shard_primary_test.go b/go/vt/vtctl/grpcvtctldserver/endtoend/init_shard_primary_test.go index 37627e1c68b..98bc07e5d40 100644 --- a/go/vt/vtctl/grpcvtctldserver/endtoend/init_shard_primary_test.go +++ b/go/vt/vtctl/grpcvtctldserver/endtoend/init_shard_primary_test.go @@ -66,7 +66,7 @@ func TestInitShardPrimary(t *testing.T) { "FAKE SET MASTER", "START SLAVE", } - tablet2.FakeMysqlDaemon.SetReplicationSourceInput = fmt.Sprintf("%v:%v", tablet1.Tablet.Hostname, tablet1.Tablet.MysqlPort) + tablet2.FakeMysqlDaemon.SetReplicationSourceInputs = append(tablet2.FakeMysqlDaemon.SetReplicationSourceInputs, fmt.Sprintf("%v:%v", tablet1.Tablet.Hostname, tablet1.Tablet.MysqlPort)) tablet3.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE RESET ALL REPLICATION", @@ -75,7 +75,7 @@ func TestInitShardPrimary(t *testing.T) { "FAKE SET MASTER", "START SLAVE", } - tablet3.FakeMysqlDaemon.SetReplicationSourceInput = fmt.Sprintf("%v:%v", tablet1.Tablet.Hostname, tablet1.Tablet.MysqlPort) + tablet3.FakeMysqlDaemon.SetReplicationSourceInputs = append(tablet3.FakeMysqlDaemon.SetReplicationSourceInputs, fmt.Sprintf("%v:%v", tablet1.Tablet.Hostname, tablet1.Tablet.MysqlPort)) for _, tablet := range []*testlib.FakeTablet{tablet1, tablet2, tablet3} { tablet.StartActionLoop(t, wr) @@ -122,7 +122,7 @@ func TestInitShardPrimaryNoFormerPrimary(t *testing.T) { "FAKE SET MASTER", "START SLAVE", } - tablet2.FakeMysqlDaemon.SetReplicationSourceInput = fmt.Sprintf("%v:%v", tablet1.Tablet.Hostname, tablet1.Tablet.MysqlPort) + tablet2.FakeMysqlDaemon.SetReplicationSourceInputs = append(tablet2.FakeMysqlDaemon.SetReplicationSourceInputs, fmt.Sprintf("%v:%v", tablet1.Tablet.Hostname, tablet1.Tablet.MysqlPort)) tablet3.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE RESET ALL REPLICATION", @@ -130,7 +130,7 @@ func TestInitShardPrimaryNoFormerPrimary(t *testing.T) { "FAKE SET MASTER", "START SLAVE", } - tablet3.FakeMysqlDaemon.SetReplicationSourceInput = fmt.Sprintf("%v:%v", tablet1.Tablet.Hostname, tablet1.Tablet.MysqlPort) + tablet3.FakeMysqlDaemon.SetReplicationSourceInputs = append(tablet3.FakeMysqlDaemon.SetReplicationSourceInputs, fmt.Sprintf("%v:%v", tablet1.Tablet.Hostname, tablet1.Tablet.MysqlPort)) for _, tablet := range []*testlib.FakeTablet{tablet1, tablet2, tablet3} { tablet.StartActionLoop(t, wr) diff --git a/go/vt/wrangler/testlib/backup_test.go b/go/vt/wrangler/testlib/backup_test.go index 2b1eef31315..0baefaf68af 100644 --- a/go/vt/wrangler/testlib/backup_test.go +++ b/go/vt/wrangler/testlib/backup_test.go @@ -175,7 +175,7 @@ func TestBackupRestore(t *testing.T) { "SHOW DATABASES": {}, } destTablet.FakeMysqlDaemon.SetReplicationPositionPos = sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition - destTablet.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(primary.Tablet) + destTablet.FakeMysqlDaemon.SetReplicationSourceInputs = append(destTablet.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet)) destTablet.StartActionLoop(t, wr) defer destTablet.StopActionLoop(t) @@ -399,7 +399,7 @@ func TestBackupRestoreLagged(t *testing.T) { "SHOW DATABASES": {}, } destTablet.FakeMysqlDaemon.SetReplicationPositionPos = destTablet.FakeMysqlDaemon.CurrentPrimaryPosition - destTablet.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(primary.Tablet) + destTablet.FakeMysqlDaemon.SetReplicationSourceInputs = append(destTablet.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet)) destTablet.StartActionLoop(t, wr) defer destTablet.StopActionLoop(t) @@ -567,7 +567,7 @@ func TestRestoreUnreachablePrimary(t *testing.T) { "SHOW DATABASES": {}, } destTablet.FakeMysqlDaemon.SetReplicationPositionPos = sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition - destTablet.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(primary.Tablet) + destTablet.FakeMysqlDaemon.SetReplicationSourceInputs = append(destTablet.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet)) destTablet.StartActionLoop(t, wr) defer destTablet.StopActionLoop(t) @@ -724,7 +724,7 @@ func TestDisableActiveReparents(t *testing.T) { "SHOW DATABASES": {}, } destTablet.FakeMysqlDaemon.SetReplicationPositionPos = sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition - destTablet.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(primary.Tablet) + destTablet.FakeMysqlDaemon.SetReplicationSourceInputs = append(destTablet.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet)) destTablet.StartActionLoop(t, wr) defer destTablet.StopActionLoop(t) diff --git a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go index 77668a7a5a1..c1491d3b189 100644 --- a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go @@ -21,6 +21,8 @@ import ( "testing" "time" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/discovery" "github.com/stretchr/testify/assert" @@ -43,6 +45,7 @@ func TestEmergencyReparentShard(t *testing.T) { discovery.SetTabletPickerRetryDelay(delay) }() discovery.SetTabletPickerRetryDelay(5 * time.Millisecond) + _ = reparentutil.SetDurabilityPolicy("none", nil) ts := memorytopo.NewServer("cell1", "cell2") wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) @@ -86,7 +89,7 @@ func TestEmergencyReparentShard(t *testing.T) { newPrimary.FakeMysqlDaemon.CurrentSourceFilePosition = mysql.Position{ GTIDSet: newPrimaryRelayLogPos, } - newPrimary.FakeMysqlDaemon.WaitPrimaryPosition = newPrimary.FakeMysqlDaemon.CurrentSourceFilePosition + newPrimary.FakeMysqlDaemon.WaitPrimaryPositions = append(newPrimary.FakeMysqlDaemon.WaitPrimaryPositions, newPrimary.FakeMysqlDaemon.CurrentSourceFilePosition) newPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE IO_THREAD", "CREATE DATABASE IF NOT EXISTS _vt", @@ -108,7 +111,7 @@ func TestEmergencyReparentShard(t *testing.T) { // old primary, will be scrapped oldPrimary.FakeMysqlDaemon.ReadOnly = false oldPrimary.FakeMysqlDaemon.ReplicationStatusError = mysql.ErrNotReplica - oldPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", } @@ -131,8 +134,8 @@ func TestEmergencyReparentShard(t *testing.T) { goodReplica1.FakeMysqlDaemon.CurrentSourceFilePosition = mysql.Position{ GTIDSet: goodReplica1RelayLogPos, } - goodReplica1.FakeMysqlDaemon.WaitPrimaryPosition = goodReplica1.FakeMysqlDaemon.CurrentSourceFilePosition - goodReplica1.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + goodReplica1.FakeMysqlDaemon.WaitPrimaryPositions = append(goodReplica1.FakeMysqlDaemon.WaitPrimaryPositions, goodReplica1.FakeMysqlDaemon.CurrentSourceFilePosition) + goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE IO_THREAD", "STOP SLAVE", @@ -158,8 +161,8 @@ func TestEmergencyReparentShard(t *testing.T) { goodReplica2.FakeMysqlDaemon.CurrentSourceFilePosition = mysql.Position{ GTIDSet: goodReplica2RelayLogPos, } - goodReplica2.FakeMysqlDaemon.WaitPrimaryPosition = goodReplica2.FakeMysqlDaemon.CurrentSourceFilePosition - goodReplica2.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + goodReplica2.FakeMysqlDaemon.WaitPrimaryPositions = append(goodReplica2.FakeMysqlDaemon.WaitPrimaryPositions, goodReplica2.FakeMysqlDaemon.CurrentSourceFilePosition) + goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) goodReplica2.StartActionLoop(t, wr) goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", @@ -191,6 +194,7 @@ func TestEmergencyReparentShardPrimaryElectNotBest(t *testing.T) { discovery.SetTabletPickerRetryDelay(delay) }() discovery.SetTabletPickerRetryDelay(5 * time.Millisecond) + _ = reparentutil.SetDurabilityPolicy("none", nil) ts := memorytopo.NewServer("cell1", "cell2") wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) @@ -227,9 +231,16 @@ func TestEmergencyReparentShardPrimaryElectNotBest(t *testing.T) { newPrimary.FakeMysqlDaemon.CurrentSourceFilePosition = mysql.Position{ GTIDSet: newPrimaryRelayLogPos, } - newPrimary.FakeMysqlDaemon.WaitPrimaryPosition = newPrimary.FakeMysqlDaemon.CurrentSourceFilePosition + newPrimary.FakeMysqlDaemon.WaitPrimaryPositions = append(newPrimary.FakeMysqlDaemon.WaitPrimaryPositions, newPrimary.FakeMysqlDaemon.CurrentSourceFilePosition) + newPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(newPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(moreAdvancedReplica.Tablet)) newPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE IO_THREAD", + "STOP SLAVE", + "FAKE SET MASTER", + "START SLAVE", + "CREATE DATABASE IF NOT EXISTS _vt", + "SUBCREATE TABLE IF NOT EXISTS _vt.reparent_journal", + "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, master_alias, replication_position) VALUES", } newPrimary.StartActionLoop(t, wr) defer newPrimary.StopActionLoop(t) @@ -265,9 +276,14 @@ func TestEmergencyReparentShardPrimaryElectNotBest(t *testing.T) { moreAdvancedReplica.FakeMysqlDaemon.CurrentSourceFilePosition = mysql.Position{ GTIDSet: moreAdvancedReplicaLogPos, } - moreAdvancedReplica.FakeMysqlDaemon.WaitPrimaryPosition = moreAdvancedReplica.FakeMysqlDaemon.CurrentSourceFilePosition + moreAdvancedReplica.FakeMysqlDaemon.SetReplicationSourceInputs = append(moreAdvancedReplica.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) + moreAdvancedReplica.FakeMysqlDaemon.WaitPrimaryPositions = append(moreAdvancedReplica.FakeMysqlDaemon.WaitPrimaryPositions, moreAdvancedReplica.FakeMysqlDaemon.CurrentSourceFilePosition) + newPrimary.FakeMysqlDaemon.WaitPrimaryPositions = append(newPrimary.FakeMysqlDaemon.WaitPrimaryPositions, moreAdvancedReplica.FakeMysqlDaemon.CurrentPrimaryPosition) moreAdvancedReplica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE IO_THREAD", + "STOP SLAVE", + "FAKE SET MASTER", + "START SLAVE", } moreAdvancedReplica.StartActionLoop(t, wr) defer moreAdvancedReplica.StopActionLoop(t) @@ -276,8 +292,7 @@ func TestEmergencyReparentShardPrimaryElectNotBest(t *testing.T) { err := wr.EmergencyReparentShard(ctx, newPrimary.Tablet.Keyspace, newPrimary.Tablet.Shard, newPrimary.Tablet.Alias, 10*time.Second, sets.NewString(), false) cancel() - assert.Error(t, err) - assert.Contains(t, err.Error(), "is not fully caught up") + assert.NoError(t, err) // check what was run err = newPrimary.FakeMysqlDaemon.CheckSuperQueryList() require.NoError(t, err) diff --git a/go/vt/wrangler/testlib/external_reparent_test.go b/go/vt/wrangler/testlib/external_reparent_test.go index fede11b2ff9..35273cf91ca 100644 --- a/go/vt/wrangler/testlib/external_reparent_test.go +++ b/go/vt/wrangler/testlib/external_reparent_test.go @@ -89,7 +89,7 @@ func TestTabletExternallyReparentedBasic(t *testing.T) { t.Fatalf("old primary should be PRIMARY but is: %v", tablet.Type) } - oldPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", "START Replica", @@ -168,7 +168,7 @@ func TestTabletExternallyReparentedToReplica(t *testing.T) { // Second test: reparent to a replica, and pretend the old // primary is still good to go. - oldPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", "START Replica", @@ -246,7 +246,7 @@ func TestTabletExternallyReparentedWithDifferentMysqlPort(t *testing.T) { newPrimary.StartActionLoop(t, wr) defer newPrimary.StopActionLoop(t) - oldPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", "START Replica", @@ -327,7 +327,7 @@ func TestTabletExternallyReparentedContinueOnUnexpectedPrimary(t *testing.T) { newPrimary.StartActionLoop(t, wr) defer newPrimary.StopActionLoop(t) - oldPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", "START Replica", @@ -404,7 +404,7 @@ func TestTabletExternallyReparentedRerun(t *testing.T) { newPrimary.StartActionLoop(t, wr) defer newPrimary.StopActionLoop(t) - oldPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", "START Replica", @@ -414,7 +414,7 @@ func TestTabletExternallyReparentedRerun(t *testing.T) { oldPrimary.StartActionLoop(t, wr) defer oldPrimary.StopActionLoop(t) - goodReplica.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + goodReplica.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) // On the good replica, we will respond to // TabletActionReplicaWasRestarted. goodReplica.StartActionLoop(t, wr) diff --git a/go/vt/wrangler/testlib/planned_reparent_shard_test.go b/go/vt/wrangler/testlib/planned_reparent_shard_test.go index da4035cdbfd..e1b663e3855 100644 --- a/go/vt/wrangler/testlib/planned_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/planned_reparent_shard_test.go @@ -58,7 +58,7 @@ func TestPlannedReparentShardNoPrimaryProvided(t *testing.T) { // new primary newPrimary.FakeMysqlDaemon.ReadOnly = true newPrimary.FakeMysqlDaemon.Replicating = true - newPrimary.FakeMysqlDaemon.WaitPrimaryPosition = mysql.Position{ + newPrimary.FakeMysqlDaemon.WaitPrimaryPositions = []mysql.Position{{ GTIDSet: mysql.MariadbGTIDSet{ 7: mysql.MariadbGTID{ Domain: 7, @@ -66,7 +66,7 @@ func TestPlannedReparentShardNoPrimaryProvided(t *testing.T) { Sequence: 990, }, }, - } + }} newPrimary.FakeMysqlDaemon.PromoteResult = mysql.Position{ GTIDSet: mysql.MariadbGTIDSet{ 7: mysql.MariadbGTID{ @@ -91,8 +91,8 @@ func TestPlannedReparentShardNoPrimaryProvided(t *testing.T) { oldPrimary.FakeMysqlDaemon.ReadOnly = false oldPrimary.FakeMysqlDaemon.Replicating = false oldPrimary.FakeMysqlDaemon.ReplicationStatusError = mysql.ErrNotReplica - oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPosition - oldPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0] + oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", "START SLAVE", @@ -105,12 +105,12 @@ func TestPlannedReparentShardNoPrimaryProvided(t *testing.T) { oldPrimary.TM.QueryServiceControl.(*tabletservermock.Controller).SetQueryServiceEnabledForTests(true) // SetReplicationSource is called on new primary to make sure it's replicating before reparenting. - newPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(oldPrimary.Tablet) + newPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(newPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(oldPrimary.Tablet)) // good replica 1 is replicating goodReplica1.FakeMysqlDaemon.ReadOnly = true goodReplica1.FakeMysqlDaemon.Replicating = true - goodReplica1.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", "FAKE SET MASTER", @@ -168,7 +168,7 @@ func TestPlannedReparentShardNoError(t *testing.T) { // new primary newPrimary.FakeMysqlDaemon.ReadOnly = true newPrimary.FakeMysqlDaemon.Replicating = true - newPrimary.FakeMysqlDaemon.WaitPrimaryPosition = mysql.Position{ + newPrimary.FakeMysqlDaemon.WaitPrimaryPositions = []mysql.Position{{ GTIDSet: mysql.MariadbGTIDSet{ 7: mysql.MariadbGTID{ Domain: 7, @@ -176,7 +176,7 @@ func TestPlannedReparentShardNoError(t *testing.T) { Sequence: 990, }, }, - } + }} newPrimary.FakeMysqlDaemon.PromoteResult = mysql.Position{ GTIDSet: mysql.MariadbGTIDSet{ 7: mysql.MariadbGTID{ @@ -201,8 +201,8 @@ func TestPlannedReparentShardNoError(t *testing.T) { oldPrimary.FakeMysqlDaemon.ReadOnly = false oldPrimary.FakeMysqlDaemon.Replicating = false oldPrimary.FakeMysqlDaemon.ReplicationStatusError = mysql.ErrNotReplica - oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPosition - oldPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0] + oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", "START SLAVE", @@ -215,12 +215,12 @@ func TestPlannedReparentShardNoError(t *testing.T) { oldPrimary.TM.QueryServiceControl.(*tabletservermock.Controller).SetQueryServiceEnabledForTests(true) // SetReplicationSource is called on new primary to make sure it's replicating before reparenting. - newPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(oldPrimary.Tablet) + newPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(newPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(oldPrimary.Tablet)) // goodReplica1 is replicating goodReplica1.FakeMysqlDaemon.ReadOnly = true goodReplica1.FakeMysqlDaemon.Replicating = true - goodReplica1.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", "FAKE SET MASTER", @@ -232,7 +232,7 @@ func TestPlannedReparentShardNoError(t *testing.T) { // goodReplica2 is not replicating goodReplica2.FakeMysqlDaemon.ReadOnly = true goodReplica2.FakeMysqlDaemon.Replicating = false - goodReplica2.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) goodReplica2.StartActionLoop(t, wr) goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", @@ -317,7 +317,7 @@ func TestPlannedReparentShardWaitForPositionFail(t *testing.T) { // new primary newPrimary.FakeMysqlDaemon.ReadOnly = true newPrimary.FakeMysqlDaemon.Replicating = true - newPrimary.FakeMysqlDaemon.WaitPrimaryPosition = mysql.Position{ + newPrimary.FakeMysqlDaemon.WaitPrimaryPositions = []mysql.Position{{ GTIDSet: mysql.MariadbGTIDSet{ 7: mysql.MariadbGTID{ Domain: 7, @@ -325,7 +325,7 @@ func TestPlannedReparentShardWaitForPositionFail(t *testing.T) { Sequence: 990, }, }, - } + }} newPrimary.FakeMysqlDaemon.PromoteResult = mysql.Position{ GTIDSet: mysql.MariadbGTIDSet{ 7: mysql.MariadbGTID{ @@ -351,7 +351,7 @@ func TestPlannedReparentShardWaitForPositionFail(t *testing.T) { oldPrimary.FakeMysqlDaemon.Replicating = false // set to incorrect value to make promote fail on WaitForReplicationPos oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.PromoteResult - oldPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", "START SLAVE", @@ -360,12 +360,12 @@ func TestPlannedReparentShardWaitForPositionFail(t *testing.T) { defer oldPrimary.StopActionLoop(t) oldPrimary.TM.QueryServiceControl.(*tabletservermock.Controller).SetQueryServiceEnabledForTests(true) // SetReplicationSource is called on new primary to make sure it's replicating before reparenting. - newPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(oldPrimary.Tablet) + newPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(newPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(oldPrimary.Tablet)) // good replica 1 is replicating goodReplica1.FakeMysqlDaemon.ReadOnly = true goodReplica1.FakeMysqlDaemon.Replicating = true - goodReplica1.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", "FAKE SET MASTER", @@ -377,7 +377,7 @@ func TestPlannedReparentShardWaitForPositionFail(t *testing.T) { // good replica 2 is not replicating goodReplica2.FakeMysqlDaemon.ReadOnly = true goodReplica2.FakeMysqlDaemon.Replicating = false - goodReplica2.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) goodReplica2.StartActionLoop(t, wr) goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", @@ -418,7 +418,7 @@ func TestPlannedReparentShardWaitForPositionTimeout(t *testing.T) { newPrimary.FakeMysqlDaemon.TimeoutHook = func() error { return context.DeadlineExceeded } newPrimary.FakeMysqlDaemon.ReadOnly = true newPrimary.FakeMysqlDaemon.Replicating = true - newPrimary.FakeMysqlDaemon.WaitPrimaryPosition = mysql.Position{ + newPrimary.FakeMysqlDaemon.WaitPrimaryPositions = []mysql.Position{{ GTIDSet: mysql.MariadbGTIDSet{ 7: mysql.MariadbGTID{ Domain: 7, @@ -426,7 +426,7 @@ func TestPlannedReparentShardWaitForPositionTimeout(t *testing.T) { Sequence: 990, }, }, - } + }} newPrimary.FakeMysqlDaemon.PromoteResult = mysql.Position{ GTIDSet: mysql.MariadbGTIDSet{ 7: mysql.MariadbGTID{ @@ -450,8 +450,8 @@ func TestPlannedReparentShardWaitForPositionTimeout(t *testing.T) { // old primary oldPrimary.FakeMysqlDaemon.ReadOnly = false oldPrimary.FakeMysqlDaemon.Replicating = false - oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPosition - oldPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0] + oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", "START SLAVE", @@ -461,11 +461,11 @@ func TestPlannedReparentShardWaitForPositionTimeout(t *testing.T) { oldPrimary.TM.QueryServiceControl.(*tabletservermock.Controller).SetQueryServiceEnabledForTests(true) // SetReplicationSource is called on new primary to make sure it's replicating before reparenting. - newPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(oldPrimary.Tablet) + newPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(newPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(oldPrimary.Tablet)) // good replica 1 is replicating goodReplica1.FakeMysqlDaemon.ReadOnly = true goodReplica1.FakeMysqlDaemon.Replicating = true - goodReplica1.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", "FAKE SET MASTER", @@ -477,7 +477,7 @@ func TestPlannedReparentShardWaitForPositionTimeout(t *testing.T) { // good replica 2 is not replicating goodReplica2.FakeMysqlDaemon.ReadOnly = true goodReplica2.FakeMysqlDaemon.Replicating = false - goodReplica2.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) goodReplica2.StartActionLoop(t, wr) goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", @@ -536,7 +536,7 @@ func TestPlannedReparentShardRelayLogError(t *testing.T) { // goodReplica1 is replicating goodReplica1.FakeMysqlDaemon.ReadOnly = true goodReplica1.FakeMysqlDaemon.Replicating = true - goodReplica1.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(primary.Tablet) + goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet)) // simulate error that will trigger a call to RestartReplication goodReplica1.FakeMysqlDaemon.SetReplicationSourceError = errors.New("Slave failed to initialize relay log info structure from the repository") goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ @@ -612,7 +612,7 @@ func TestPlannedReparentShardRelayLogErrorStartReplication(t *testing.T) { goodReplica1.FakeMysqlDaemon.ReadOnly = true goodReplica1.FakeMysqlDaemon.Replicating = true goodReplica1.FakeMysqlDaemon.IOThreadRunning = false - goodReplica1.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(primary.Tablet) + goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet)) goodReplica1.FakeMysqlDaemon.CurrentSourceHost = primary.Tablet.MysqlHostname goodReplica1.FakeMysqlDaemon.CurrentSourcePort = int(primary.Tablet.MysqlPort) // simulate error that will trigger a call to RestartReplication @@ -670,7 +670,7 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { newPrimary.FakeMysqlDaemon.Replicating = true // make promote fail newPrimary.FakeMysqlDaemon.PromoteError = errors.New("some error") - newPrimary.FakeMysqlDaemon.WaitPrimaryPosition = mysql.Position{ + newPrimary.FakeMysqlDaemon.WaitPrimaryPositions = []mysql.Position{{ GTIDSet: mysql.MariadbGTIDSet{ 7: mysql.MariadbGTID{ Domain: 7, @@ -678,7 +678,7 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { Sequence: 990, }, }, - } + }} newPrimary.FakeMysqlDaemon.PromoteResult = mysql.Position{ GTIDSet: mysql.MariadbGTIDSet{ 7: mysql.MariadbGTID{ @@ -703,8 +703,8 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { oldPrimary.FakeMysqlDaemon.ReadOnly = false oldPrimary.FakeMysqlDaemon.Replicating = false oldPrimary.FakeMysqlDaemon.ReplicationStatusError = mysql.ErrNotReplica - oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPosition - oldPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0] + oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", "START SLAVE", @@ -714,11 +714,11 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { oldPrimary.TM.QueryServiceControl.(*tabletservermock.Controller).SetQueryServiceEnabledForTests(true) // SetReplicationSource is called on new primary to make sure it's replicating before reparenting. - newPrimary.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(oldPrimary.Tablet) + newPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(newPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(oldPrimary.Tablet)) // good replica 1 is replicating goodReplica1.FakeMysqlDaemon.ReadOnly = true goodReplica1.FakeMysqlDaemon.Replicating = true - goodReplica1.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", "FAKE SET MASTER", @@ -730,7 +730,7 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { // good replica 2 is not replicating goodReplica2.FakeMysqlDaemon.ReadOnly = true goodReplica2.FakeMysqlDaemon.Replicating = false - goodReplica2.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(newPrimary.Tablet) + goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) goodReplica2.StartActionLoop(t, wr) goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", @@ -824,7 +824,7 @@ func TestPlannedReparentShardSamePrimary(t *testing.T) { // good replica 1 is replicating goodReplica1.FakeMysqlDaemon.ReadOnly = true goodReplica1.FakeMysqlDaemon.Replicating = true - goodReplica1.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(oldPrimary.Tablet) + goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", "FAKE SET MASTER", @@ -836,7 +836,7 @@ func TestPlannedReparentShardSamePrimary(t *testing.T) { // goodReplica2 is not replicating goodReplica2.FakeMysqlDaemon.ReadOnly = true goodReplica2.FakeMysqlDaemon.Replicating = false - goodReplica2.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(oldPrimary.Tablet) + goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica2.StartActionLoop(t, wr) goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE SET MASTER", diff --git a/go/vt/wrangler/testlib/reparent_utils_test.go b/go/vt/wrangler/testlib/reparent_utils_test.go index e9487fc671e..8cc157839b9 100644 --- a/go/vt/wrangler/testlib/reparent_utils_test.go +++ b/go/vt/wrangler/testlib/reparent_utils_test.go @@ -147,7 +147,7 @@ func TestReparentTablet(t *testing.T) { // which ends up making this test unpredictable. replica.FakeMysqlDaemon.Replicating = true replica.FakeMysqlDaemon.IOThreadRunning = true - replica.FakeMysqlDaemon.SetReplicationSourceInput = topoproto.MysqlAddr(primary.Tablet) + replica.FakeMysqlDaemon.SetReplicationSourceInputs = append(replica.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet)) replica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", "FAKE SET MASTER", From 620978154cb988f1c7443c2a863adfea093fa857 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 00:18:52 +0530 Subject: [PATCH 127/176] add test for reparenting replicas Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/util_test.go | 398 ++++++++++++++++++++++++++ 1 file changed, 398 insertions(+) diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index 7135bf570a1..91d7a688f0a 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -18,9 +18,17 @@ package reparentutil import ( "context" + "fmt" "testing" "time" + "k8s.io/apimachinery/pkg/util/sets" + + vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" + "vitess.io/vitess/go/vt/topo/memorytopo" + "vitess.io/vitess/go/vt/topotools/events" + "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" + "github.com/stretchr/testify/require" "vitess.io/vitess/go/test/utils" @@ -601,3 +609,393 @@ func TestCheckIfConstraintsSatisfied(t *testing.T) { }) } } + +func TestReparentReplicas(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + emergencyReparentOps EmergencyReparentOptions + tmc *testutil.TabletManagerClient + unlockTopo bool + newPrimaryTabletAlias string + ts *topo.Server + keyspace string + shard string + tablets []*topodatapb.Tablet + tabletMap map[string]*topo.TabletInfo + statusMap map[string]*replicationdatapb.StopReplicationStatus + shouldErr bool + errShouldContain string + }{ + { + name: "success", + emergencyReparentOps: NewEmergencyReparentOptions(nil, sets.NewString("zone1-0000000404"), 0, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000101": nil, + "zone1-0000000102": nil, + "zone1-0000000404": assert.AnError, // okay, because we're ignoring it. + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Hostname: "primary-elect", + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Hostname: "requires force start", + }, + }, + "zone1-0000000404": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 404, + }, + Hostname: "ignored tablet", + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{ + "zone1-0000000101": { // forceStart = false + Before: &replicationdatapb.Status{ + IoThreadRunning: false, + SqlThreadRunning: false, + }, + }, + "zone1-0000000102": { // forceStart = true + Before: &replicationdatapb.Status{ + IoThreadRunning: true, + SqlThreadRunning: true, + }, + }, + }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: false, + }, + { + name: "MasterPosition error", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + tmc: &testutil.TabletManagerClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: fmt.Errorf("primary position error"), + }, + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: "primary position error", + }, + { + name: "cannot repopulate reparent journal on new primary", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": assert.AnError, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: "failed to PopulateReparentJournal on primary", + }, + { + name: "all replicas failing to SetMaster does fail the promotion", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + + SetMasterResults: map[string]error{ + // everyone fails, we all fail + "zone1-0000000101": assert.AnError, + "zone1-0000000102": assert.AnError, + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-00000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: " replica(s) failed", + }, + { + name: "all replicas slow to SetMaster does fail the promotion", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, time.Millisecond*10, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + SetMasterDelays: map[string]time.Duration{ + // nothing is failing, we're just slow + "zone1-0000000101": time.Millisecond * 100, + "zone1-0000000102": time.Millisecond * 75, + }, + SetMasterResults: map[string]error{ + "zone1-0000000101": nil, + "zone1-0000000102": nil, + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + shouldErr: true, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + errShouldContain: "context deadline exceeded", + }, + { + name: "one replica failing to SetMaster does not fail the promotion", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000101": nil, // this one succeeds, so we're good + "zone1-0000000102": assert.AnError, + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: false, + }, + } + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + logger := logutil.NewMemoryLogger() + ev := &events.Reparent{} + + testutil.AddShards(ctx, t, tt.ts, &vtctldatapb.Shard{ + Keyspace: tt.keyspace, + Name: tt.shard, + }) + + if !tt.unlockTopo { + var ( + unlock func(*error) + lerr error + ) + + ctx, unlock, lerr = tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "test lock") + require.NoError(t, lerr, "could not lock %s/%s for test", tt.keyspace, tt.shard) + + defer func() { + unlock(&lerr) + require.NoError(t, lerr, "could not unlock %s/%s after test", tt.keyspace, tt.shard) + }() + } + tabletInfo := tt.tabletMap[tt.newPrimaryTabletAlias] + + _, err := reparentReplicas(ctx, ev, logger, tt.tmc, tabletInfo.Tablet, "", tt.tabletMap, tt.statusMap, tt.emergencyReparentOps, false, true) + if tt.shouldErr { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errShouldContain) + return + } + + assert.NoError(t, err) + }) + } +} From 0b57ed575349a3ef8f5e51be78db7f8bb4bd72d3 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 00:47:43 +0530 Subject: [PATCH 128/176] add test for promoting intermediate primary Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/util_test.go | 286 ++++++++++++++++++++++++++ 1 file changed, 286 insertions(+) diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index 91d7a688f0a..cf920497a4f 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -999,3 +999,289 @@ func TestReparentReplicas(t *testing.T) { }) } } + +func TestPromoteIntermediatePrimary(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + emergencyReparentOps EmergencyReparentOptions + tmc *testutil.TabletManagerClient + unlockTopo bool + newPrimaryTabletAlias string + ts *topo.Server + keyspace string + shard string + tablets []*topodatapb.Tablet + tabletMap map[string]*topo.TabletInfo + statusMap map[string]*replicationdatapb.StopReplicationStatus + shouldErr bool + errShouldContain string + result []*topodatapb.Tablet + }{ + { + name: "success", + emergencyReparentOps: NewEmergencyReparentOptions(nil, sets.NewString("zone1-0000000404"), 0, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000101": nil, + "zone1-0000000102": nil, + "zone1-0000000404": assert.AnError, // okay, because we're ignoring it. + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Hostname: "primary-elect", + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Hostname: "requires force start", + }, + }, + "zone1-0000000404": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 404, + }, + Hostname: "ignored tablet", + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{ + "zone1-0000000101": { // forceStart = false + Before: &replicationdatapb.Status{ + IoThreadRunning: false, + SqlThreadRunning: false, + }, + }, + "zone1-0000000102": { // forceStart = true + Before: &replicationdatapb.Status{ + IoThreadRunning: true, + SqlThreadRunning: true, + }, + }, + }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: false, + result: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Hostname: "primary-elect", + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Hostname: "requires force start", + }, + }, + }, + { + name: "all replicas failed", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + + SetMasterResults: map[string]error{ + // everyone fails, we all fail + "zone1-0000000101": assert.AnError, + "zone1-0000000102": assert.AnError, + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-00000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: " replica(s) failed", + }, + { + name: "one replica failed", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000101": nil, // this one succeeds, so we're good + "zone1-0000000102": assert.AnError, + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: false, + result: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + logger := logutil.NewMemoryLogger() + ev := &events.Reparent{} + + testutil.AddShards(ctx, t, tt.ts, &vtctldatapb.Shard{ + Keyspace: tt.keyspace, + Name: tt.shard, + }) + + if !tt.unlockTopo { + var ( + unlock func(*error) + lerr error + ) + + ctx, unlock, lerr = tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "test lock") + require.NoError(t, lerr, "could not lock %s/%s for test", tt.keyspace, tt.shard) + + defer func() { + unlock(&lerr) + require.NoError(t, lerr, "could not unlock %s/%s after test", tt.keyspace, tt.shard) + }() + } + tabletInfo := tt.tabletMap[tt.newPrimaryTabletAlias] + + res, err := promoteIntermediatePrimary(ctx, tt.tmc, ev, logger, tabletInfo.Tablet, "", tt.tabletMap, tt.statusMap, tt.emergencyReparentOps) + if tt.shouldErr { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errShouldContain) + return + } + + assert.NoError(t, err) + assert.ElementsMatch(t, tt.result, res) + }) + } +} From dd9333028fac35c8ec6063326558c690b5f3ac8f Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 01:25:45 +0530 Subject: [PATCH 129/176] add test for getting better candidate Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter.go | 12 +- go/vt/vtctl/reparentutil/util.go | 42 +- go/vt/vtctl/reparentutil/util_test.go | 369 ++++++++++++++++++ 3 files changed, 406 insertions(+), 17 deletions(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 0a4d5dfedfb..8869cecb37e 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -215,7 +215,11 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve erp.logger.Infof("intermediate primary selected - %v", intermediatePrimary.Alias) // check weather the primary candidate selected is ideal or if it can be improved later - isIdeal := intermediateCandidateIsIdeal(erp.logger, intermediatePrimary, prevPrimary, validCandidateTablets, tabletMap, opts) + var isIdeal bool + isIdeal, err = intermediateCandidateIsIdeal(erp.logger, intermediatePrimary, prevPrimary, validCandidateTablets, tabletMap, opts) + if err != nil { + return err + } erp.logger.Infof("intermediate primary is ideal - %v", isIdeal) // Check (again) we still have the topology lock. @@ -236,7 +240,11 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // try to find a better candidate using the list we got back - betterCandidate := getBetterCandidate(erp.logger, intermediatePrimary, prevPrimary, validReplacementCandidates, tabletMap, opts) + var betterCandidate *topodatapb.Tablet + betterCandidate, err = getBetterCandidate(erp.logger, intermediatePrimary, prevPrimary, validReplacementCandidates, tabletMap, opts) + if err != nil { + return err + } // if our better candidate is different from our previous candidate, then we wait for it to catch up to the intermediate primary if !topoproto.TabletAliasEqual(betterCandidate.Alias, intermediatePrimary.Alias) { diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 18946a14bf8..e382c1ce6e0 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -461,24 +461,36 @@ func getValidCandidatesAndPositionsAsList(validCandidates map[string]mysql.Posit } // intermediateCandidateIsIdeal is used to find whether the intermediate candidate that ERS chose is also the ideal one or not -func intermediateCandidateIsIdeal(logger logutil.Logger, newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) bool { +func intermediateCandidateIsIdeal(logger logutil.Logger, newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (bool, error) { // we try to find a better candidate with the current list of valid candidates, and if it matches our current primary candidate, then we return true - return getBetterCandidate(logger, newPrimary, prevPrimary, validCandidates, tabletMap, opts) == newPrimary + candidate, err := getBetterCandidate(logger, newPrimary, prevPrimary, validCandidates, tabletMap, opts) + if err != nil { + return false, err + } + return candidate == newPrimary, nil } // getBetterCandidate is used to find a better candidate for ERS promotion -func getBetterCandidate(logger logutil.Logger, newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (candidate *topodatapb.Tablet) { +func getBetterCandidate(logger logutil.Logger, newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (candidate *topodatapb.Tablet, err error) { defer func() { - logger.Infof("found better candidate - %v", candidate.Alias) + if candidate != nil { + logger.Infof("found better candidate - %v", candidate.Alias) + } }() if opts.newPrimaryAlias != nil { // explicit request to promote a specific tablet requestedPrimaryAlias := topoproto.TabletAliasString(opts.newPrimaryAlias) requestedPrimaryInfo, isFound := tabletMap[requestedPrimaryAlias] - if isFound { - return requestedPrimaryInfo.Tablet + if !isFound { + return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", requestedPrimaryAlias) } + for _, validCandidate := range validCandidates { + if topoproto.TabletAliasEqual(validCandidate.Alias, opts.newPrimaryAlias) { + return requestedPrimaryInfo.Tablet, nil + } + } + return nil, vterrors.Errorf(vtrpc.Code_ABORTED, "requested candidate %v is not in valid candidates list", requestedPrimaryAlias) } var preferredCandidates []*topodatapb.Tablet var neutralReplicas []*topodatapb.Tablet @@ -503,12 +515,12 @@ func getBetterCandidate(logger logutil.Logger, newPrimary, prevPrimary *topodata // check whether the one we promoted is in the same cell and belongs to the preferred candidates list candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, true, true) if candidate != nil { - return candidate + return candidate, nil } // check whether there is some other tablet in the same cell belonging to the preferred candidates list candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, false, true) if candidate != nil { - return candidate + return candidate, nil } // we do not have a preferred candidate in the same cell @@ -516,38 +528,38 @@ func getBetterCandidate(logger logutil.Logger, newPrimary, prevPrimary *topodata // check whether the one we promoted belongs to the preferred candidates list candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, true, false) if candidate != nil { - return candidate + return candidate, nil } // check whether there is some other tablet belonging to the preferred candidates list candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, false, false) if candidate != nil { - return candidate + return candidate, nil } } // repeat the same process for the neutral candidates list candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, true, true) if candidate != nil { - return candidate + return candidate, nil } candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, false, true) if candidate != nil { - return candidate + return candidate, nil } if !opts.preventCrossCellPromotion { candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, true, false) if candidate != nil { - return candidate + return candidate, nil } candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, false, false) if candidate != nil { - return candidate + return candidate, nil } } // return the one that we have if nothing found - return newPrimary + return newPrimary, nil } func findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary *topodatapb.Tablet, possibleCandidates []*topodatapb.Tablet, checkEqualPrimary bool, checkSameCell bool) *topodatapb.Tablet { diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index cf920497a4f..95a1b4a61e9 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -1285,3 +1285,372 @@ func TestPromoteIntermediatePrimary(t *testing.T) { }) } } + +func TestGetBetterCandidate(t *testing.T) { + tests := []struct { + name string + emergencyReparentOps EmergencyReparentOptions + newPrimary *topodatapb.Tablet + prevPrimary *topodatapb.Tablet + validCandidates []*topodatapb.Tablet + tabletMap map[string]*topo.TabletInfo + err string + result *topodatapb.Tablet + }{ + { + name: "explicit request for a primary tablet", + emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }}, + newPrimary: nil, + prevPrimary: nil, + validCandidates: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + }, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, { + name: "explicit request for a primary tablet not in valid list", + emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }}, + newPrimary: nil, + prevPrimary: nil, + validCandidates: nil, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + }, + err: "requested candidate zone1-0000000100 is not in valid candidates list", + }, { + name: "explicit request for a primary tablet not in tablet map", + emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }}, + newPrimary: nil, + prevPrimary: nil, + validCandidates: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + err: "candidate zone1-0000000100 not found in the tablet map; this an impossible situation", + }, { + name: "preferred candidate in the same cell same as our replica", + emergencyReparentOps: EmergencyReparentOptions{}, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + validCandidates: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_REPLICA, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + Type: topodatapb.TabletType_REPLICA, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + tabletMap: nil, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, { + name: "preferred candidate in the same cell different from original replica", + emergencyReparentOps: EmergencyReparentOptions{}, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + validCandidates: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + Type: topodatapb.TabletType_REPLICA, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + tabletMap: nil, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, { + name: "preferred candidate in the different cell same as original replica", + emergencyReparentOps: EmergencyReparentOptions{}, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + validCandidates: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + Type: topodatapb.TabletType_PRIMARY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 102, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + tabletMap: nil, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + }, + }, { + name: "preferred candidate in the different cell different from original replica", + emergencyReparentOps: EmergencyReparentOptions{}, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + validCandidates: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 102, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + tabletMap: nil, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 102, + }, + }, + }, { + name: "prevent cross cell promotion", + emergencyReparentOps: EmergencyReparentOptions{preventCrossCellPromotion: true}, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + validCandidates: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 102, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + tabletMap: nil, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _ = SetDurabilityPolicy("none", nil) + logger := logutil.NewMemoryLogger() + res, err := getBetterCandidate(logger, test.newPrimary, test.newPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) + if test.err != "" { + assert.EqualError(t, err, test.err) + return + } + assert.NoError(t, err) + assert.True(t, topoproto.TabletAliasEqual(res.Alias, test.result.Alias)) + }) + } +} From d730b904ca50252fe48601347732894ff175cb97 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 01:45:54 +0530 Subject: [PATCH 130/176] add test for getting valid candidates and position list Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/ers_sorted_test.go | 2 +- go/vt/vtctl/reparentutil/util_test.go | 115 ++++++++++++++++++++ 2 files changed, 116 insertions(+), 1 deletion(-) diff --git a/go/vt/vtctl/reparentutil/ers_sorted_test.go b/go/vt/vtctl/reparentutil/ers_sorted_test.go index 538cb2d834f..d22aacd216a 100644 --- a/go/vt/vtctl/reparentutil/ers_sorted_test.go +++ b/go/vt/vtctl/reparentutil/ers_sorted_test.go @@ -62,7 +62,7 @@ func TestErsSorter(t *testing.T) { mysqlGTID1 := mysql.Mysql56GTID{ Server: sid1, - Sequence: 10, + Sequence: 9, } mysqlGTID2 := mysql.Mysql56GTID{ Server: sid2, diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index 95a1b4a61e9..16879fc66df 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -22,6 +22,8 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql" + "k8s.io/apimachinery/pkg/util/sets" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" @@ -1654,3 +1656,116 @@ func TestGetBetterCandidate(t *testing.T) { }) } } + +func TestGetValidCandidatesAndPositionsAsList(t *testing.T) { + sid1 := mysql.SID{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + mysqlGTID1 := mysql.Mysql56GTID{ + Server: sid1, + Sequence: 9, + } + mysqlGTID2 := mysql.Mysql56GTID{ + Server: sid1, + Sequence: 10, + } + mysqlGTID3 := mysql.Mysql56GTID{ + Server: sid1, + Sequence: 11, + } + + positionMostAdvanced := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID1) + positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID2) + positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID3) + + positionIntermediate1 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionIntermediate1.GTIDSet = positionIntermediate1.GTIDSet.AddGTID(mysqlGTID1) + + positionIntermediate2 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionIntermediate2.GTIDSet = positionIntermediate2.GTIDSet.AddGTID(mysqlGTID1) + positionIntermediate2.GTIDSet = positionIntermediate2.GTIDSet.AddGTID(mysqlGTID2) + + tests := []struct { + name string + validCandidates map[string]mysql.Position + tabletMap map[string]*topo.TabletInfo + tabletRes []*topodatapb.Tablet + }{ + { + name: "test conversion", + validCandidates: map[string]mysql.Position{ + "zone1-0000000100": positionMostAdvanced, + "zone1-0000000101": positionIntermediate1, + "zone1-0000000102": positionIntermediate2, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Hostname: "primary-elect", + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Hostname: "requires force start", + }, + }, + "zone1-0000000404": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 404, + }, + Hostname: "ignored tablet", + }, + }, + }, + tabletRes: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Hostname: "primary-elect", + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Hostname: "requires force start", + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + tabletRes, posRes, err := getValidCandidatesAndPositionsAsList(test.validCandidates, test.tabletMap) + assert.NoError(t, err) + assert.ElementsMatch(t, test.tabletRes, tabletRes) + assert.Equal(t, len(tabletRes), len(posRes)) + for i, tablet := range tabletRes { + assert.Equal(t, test.validCandidates[topoproto.TabletAliasString(tablet.Alias)], posRes[i]) + } + }) + } +} From 1d4b981f035a9c7119fcc67c6429b10b60c87284 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 11:00:52 +0530 Subject: [PATCH 131/176] fix error in fakeMySQlDaemon Signed-off-by: Manan Gupta --- go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go index bae45a83e18..6f7749a7ae2 100644 --- a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go +++ b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go @@ -376,11 +376,15 @@ func (fmd *FakeMysqlDaemon) SetReplicationPosition(ctx context.Context, pos mysq // SetReplicationSource is part of the MysqlDaemon interface. func (fmd *FakeMysqlDaemon) SetReplicationSource(ctx context.Context, host string, port int, stopReplicationBefore bool, startReplicationAfter bool) error { input := fmt.Sprintf("%v:%v", host, port) + found := false for _, sourceInput := range fmd.SetReplicationSourceInputs { - if sourceInput != input { - return fmt.Errorf("wrong input for SetReplicationSourceCommands: expected a value in %v got %v", fmd.SetReplicationSourceInputs, input) + if sourceInput == input { + found = true } } + if !found { + return fmt.Errorf("wrong input for SetReplicationSourceCommands: expected a value in %v got %v", fmd.SetReplicationSourceInputs, input) + } if fmd.SetReplicationSourceError != nil { return fmd.SetReplicationSourceError } From ea1874b0d5ee3ac42fdfb11fcf7b8a574b013677 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 12:06:56 +0530 Subject: [PATCH 132/176] use vitess log instead of golang log in durability Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/durability.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/vt/vtctl/reparentutil/durability.go b/go/vt/vtctl/reparentutil/durability.go index 16d8d78659a..073c55684d3 100644 --- a/go/vt/vtctl/reparentutil/durability.go +++ b/go/vt/vtctl/reparentutil/durability.go @@ -19,9 +19,10 @@ package reparentutil import ( "fmt" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/orchestrator/external/golib/log" topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) From 8a2257aa532b2e625f1f34c8e36cc6aff782802c Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 12:43:13 +0530 Subject: [PATCH 133/176] added test for waiting for catching up Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter.go | 2 +- go/vt/vtctl/reparentutil/util.go | 4 +- go/vt/vtctl/reparentutil/util_test.go | 117 ++++++++++++++++++ 3 files changed, 120 insertions(+), 3 deletions(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 8869cecb37e..2dec0e4254c 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -248,7 +248,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // if our better candidate is different from our previous candidate, then we wait for it to catch up to the intermediate primary if !topoproto.TabletAliasEqual(betterCandidate.Alias, intermediatePrimary.Alias) { - err = waitForCatchingUp(ctx, erp.tmc, erp.ts, ev, erp.logger, intermediatePrimary, betterCandidate, opts.lockAction, tabletMap, statusMap, opts) + err = waitForCatchingUp(ctx, erp.tmc, erp.logger, intermediatePrimary, betterCandidate) if err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index e382c1ce6e0..a463cbff403 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -351,8 +351,8 @@ func FindCurrentPrimary(tabletMap map[string]*topo.TabletInfo, logger logutil.Lo } // waitForCatchingUp promotes the newer candidate over the primary candidate that we have, but it does not set to start accepting writes -func waitForCatchingUp(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, ev *events.Reparent, logger logutil.Logger, prevPrimary, newPrimary *topodatapb.Tablet, - lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions) error { +func waitForCatchingUp(ctx context.Context, tmc tmclient.TabletManagerClient, logger logutil.Logger, prevPrimary, newPrimary *topodatapb.Tablet) error { + logger.Infof("waiting for %v to catch up to %v", newPrimary.Alias, prevPrimary.Alias) // Find the primary position of the previous primary pos, err := tmc.MasterPosition(ctx, prevPrimary) if err != nil { diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index 16879fc66df..3cc9c7d47f3 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -1769,3 +1769,120 @@ func TestGetValidCandidatesAndPositionsAsList(t *testing.T) { }) } } + +func TestWaitForCatchingUp(t *testing.T) { + tests := []struct { + name string + tmc tmclient.TabletManagerClient + prevPrimary *topodatapb.Tablet + newPrimary *topodatapb.Tablet + err string + }{ + { + name: "success", + tmc: &testutil.TabletManagerClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "abc", + Error: nil, + }, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000101": { + "abc": nil, + }, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, { + name: "error in primary position", + tmc: &testutil.TabletManagerClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "abc", + Error: fmt.Errorf("found error in primary position"), + }, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000101": { + "abc": nil, + }, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + err: "found error in primary position", + }, { + name: "error in waiting for position", + tmc: &testutil.TabletManagerClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "abc", + Error: nil, + }, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000101": { + "abc": fmt.Errorf("found error in waiting for position"), + }, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + err: "found error in waiting for position", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + logger := logutil.NewMemoryLogger() + err := waitForCatchingUp(ctx, test.tmc, logger, test.prevPrimary, test.newPrimary) + if test.err != "" { + assert.EqualError(t, err, test.err) + } else { + assert.NoError(t, err) + } + }) + } +} From 5efd4c8943ada5e6f09a0b1307df025585dfd137 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 12:46:19 +0530 Subject: [PATCH 134/176] handle data race in test Signed-off-by: Manan Gupta --- go/vt/vtctl/grpcvtctldserver/server_slow_test.go | 3 --- go/vt/vtctl/grpcvtctldserver/server_test.go | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/go/vt/vtctl/grpcvtctldserver/server_slow_test.go b/go/vt/vtctl/grpcvtctldserver/server_slow_test.go index be8e4a2f468..0f6a4d7cf25 100644 --- a/go/vt/vtctl/grpcvtctldserver/server_slow_test.go +++ b/go/vt/vtctl/grpcvtctldserver/server_slow_test.go @@ -21,8 +21,6 @@ import ( "testing" "time" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -295,7 +293,6 @@ func TestEmergencyReparentShardSlow(t *testing.T) { } ctx := context.Background() - _ = reparentutil.SetDurabilityPolicy("none", nil) for _, tt := range tests { tt := tt diff --git a/go/vt/vtctl/grpcvtctldserver/server_test.go b/go/vt/vtctl/grpcvtctldserver/server_test.go index 666ac83999a..9cea6d7f7bf 100644 --- a/go/vt/vtctl/grpcvtctldserver/server_test.go +++ b/go/vt/vtctl/grpcvtctldserver/server_test.go @@ -66,6 +66,7 @@ func init() { tmclient.RegisterTabletManagerClientFactory("grpcvtctldserver.test", func() tmclient.TabletManagerClient { return nil }) + _ = reparentutil.SetDurabilityPolicy("none", nil) } func TestAddCellInfo(t *testing.T) { @@ -2745,7 +2746,6 @@ func TestEmergencyReparentShard(t *testing.T) { } ctx := context.Background() - _ = reparentutil.SetDurabilityPolicy("none", nil) for _, tt := range tests { tt := tt From 5102ae76cbeec559d927324b7afe2b744b8d0299 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 12:55:09 +0530 Subject: [PATCH 135/176] added test for restricting valid candidate list Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/util_test.go | 70 +++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index 3cc9c7d47f3..d957df76691 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -1886,3 +1886,73 @@ func TestWaitForCatchingUp(t *testing.T) { }) } } + +func TestResrictValidCandidates(t *testing.T) { + tests := []struct { + name string + validCandidates map[string]mysql.Position + tabletMap map[string]*topo.TabletInfo + result map[string]mysql.Position + }{ + { + name: "remove experimental tablets", + validCandidates: map[string]mysql.Position{ + "zone1-0000000100": {}, + "zone1-0000000101": {}, + "zone1-0000000102": {}, + "zone1-0000000103": {}, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + Type: topodatapb.TabletType_RDONLY, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_EXPERIMENTAL, + }, + }, + "zone1-0000000103": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 103, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, + }, + result: map[string]mysql.Position{ + "zone1-0000000100": {}, + "zone1-0000000101": {}, + "zone1-0000000103": {}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + res, err := restrictValidCandidates(test.validCandidates, test.tabletMap) + assert.NoError(t, err) + assert.Equal(t, res, test.result) + }) + } +} From cf7e9b5958829305448c1cfa467af6d9139ddfbc Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 14:10:02 +0530 Subject: [PATCH 136/176] added tests for finding intermediate primary and added split brain detection Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/util.go | 8 + go/vt/vtctl/reparentutil/util_test.go | 330 ++++++++++++++++++++++++++ 2 files changed, 338 insertions(+) diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index a463cbff403..c51698fde5e 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -422,6 +422,14 @@ func findIntermediatePrimaryCandidate(logger logutil.Logger, prevPrimary *topoda winningPrimaryTablet := validTablets[0] winningPosition := tabletPositions[0] + // We have already removed the tablets with errant GTIDs before calling this function. At this point our winning position must be a + // superset of all the other valid positions. If that is not the case, then we have a split brain scenario, and we should abort the ERS + for i, position := range tabletPositions { + if !winningPosition.AtLeast(position) { + return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "split brain detected between servers - %v and %v", winningPrimaryTablet.Alias, validTablets[i].Alias) + } + } + // If we were requested to elect a particular primary, verify it's a valid // candidate (non-zero position, no errant GTIDs) // Also, if the candidate is diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index d957df76691..5554b20f8e5 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -1956,3 +1956,333 @@ func TestResrictValidCandidates(t *testing.T) { }) } } + +func TestFindIntermediatePrimaryCandidate(t *testing.T) { + sid1 := mysql.SID{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + mysqlGTID1 := mysql.Mysql56GTID{ + Server: sid1, + Sequence: 9, + } + mysqlGTID2 := mysql.Mysql56GTID{ + Server: sid1, + Sequence: 10, + } + mysqlGTID3 := mysql.Mysql56GTID{ + Server: sid1, + Sequence: 11, + } + + positionMostAdvanced := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID1) + positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID2) + positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID3) + + positionIntermediate1 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionIntermediate1.GTIDSet = positionIntermediate1.GTIDSet.AddGTID(mysqlGTID1) + + positionIntermediate2 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionIntermediate2.GTIDSet = positionIntermediate2.GTIDSet.AddGTID(mysqlGTID1) + positionIntermediate2.GTIDSet = positionIntermediate2.GTIDSet.AddGTID(mysqlGTID2) + + positionOnly2 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionOnly2.GTIDSet = positionOnly2.GTIDSet.AddGTID(mysqlGTID2) + + positionEmpty := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + + tests := []struct { + name string + validCandidates map[string]mysql.Position + tabletMap map[string]*topo.TabletInfo + prevPrimary *topodatapb.Tablet + emergencyReparentOps EmergencyReparentOptions + result *topodatapb.Tablet + err string + }{ + { + name: "choose most advanced - with nil previous primary", + validCandidates: map[string]mysql.Position{ + "zone1-0000000100": positionMostAdvanced, + "zone1-0000000101": positionIntermediate1, + "zone1-0000000102": positionIntermediate2, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + "zone1-0000000404": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 404, + }, + Hostname: "ignored tablet", + }, + }, + }, + prevPrimary: nil, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, { + name: "choose most advanced in the same cell of previous primary", + validCandidates: map[string]mysql.Position{ + "zone1-0000000100": positionMostAdvanced, + "zone1-0000000101": positionIntermediate1, + "zone2-0000000100": positionMostAdvanced, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone2-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 100, + }, + }, + }, + "zone1-0000000404": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 404, + }, + Hostname: "ignored tablet", + }, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, { + name: "choose most advanced with the best promotion rule", + validCandidates: map[string]mysql.Position{ + "zone1-0000000100": positionMostAdvanced, + "zone1-0000000101": positionIntermediate1, + "zone1-0000000102": positionMostAdvanced, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, + }, + "zone1-0000000404": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 404, + }, + Hostname: "ignored tablet", + }, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, { + name: "choose most advanced with explicit request", + emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }}, + validCandidates: map[string]mysql.Position{ + "zone1-0000000100": positionMostAdvanced, + "zone1-0000000101": positionIntermediate1, + "zone1-0000000102": positionMostAdvanced, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, + }, + "zone1-0000000404": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 404, + }, + Hostname: "ignored tablet", + }, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, { + name: "split brain detection", + emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }}, + validCandidates: map[string]mysql.Position{ + "zone1-0000000100": positionOnly2, + "zone1-0000000101": positionIntermediate1, + "zone1-0000000102": positionEmpty, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, + }, + "zone1-0000000404": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 404, + }, + Hostname: "ignored tablet", + }, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + err: "split brain detected between servers", + }, + } + + _ = SetDurabilityPolicy("none", nil) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + winningTablet, _, err := findIntermediatePrimaryCandidate(logutil.NewMemoryLogger(), test.prevPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) + if test.err != "" { + assert.Error(t, err) + assert.Contains(t, err.Error(), test.err) + } else { + assert.NoError(t, err) + assert.True(t, topoproto.TabletAliasEqual(test.result.Alias, winningTablet.Alias)) + } + }) + } +} From 3f74bcaa617f0685b8e6826755e42c3b0c517e2d Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 14:13:32 +0530 Subject: [PATCH 137/176] remove a todo Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/topology_recovery.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 2ac13d7aeda..78b8da4e711 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -679,7 +679,6 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat } postErsCompletion(topologyRecovery, analysisEntry, skipProcesses, promotedReplica) - // TODO: fix recovery attempted return true, topologyRecovery, err } From 11d2e0d4229ee226666e535797bd3a630d8159c8 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 14:50:15 +0530 Subject: [PATCH 138/176] move ers tests to their own package Signed-off-by: Manan Gupta --- .../endtoend/reparent/{ => ers}/ers_test.go | 12 +- go/test/endtoend/reparent/ers/utils_test.go | 431 ++++++++++++++++++ .../{ => prs}/reparent_range_based_test.go | 2 +- .../reparent/{ => prs}/reparent_test.go | 2 +- .../endtoend/reparent/{ => prs}/utils_test.go | 7 +- 5 files changed, 440 insertions(+), 14 deletions(-) rename go/test/endtoend/reparent/{ => ers}/ers_test.go (93%) create mode 100644 go/test/endtoend/reparent/ers/utils_test.go rename go/test/endtoend/reparent/{ => prs}/reparent_range_based_test.go (98%) rename go/test/endtoend/reparent/{ => prs}/reparent_test.go (99%) rename go/test/endtoend/reparent/{ => prs}/utils_test.go (98%) diff --git a/go/test/endtoend/reparent/ers_test.go b/go/test/endtoend/reparent/ers/ers_test.go similarity index 93% rename from go/test/endtoend/reparent/ers_test.go rename to go/test/endtoend/reparent/ers/ers_test.go index f2f1481e7ee..03345b199cf 100644 --- a/go/test/endtoend/reparent/ers_test.go +++ b/go/test/endtoend/reparent/ers/ers_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package reparent +package ers import ( "context" @@ -37,7 +37,7 @@ func TestTrivialERS(t *testing.T) { // We should be able to do a series of ERS-es, even if nothing // is down, without issue for i := 1; i <= 4; i++ { - out, err := ers(t, nil, "30s") + out, err := ers(nil, "60s", "30s") log.Infof("ERS loop %d. EmergencyReparentShard Output: %v", i, out) require.NoError(t, err) time.Sleep(5 * time.Second) @@ -68,11 +68,11 @@ func TestReparentIgnoreReplicas(t *testing.T) { stopTablet(t, tab3, true) // We expect this one to fail because we have an unreachable replica - out, err := ers(t, nil, "30s") + out, err := ers(nil, "60s", "30s") require.NotNil(t, err, out) // Now let's run it again, but set the command to ignore the unreachable replica. - out, err = ersIgnoreTablet(t, nil, "30s", []*cluster.Vttablet{tab3}) + out, err = ersIgnoreTablet(nil, "60s", "30s", []*cluster.Vttablet{tab3}) require.Nil(t, err, out) // We'll bring back the replica we took down. @@ -111,7 +111,7 @@ func TestERSPromoteRdonly(t *testing.T) { stopTablet(t, tab1, true) // We expect this one to fail because we have ignored all the replicas and have only the rdonly's which should not be promoted - out, err := ersIgnoreTablet(t, nil, "30s", []*cluster.Vttablet{tab4}) + out, err := ersIgnoreTablet(nil, "30s", "30s", []*cluster.Vttablet{tab4}) require.NotNil(t, err, out) out, err = clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetShard", keyspaceShard) @@ -133,7 +133,7 @@ func TestERSPrefersSameCell(t *testing.T) { stopTablet(t, tab1, true) // We expect that tab3 will be promoted since it is in the same cell as the previous primary - out, err := ersIgnoreTablet(t, nil, "30s", []*cluster.Vttablet{tab2}) + out, err := ersIgnoreTablet(nil, "60s", "30s", []*cluster.Vttablet{tab2}) require.NoError(t, err, out) newPrimary := getNewPrimary(t) diff --git a/go/test/endtoend/reparent/ers/utils_test.go b/go/test/endtoend/reparent/ers/utils_test.go new file mode 100644 index 00000000000..3f96d205031 --- /dev/null +++ b/go/test/endtoend/reparent/ers/utils_test.go @@ -0,0 +1,431 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ers + +import ( + "context" + "fmt" + "os" + "os/exec" + "path" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "vitess.io/vitess/go/json2" + "vitess.io/vitess/go/vt/log" + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/endtoend/cluster" + tmc "vitess.io/vitess/go/vt/vttablet/grpctmclient" +) + +var ( + // ClusterInstance instance to be used for test with different params + clusterInstance *cluster.LocalProcessCluster + tmClient *tmc.Client + keyspaceName = "ks" + dbName = "vt_" + keyspaceName + username = "vt_dba" + hostname = "localhost" + insertVal = 1 + insertSQL = "insert into vt_insert_test(id, msg) values (%d, 'test %d')" + sqlSchema = ` + create table vt_insert_test ( + id bigint, + msg varchar(64), + primary key (id) + ) Engine=InnoDB +` + cell1 = "zone1" + cell2 = "zone2" + shardName = "0" + keyspaceShard = keyspaceName + "/" + shardName + + tab1, tab2, tab3, tab4 *cluster.Vttablet +) + +func setupReparentCluster(t *testing.T) { + tablets := setupCluster(context.Background(), t, shardName, []string{cell1, cell2}, []int{3, 1}) + tab1, tab2, tab3, tab4 = tablets[0], tablets[1], tablets[2], tablets[3] +} + +func teardownCluster() { + clusterInstance.Teardown() +} + +func setupCluster(ctx context.Context, t *testing.T, shardName string, cells []string, numTablets []int) []*cluster.Vttablet { + var tablets []*cluster.Vttablet + clusterInstance = cluster.NewCluster(cells[0], hostname) + keyspace := &cluster.Keyspace{Name: keyspaceName} + // Start topo server + err := clusterInstance.StartTopo() + if err != nil { + t.Fatalf("Error starting topo: %s", err.Error()) + } + err = clusterInstance.TopoProcess.ManageTopoDir("mkdir", "/vitess/"+cells[0]) + if err != nil { + t.Fatalf("Error managing topo: %s", err.Error()) + } + numCell := 1 + for numCell < len(cells) { + err = clusterInstance.VtctlProcess.AddCellInfo(cells[numCell]) + if err != nil { + t.Fatalf("Error managing topo: %s", err.Error()) + } + numCell++ + } + + // Adding another cell in the same cluster + numCell = 0 + for numCell < len(cells) { + i := 0 + for i < numTablets[numCell] { + i++ + tablet := clusterInstance.NewVttabletInstance("replica", 100*(numCell+1)+i, cells[numCell]) + tablets = append(tablets, tablet) + } + numCell++ + } + + shard := &cluster.Shard{Name: shardName} + shard.Vttablets = tablets + + clusterInstance.VtTabletExtraArgs = []string{ + "-lock_tables_timeout", "5s", + "-enable_semi_sync", + "-init_populate_metadata", + "-track_schema_versions=true", + } + + // Initialize Cluster + err = clusterInstance.SetupCluster(keyspace, []cluster.Shard{*shard}) + if err != nil { + t.Fatalf("Cannot launch cluster: %s", err.Error()) + } + + //Start MySql + var mysqlCtlProcessList []*exec.Cmd + for _, shard := range clusterInstance.Keyspaces[0].Shards { + for _, tablet := range shard.Vttablets { + log.Infof("Starting MySql for tablet %v", tablet.Alias) + proc, err := tablet.MysqlctlProcess.StartProcess() + if err != nil { + t.Fatalf("Error starting start mysql: %s", err.Error()) + } + mysqlCtlProcessList = append(mysqlCtlProcessList, proc) + } + } + + // Wait for mysql processes to start + for _, proc := range mysqlCtlProcessList { + if err := proc.Wait(); err != nil { + t.Fatalf("Error starting mysql: %s", err.Error()) + } + } + + // create tablet manager client + tmClient = tmc.NewClient() + setupShard(ctx, t, shardName, tablets) + return tablets +} + +func setupShard(ctx context.Context, t *testing.T, shardName string, tablets []*cluster.Vttablet) { + for _, tablet := range tablets { + // create database + err := tablet.VttabletProcess.CreateDB(keyspaceName) + require.NoError(t, err) + // Start the tablet + err = tablet.VttabletProcess.Setup() + require.NoError(t, err) + } + + for _, tablet := range tablets { + err := tablet.VttabletProcess.WaitForTabletStatuses([]string{"SERVING", "NOT_SERVING"}) + require.NoError(t, err) + } + + // Force the replica to reparent assuming that all the datasets are identical. + err := clusterInstance.VtctlclientProcess.ExecuteCommand("InitShardPrimary", + "-force", fmt.Sprintf("%s/%s", keyspaceName, shardName), tablets[0].Alias) + require.NoError(t, err) + + validateTopology(t, true) + + // create Tables + runSQL(ctx, t, sqlSchema, tablets[0]) + + checkPrimaryTablet(t, tablets[0]) + + validateTopology(t, false) + time.Sleep(100 * time.Millisecond) // wait for replication to catchup + strArray := getShardReplicationPositions(t, keyspaceName, shardName, true) + assert.Equal(t, len(tablets), len(strArray)) + assert.Contains(t, strArray[0], "primary") // primary first +} + +//endregion + +//region database queries +func getMysqlConnParam(tablet *cluster.Vttablet) mysql.ConnParams { + connParams := mysql.ConnParams{ + Uname: username, + DbName: dbName, + UnixSocket: path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d/mysql.sock", tablet.TabletUID)), + } + return connParams +} + +func runSQL(ctx context.Context, t *testing.T, sql string, tablet *cluster.Vttablet) *sqltypes.Result { + tabletParams := getMysqlConnParam(tablet) + conn, err := mysql.Connect(ctx, &tabletParams) + require.Nil(t, err) + defer conn.Close() + return execute(t, conn, sql) +} + +func execute(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { + t.Helper() + qr, err := conn.ExecuteFetch(query, 1000, true) + require.Nil(t, err) + return qr +} + +//endregion + +// region ers + +func ers(tab *cluster.Vttablet, totalTimeout, waitReplicasTimeout string) (string, error) { + return ersIgnoreTablet(tab, totalTimeout, waitReplicasTimeout, nil) +} + +func ersIgnoreTablet(tab *cluster.Vttablet, timeout, waitReplicasTimeout string, tabletsToIgnore []*cluster.Vttablet) (string, error) { + var args []string + if timeout != "" { + args = append(args, "-action_timeout", timeout) + } + args = append(args, "EmergencyReparentShard", "-keyspace_shard", fmt.Sprintf("%s/%s", keyspaceName, shardName)) + if tab != nil { + args = append(args, "-new_primary", tab.Alias) + } + if waitReplicasTimeout != "" { + args = append(args, "-wait_replicas_timeout", waitReplicasTimeout) + } + if len(tabletsToIgnore) != 0 { + tabsString := "" + for _, vttablet := range tabletsToIgnore { + if tabsString == "" { + tabsString = vttablet.Alias + } else { + tabsString = tabsString + "," + vttablet.Alias + } + } + args = append(args, "-ignore_replicas", tabsString) + } + return clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput(args...) +} + +func ersWithVtctl() (string, error) { + args := []string{"EmergencyReparentShard", "-keyspace_shard", fmt.Sprintf("%s/%s", keyspaceName, shardName)} + return clusterInstance.VtctlProcess.ExecuteCommandWithOutput(args...) +} + +// endregion + +// region validations + +func validateTopology(t *testing.T, pingTablets bool) { + args := []string{"Validate"} + + if pingTablets { + args = append(args, "-ping-tablets=true") + } + out, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput(args...) + require.Empty(t, out) + require.NoError(t, err) +} + +func confirmReplication(t *testing.T, primary *cluster.Vttablet, replicas []*cluster.Vttablet) { + ctx := context.Background() + insertVal++ + n := insertVal // unique value ... + // insert data into the new primary, check the connected replica work + insertSQL := fmt.Sprintf(insertSQL, n, n) + runSQL(ctx, t, insertSQL, primary) + time.Sleep(100 * time.Millisecond) + for _, tab := range replicas { + err := checkInsertedValues(ctx, t, tab, n) + require.NoError(t, err) + } +} + +func confirmOldPrimaryIsHangingAround(t *testing.T) { + out, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("Validate") + require.Error(t, err) + require.Contains(t, out, "already has primary") +} + +// Makes sure the tablet type is primary, and its health check agrees. +func checkPrimaryTablet(t *testing.T, tablet *cluster.Vttablet) { + result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", tablet.Alias) + require.NoError(t, err) + var tabletInfo topodatapb.Tablet + err = json2.Unmarshal([]byte(result), &tabletInfo) + require.NoError(t, err) + assert.Equal(t, topodatapb.TabletType_PRIMARY, tabletInfo.GetType()) + + // make sure the health stream is updated + result, err = clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("VtTabletStreamHealth", "-count", "1", tablet.Alias) + require.NoError(t, err) + var streamHealthResponse querypb.StreamHealthResponse + + err = json2.Unmarshal([]byte(result), &streamHealthResponse) + require.NoError(t, err) + + assert.True(t, streamHealthResponse.GetServing()) + tabletType := streamHealthResponse.GetTarget().GetTabletType() + assert.Equal(t, topodatapb.TabletType_PRIMARY, tabletType) +} + +// isHealthyPrimaryTablet will return if tablet is primary AND healthy. +func isHealthyPrimaryTablet(t *testing.T, tablet *cluster.Vttablet) bool { + result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", tablet.Alias) + require.Nil(t, err) + var tabletInfo topodatapb.Tablet + err = json2.Unmarshal([]byte(result), &tabletInfo) + require.Nil(t, err) + if tabletInfo.GetType() != topodatapb.TabletType_PRIMARY { + return false + } + + // make sure the health stream is updated + result, err = clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("VtTabletStreamHealth", "-count", "1", tablet.Alias) + require.Nil(t, err) + var streamHealthResponse querypb.StreamHealthResponse + + err = json2.Unmarshal([]byte(result), &streamHealthResponse) + require.Nil(t, err) + + assert.True(t, streamHealthResponse.GetServing()) + tabletType := streamHealthResponse.GetTarget().GetTabletType() + return tabletType == topodatapb.TabletType_PRIMARY +} + +func checkInsertedValues(ctx context.Context, t *testing.T, tablet *cluster.Vttablet, index int) error { + // wait until it gets the data + timeout := time.Now().Add(15 * time.Second) + i := 0 + for time.Now().Before(timeout) { + selectSQL := fmt.Sprintf("select msg from vt_insert_test where id=%d", index) + qr := runSQL(ctx, t, selectSQL, tablet) + if len(qr.Rows) == 1 { + return nil + } + t := time.Duration(300 * i) + time.Sleep(t * time.Millisecond) + i++ + } + return fmt.Errorf("data is not yet replicated on tablet %s", tablet.Alias) +} + +// endregion + +// region tablet operations + +func stopTablet(t *testing.T, tab *cluster.Vttablet, stopDatabase bool) { + err := tab.VttabletProcess.TearDownWithTimeout(30 * time.Second) + require.NoError(t, err) + if stopDatabase { + err = tab.MysqlctlProcess.Stop() + require.NoError(t, err) + } +} + +func restartTablet(t *testing.T, tab *cluster.Vttablet) { + tab.MysqlctlProcess.InitMysql = false + err := tab.MysqlctlProcess.Start() + require.NoError(t, err) + err = clusterInstance.VtctlclientProcess.InitTablet(tab, tab.Cell, keyspaceName, hostname, shardName) + require.NoError(t, err) +} + +func resurrectTablet(ctx context.Context, t *testing.T, tab *cluster.Vttablet) { + tab.MysqlctlProcess.InitMysql = false + err := tab.MysqlctlProcess.Start() + require.NoError(t, err) + err = clusterInstance.VtctlclientProcess.InitTablet(tab, tab.Cell, keyspaceName, hostname, shardName) + require.NoError(t, err) + + // As there is already a primary the new replica will come directly in SERVING state + tab1.VttabletProcess.ServingStatus = "SERVING" + // Start the tablet + err = tab.VttabletProcess.Setup() + require.NoError(t, err) + + err = checkInsertedValues(ctx, t, tab, insertVal) + require.NoError(t, err) +} + +func deleteTablet(t *testing.T, tab *cluster.Vttablet) { + err := clusterInstance.VtctlclientProcess.ExecuteCommand( + "DeleteTablet", + "-allow_primary", + tab.Alias) + require.NoError(t, err) +} + +// endregion + +// region get info + +func getNewPrimary(t *testing.T) *cluster.Vttablet { + var newPrimary *cluster.Vttablet + for _, tablet := range []*cluster.Vttablet{tab2, tab3, tab4} { + if isHealthyPrimaryTablet(t, tablet) { + newPrimary = tablet + break + } + } + require.NotNil(t, newPrimary) + return newPrimary +} + +func getShardReplicationPositions(t *testing.T, keyspaceName, shardName string, doPrint bool) []string { + output, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput( + "ShardReplicationPositions", fmt.Sprintf("%s/%s", keyspaceName, shardName)) + require.NoError(t, err) + strArray := strings.Split(output, "\n") + if strArray[len(strArray)-1] == "" { + strArray = strArray[:len(strArray)-1] // Truncate slice, remove empty line + } + if doPrint { + log.Infof("Positions:") + for _, pos := range strArray { + log.Infof("\t%s", pos) + } + } + return strArray +} + +// endregion diff --git a/go/test/endtoend/reparent/reparent_range_based_test.go b/go/test/endtoend/reparent/prs/reparent_range_based_test.go similarity index 98% rename from go/test/endtoend/reparent/reparent_range_based_test.go rename to go/test/endtoend/reparent/prs/reparent_range_based_test.go index eb9a23b1c13..bd092da6920 100644 --- a/go/test/endtoend/reparent/reparent_range_based_test.go +++ b/go/test/endtoend/reparent/prs/reparent_range_based_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package reparent +package prs import ( "context" diff --git a/go/test/endtoend/reparent/reparent_test.go b/go/test/endtoend/reparent/prs/reparent_test.go similarity index 99% rename from go/test/endtoend/reparent/reparent_test.go rename to go/test/endtoend/reparent/prs/reparent_test.go index 12220cba3b0..a7e78f95f8d 100644 --- a/go/test/endtoend/reparent/reparent_test.go +++ b/go/test/endtoend/reparent/prs/reparent_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package reparent +package prs import ( "context" diff --git a/go/test/endtoend/reparent/utils_test.go b/go/test/endtoend/reparent/prs/utils_test.go similarity index 98% rename from go/test/endtoend/reparent/utils_test.go rename to go/test/endtoend/reparent/prs/utils_test.go index 5b34f7637fe..b8a3d48b22b 100644 --- a/go/test/endtoend/reparent/utils_test.go +++ b/go/test/endtoend/reparent/prs/utils_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package reparent +package prs import ( "context" @@ -272,11 +272,6 @@ func ersIgnoreTablet(t *testing.T, tab *cluster.Vttablet, timeout string, tablet return clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput(args...) } -func ersWithVtctl() (string, error) { - args := []string{"EmergencyReparentShard", "-keyspace_shard", fmt.Sprintf("%s/%s", keyspaceName, shardName)} - return clusterInstance.VtctlProcess.ExecuteCommandWithOutput(args...) -} - func checkReparentFromOutside(t *testing.T, tablet *cluster.Vttablet, downPrimary bool, baseTime int64) { result, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetShardReplication", cell1, keyspaceShard) require.Nil(t, err, "error should be Nil") From ee1d628dd1d06ed10253ad3ee004e67a29556426 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 14:54:04 +0530 Subject: [PATCH 139/176] update config file for tests Signed-off-by: Manan Gupta --- go/vt/sqlparser/cached_size.go | 14 ++++++++++++++ go/vt/vtgate/engine/cached_size.go | 8 ++++---- test/config.json | 13 +++++++++++-- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/go/vt/sqlparser/cached_size.go b/go/vt/sqlparser/cached_size.go index 611eb65a7f1..0ee325a3f70 100644 --- a/go/vt/sqlparser/cached_size.go +++ b/go/vt/sqlparser/cached_size.go @@ -1624,6 +1624,20 @@ func (cached *RevertMigration) CachedSize(alloc bool) int64 { } return size } +func (cached *RootNode) CachedSize(alloc bool) int64 { + if cached == nil { + return int64(0) + } + size := int64(0) + if alloc { + size += int64(16) + } + // field SQLNode vitess.io/vitess/go/vt/sqlparser.SQLNode + if cc, ok := cached.SQLNode.(cachedObject); ok { + size += cc.CachedSize(true) + } + return size +} func (cached *SRollback) CachedSize(alloc bool) int64 { if cached == nil { return int64(0) diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index a1db2e3f185..aaf0391fe1b 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -197,7 +197,7 @@ func (cached *GroupByParams) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(32) + size += int64(48) } // field Expr vitess.io/vitess/go/vt/sqlparser.Expr if cc, ok := cached.Expr.(cachedObject); ok { @@ -345,7 +345,7 @@ func (cached *MemorySort) CachedSize(alloc bool) int64 { size += cached.UpperLimit.CachedSize(false) // field OrderBy []vitess.io/vitess/go/vt/vtgate/engine.OrderByParams { - size += hack.RuntimeAllocSize(int64(cap(cached.OrderBy)) * int64(32)) + size += hack.RuntimeAllocSize(int64(cap(cached.OrderBy)) * int64(33)) } // field Input vitess.io/vitess/go/vt/vtgate/engine.Primitive if cc, ok := cached.Input.(cachedObject); ok { @@ -372,7 +372,7 @@ func (cached *MergeSort) CachedSize(alloc bool) int64 { } // field OrderBy []vitess.io/vitess/go/vt/vtgate/engine.OrderByParams { - size += hack.RuntimeAllocSize(int64(cap(cached.OrderBy)) * int64(32)) + size += hack.RuntimeAllocSize(int64(cap(cached.OrderBy)) * int64(33)) } return size } @@ -599,7 +599,7 @@ func (cached *Route) CachedSize(alloc bool) int64 { } // field OrderBy []vitess.io/vitess/go/vt/vtgate/engine.OrderByParams { - size += hack.RuntimeAllocSize(int64(cap(cached.OrderBy)) * int64(32)) + size += hack.RuntimeAllocSize(int64(cap(cached.OrderBy)) * int64(33)) } // field SysTableTableSchema []vitess.io/vitess/go/vt/vtgate/evalengine.Expr { diff --git a/test/config.json b/test/config.json index 7ee81b29097..d3d399e1e56 100644 --- a/test/config.json +++ b/test/config.json @@ -413,9 +413,18 @@ "RetryMax": 1, "Tags": [] }, - "reparent": { + "ers": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/reparent"], + "Args": ["vitess.io/vitess/go/test/endtoend/reparent/ers"], + "Command": [], + "Manual": false, + "Shard": "14", + "RetryMax": 1, + "Tags": [] + }, + "prs": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/reparent/prs"], "Command": [], "Manual": false, "Shard": "14", From cb0f05cf30c66ccb75bd1af366c446b756054c9a Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 15:36:44 +0530 Subject: [PATCH 140/176] remove unused code Signed-off-by: Manan Gupta --- go/test/endtoend/reparent/prs/utils_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/go/test/endtoend/reparent/prs/utils_test.go b/go/test/endtoend/reparent/prs/utils_test.go index b8a3d48b22b..6c4b8211bda 100644 --- a/go/test/endtoend/reparent/prs/utils_test.go +++ b/go/test/endtoend/reparent/prs/utils_test.go @@ -479,14 +479,6 @@ func stopTablet(t *testing.T, tab *cluster.Vttablet, stopDatabase bool) { } } -func restartTablet(t *testing.T, tab *cluster.Vttablet) { - tab.MysqlctlProcess.InitMysql = false - err := tab.MysqlctlProcess.Start() - require.NoError(t, err) - err = clusterInstance.VtctlclientProcess.InitTablet(tab, tab.Cell, keyspaceName, hostname, shardName) - require.NoError(t, err) -} - func resurrectTablet(ctx context.Context, t *testing.T, tab *cluster.Vttablet) { tab.MysqlctlProcess.InitMysql = false err := tab.MysqlctlProcess.Start() From 786217fbada908af424ecebd9ac99f8eb380a42e Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 24 Sep 2021 21:51:53 +0530 Subject: [PATCH 141/176] force tablet refresh in vtorc Signed-off-by: Manan Gupta --- go/vt/orchestrator/http/api.go | 2 +- go/vt/orchestrator/logic/command_applier.go | 2 +- go/vt/orchestrator/logic/orchestrator.go | 10 +++++----- go/vt/orchestrator/logic/tablet_discovery.go | 4 ++-- go/vt/orchestrator/logic/topology_recovery.go | 5 ++++- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/go/vt/orchestrator/http/api.go b/go/vt/orchestrator/http/api.go index e2c6224aa94..77e6fe410fc 100644 --- a/go/vt/orchestrator/http/api.go +++ b/go/vt/orchestrator/http/api.go @@ -218,7 +218,7 @@ func (this *HttpAPI) Discover(params martini.Params, r render.Render, req *http. if orcraft.IsRaftEnabled() { orcraft.PublishCommand("discover", instanceKey) } else { - logic.DiscoverInstance(instanceKey) + logic.DiscoverInstance(instanceKey, false) } Respond(r, &APIResponse{Code: OK, Message: fmt.Sprintf("Instance discovered: %+v", instance.Key), Details: instance}) diff --git a/go/vt/orchestrator/logic/command_applier.go b/go/vt/orchestrator/logic/command_applier.go index 275b0c104e8..f56a101135f 100644 --- a/go/vt/orchestrator/logic/command_applier.go +++ b/go/vt/orchestrator/logic/command_applier.go @@ -93,7 +93,7 @@ func (applier *CommandApplier) discover(value []byte) interface{} { if err := json.Unmarshal(value, &instanceKey); err != nil { return log.Errore(err) } - DiscoverInstance(instanceKey) + DiscoverInstance(instanceKey, false) return nil } diff --git a/go/vt/orchestrator/logic/orchestrator.go b/go/vt/orchestrator/logic/orchestrator.go index ff3f84fd850..fff4ee52a1a 100644 --- a/go/vt/orchestrator/logic/orchestrator.go +++ b/go/vt/orchestrator/logic/orchestrator.go @@ -171,7 +171,7 @@ func handleDiscoveryRequests() { continue } - DiscoverInstance(instanceKey) + DiscoverInstance(instanceKey, false) discoveryQueue.Release(instanceKey) } }() @@ -181,7 +181,7 @@ func handleDiscoveryRequests() { // DiscoverInstance will attempt to discover (poll) an instance (unless // it is already up to date) and will also ensure that its primary and // replicas (if any) are also checked. -func DiscoverInstance(instanceKey inst.InstanceKey) { +func DiscoverInstance(instanceKey inst.InstanceKey, forceDiscovery bool) { if inst.InstanceIsForgotten(&instanceKey) { log.Debugf("discoverInstance: skipping discovery of %+v because it is set to be forgotten", instanceKey) return @@ -216,7 +216,7 @@ func DiscoverInstance(instanceKey inst.InstanceKey) { // Calculate the expiry period each time as InstancePollSeconds // _may_ change during the run of the process (via SIGHUP) and // it is not possible to change the cache's default expiry.. - if existsInCacheError := recentDiscoveryOperationKeys.Add(instanceKey.DisplayString(), true, instancePollSecondsDuration()); existsInCacheError != nil { + if existsInCacheError := recentDiscoveryOperationKeys.Add(instanceKey.DisplayString(), true, instancePollSecondsDuration()); existsInCacheError != nil && !forceDiscovery { // Just recently attempted return } @@ -224,7 +224,7 @@ func DiscoverInstance(instanceKey inst.InstanceKey) { latency.Start("backend") instance, found, _ := inst.ReadInstance(&instanceKey) latency.Stop("backend") - if found && instance.IsUpToDate && instance.IsLastCheckValid { + if !forceDiscovery && found && instance.IsUpToDate && instance.IsLastCheckValid { // we've already discovered this one. Skip! return } @@ -506,7 +506,7 @@ func ContinuousDiscovery() { } }() case <-tabletTopoTick: - go RefreshTablets() + go RefreshTablets(false) } } } diff --git a/go/vt/orchestrator/logic/tablet_discovery.go b/go/vt/orchestrator/logic/tablet_discovery.go index fc6f2b50a79..08830b67833 100644 --- a/go/vt/orchestrator/logic/tablet_discovery.go +++ b/go/vt/orchestrator/logic/tablet_discovery.go @@ -67,9 +67,9 @@ func OpenTabletDiscovery() <-chan time.Time { } // RefreshTablets reloads the tablets from topo. -func RefreshTablets() { +func RefreshTablets(forceRefresh bool) { refreshTabletsUsing(func(instanceKey *inst.InstanceKey) { - DiscoverInstance(*instanceKey) + DiscoverInstance(*instanceKey, forceRefresh) }) } diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 78b8da4e711..28ae219253a 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -669,7 +669,10 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat AuditTopologyRecovery(topologyRecovery, value) })).ReparentShard(context.Background(), tablet.Keyspace, tablet.Shard, reparentFunctions) - RefreshTablets() + // here we need to forcefully refresh all the tablets otherwise old information is used and failover scenarios are spawned off which are not required + // For example, if we do not refresh the tablets forcefully and the new primary is found in the cache then its source key is not updated and this spawns off + // PrimaryHasPrimary analysis which runs another ERS + RefreshTablets(true) var promotedReplica *inst.Instance if ev.NewPrimary != nil { promotedReplica, _, _ = inst.ReadInstance(&inst.InstanceKey{ From 485b85de510eb12e774a1b45978082184c3c9b2c Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Sat, 25 Sep 2021 00:16:13 +0530 Subject: [PATCH 142/176] updated functions and added comments Signed-off-by: Manan Gupta --- .../endtoend/vtorc/primary_failure_test.go | 4 ++ .../reparentutil/emergency_reparenter.go | 22 +++++------ go/vt/vtctl/reparentutil/ers_sorter.go | 2 +- go/vt/vtctl/reparentutil/util.go | 16 ++++---- go/vt/vtctl/reparentutil/util_test.go | 38 ++++++++++++++----- 5 files changed, 53 insertions(+), 29 deletions(-) diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index 3ba5ea18af0..b6a94617eb9 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -155,6 +155,8 @@ func TestCrossDataCenterFailureError(t *testing.T) { // covers part of the test case master-failover-lost-replicas from orchestrator func TestLostRdonlyOnPrimaryFailure(t *testing.T) { // new version of ERS does not check for lost replicas yet + // Earlier any replicas that were not able to replicate from the previous primary + // were detected by vtorc and could be configured to have their sources detached t.Skip() defer cluster.PanicHandler(t) setupVttabletsAndVtorc(t, 2, 2, nil, "test_config.json") @@ -278,6 +280,8 @@ func TestPromotionLagSuccess(t *testing.T) { // covers the test case master-failover-fail-promotion-lag-minutes-failure from orchestrator func TestPromotionLagFailure(t *testing.T) { // new version of ERS does not check for promotion lag yet + // Earlier vtorc used to check that the promotion lag between the new primary and the old one + // was smaller than the configured value, otherwise it would fail the promotion t.Skip() defer cluster.PanicHandler(t) setupVttabletsAndVtorc(t, 3, 1, nil, "test_config_promotion_failure.json") diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 2dec0e4254c..ad89453ff7c 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -204,19 +204,19 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } - // find the intermediate primary candidate that we want to replicate from. This will always be the most advanced tablet that we have - // We let all the other tablets replicate from this primary. We will then try to choose a better candidate and let it catch up - var intermediatePrimary *topodatapb.Tablet + // find the intermediate replication source that we want to replicate from. This will always be the most advanced tablet that we have + // We let all the other tablets replicate from this tablet. We will then try to choose a better candidate and let it catch up + var intermediateSource *topodatapb.Tablet var validCandidateTablets []*topodatapb.Tablet - intermediatePrimary, validCandidateTablets, err = findIntermediatePrimaryCandidate(erp.logger, prevPrimary, validCandidates, tabletMap, opts) + intermediateSource, validCandidateTablets, err = findMostAdvanced(erp.logger, prevPrimary, validCandidates, tabletMap, opts) if err != nil { return err } - erp.logger.Infof("intermediate primary selected - %v", intermediatePrimary.Alias) + erp.logger.Infof("intermediate primary selected - %v", intermediateSource.Alias) // check weather the primary candidate selected is ideal or if it can be improved later var isIdeal bool - isIdeal, err = intermediateCandidateIsIdeal(erp.logger, intermediatePrimary, prevPrimary, validCandidateTablets, tabletMap, opts) + isIdeal, err = intermediateCandidateIsIdeal(erp.logger, intermediateSource, prevPrimary, validCandidateTablets, tabletMap, opts) if err != nil { return err } @@ -228,27 +228,27 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // initialize the newPrimary with the intermediate primary, override this value if it is not the ideal candidate - newPrimary := intermediatePrimary + newPrimary := intermediateSource if !isIdeal { // we now promote our intermediate primary candidate and also reparent all the other tablets to start replicating from this candidate // we do not promote the tablet or change the shard record. We only change the replication for all the other tablets // it also returns the list of the tablets that started replication successfully including itself. These are the candidates that we can use to find a replacement var validReplacementCandidates []*topodatapb.Tablet - validReplacementCandidates, err = promoteIntermediatePrimary(ctx, erp.tmc, ev, erp.logger, intermediatePrimary, opts.lockAction, tabletMap, statusMap, opts) + validReplacementCandidates, err = promoteIntermediatePrimary(ctx, erp.tmc, ev, erp.logger, intermediateSource, opts.lockAction, tabletMap, statusMap, opts) if err != nil { return err } // try to find a better candidate using the list we got back var betterCandidate *topodatapb.Tablet - betterCandidate, err = getBetterCandidate(erp.logger, intermediatePrimary, prevPrimary, validReplacementCandidates, tabletMap, opts) + betterCandidate, err = identifyPrimaryCandidate(erp.logger, intermediateSource, prevPrimary, validReplacementCandidates, tabletMap, opts) if err != nil { return err } // if our better candidate is different from our previous candidate, then we wait for it to catch up to the intermediate primary - if !topoproto.TabletAliasEqual(betterCandidate.Alias, intermediatePrimary.Alias) { - err = waitForCatchingUp(ctx, erp.tmc, erp.logger, intermediatePrimary, betterCandidate) + if !topoproto.TabletAliasEqual(betterCandidate.Alias, intermediateSource.Alias) { + err = waitForCatchingUp(ctx, erp.tmc, erp.logger, intermediateSource, betterCandidate) if err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/ers_sorter.go b/go/vt/vtctl/reparentutil/ers_sorter.go index fc287d0f6c0..aabb75ab745 100644 --- a/go/vt/vtctl/reparentutil/ers_sorter.go +++ b/go/vt/vtctl/reparentutil/ers_sorter.go @@ -53,7 +53,7 @@ func (ersSorter *ErsSorter) Swap(i, j int) { // Less implements the Interface for sorting func (ersSorter *ErsSorter) Less(i, j int) bool { - // Returning "true" in this function means [i] is before [j], + // Returning "true" in this function means [i] is before [j] in the sorting order, // which will lead to [i] be a better candidate for promotion // Should not happen diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index c51698fde5e..e4cb0b224b3 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -386,8 +386,8 @@ func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMa if !ok { return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) } - // We do not allow Experimental type of tablets to be considered for replication - if candidateInfo.Type == topodatapb.TabletType_EXPERIMENTAL { + // We do not allow BACKUP, DRAINED or RESTORE type of tablets to be considered for being the replication source or the candidate for primary + if candidateInfo.Type == topodatapb.TabletType_BACKUP || candidateInfo.Type == topodatapb.TabletType_RESTORE || candidateInfo.Type == topodatapb.TabletType_DRAINED { continue } restrictedValidCandidates[candidate] = position @@ -395,8 +395,8 @@ func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMa return restrictedValidCandidates, nil } -// findIntermediatePrimaryCandidate finds the intermediate primary candidate for ERS. We always choose the most advanced one from our valid candidates list -func findIntermediatePrimaryCandidate(logger logutil.Logger, prevPrimary *topodatapb.Tablet, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (*topodatapb.Tablet, []*topodatapb.Tablet, error) { +// findMostAdvanced finds the intermediate primary candidate for ERS. We always choose the most advanced one from our valid candidates list +func findMostAdvanced(logger logutil.Logger, prevPrimary *topodatapb.Tablet, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (*topodatapb.Tablet, []*topodatapb.Tablet, error) { logger.Infof("started finding the intermediate primary candidate") // convert the valid candidates into a list so that we can use it for sorting validTablets, tabletPositions, err := getValidCandidatesAndPositionsAsList(validCandidates, tabletMap) @@ -423,7 +423,7 @@ func findIntermediatePrimaryCandidate(logger logutil.Logger, prevPrimary *topoda winningPosition := tabletPositions[0] // We have already removed the tablets with errant GTIDs before calling this function. At this point our winning position must be a - // superset of all the other valid positions. If that is not the case, then we have a split brain scenario, and we should abort the ERS + // superset of all the other valid positions. If that is not the case, then we have a split brain scenario, and we should cancel the ERS for i, position := range tabletPositions { if !winningPosition.AtLeast(position) { return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "split brain detected between servers - %v and %v", winningPrimaryTablet.Alias, validTablets[i].Alias) @@ -471,15 +471,15 @@ func getValidCandidatesAndPositionsAsList(validCandidates map[string]mysql.Posit // intermediateCandidateIsIdeal is used to find whether the intermediate candidate that ERS chose is also the ideal one or not func intermediateCandidateIsIdeal(logger logutil.Logger, newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (bool, error) { // we try to find a better candidate with the current list of valid candidates, and if it matches our current primary candidate, then we return true - candidate, err := getBetterCandidate(logger, newPrimary, prevPrimary, validCandidates, tabletMap, opts) + candidate, err := identifyPrimaryCandidate(logger, newPrimary, prevPrimary, validCandidates, tabletMap, opts) if err != nil { return false, err } return candidate == newPrimary, nil } -// getBetterCandidate is used to find a better candidate for ERS promotion -func getBetterCandidate(logger logutil.Logger, newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (candidate *topodatapb.Tablet, err error) { +// identifyPrimaryCandidate is used to find a better candidate for ERS promotion +func identifyPrimaryCandidate(logger logutil.Logger, newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (candidate *topodatapb.Tablet, err error) { defer func() { if candidate != nil { logger.Infof("found better candidate - %v", candidate.Alias) diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index 5554b20f8e5..78ac667435a 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -1288,7 +1288,7 @@ func TestPromoteIntermediatePrimary(t *testing.T) { } } -func TestGetBetterCandidate(t *testing.T) { +func TestIdentifyPrimaryCandidate(t *testing.T) { tests := []struct { name string emergencyReparentOps EmergencyReparentOptions @@ -1646,7 +1646,7 @@ func TestGetBetterCandidate(t *testing.T) { t.Run(test.name, func(t *testing.T) { _ = SetDurabilityPolicy("none", nil) logger := logutil.NewMemoryLogger() - res, err := getBetterCandidate(logger, test.newPrimary, test.newPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) + res, err := identifyPrimaryCandidate(logger, test.newPrimary, test.newPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) if test.err != "" { assert.EqualError(t, err, test.err) return @@ -1887,7 +1887,7 @@ func TestWaitForCatchingUp(t *testing.T) { } } -func TestResrictValidCandidates(t *testing.T) { +func TestRestrictValidCandidates(t *testing.T) { tests := []struct { name string validCandidates map[string]mysql.Position @@ -1895,12 +1895,14 @@ func TestResrictValidCandidates(t *testing.T) { result map[string]mysql.Position }{ { - name: "remove experimental tablets", + name: "remove invalid tablets", validCandidates: map[string]mysql.Position{ "zone1-0000000100": {}, "zone1-0000000101": {}, "zone1-0000000102": {}, "zone1-0000000103": {}, + "zone1-0000000104": {}, + "zone1-0000000105": {}, }, tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { @@ -1927,7 +1929,7 @@ func TestResrictValidCandidates(t *testing.T) { Cell: "zone1", Uid: 102, }, - Type: topodatapb.TabletType_EXPERIMENTAL, + Type: topodatapb.TabletType_RESTORE, }, }, "zone1-0000000103": { @@ -1936,14 +1938,32 @@ func TestResrictValidCandidates(t *testing.T) { Cell: "zone1", Uid: 103, }, - Type: topodatapb.TabletType_REPLICA, + Type: topodatapb.TabletType_DRAINED, + }, + }, + "zone1-0000000104": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 104, + }, + Type: topodatapb.TabletType_SPARE, + }, + }, + "zone1-0000000105": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 103, + }, + Type: topodatapb.TabletType_BACKUP, }, }, }, result: map[string]mysql.Position{ "zone1-0000000100": {}, "zone1-0000000101": {}, - "zone1-0000000103": {}, + "zone1-0000000104": {}, }, }, } @@ -1957,7 +1977,7 @@ func TestResrictValidCandidates(t *testing.T) { } } -func TestFindIntermediatePrimaryCandidate(t *testing.T) { +func TestFindMostAdvanced(t *testing.T) { sid1 := mysql.SID{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} mysqlGTID1 := mysql.Mysql56GTID{ Server: sid1, @@ -2275,7 +2295,7 @@ func TestFindIntermediatePrimaryCandidate(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - winningTablet, _, err := findIntermediatePrimaryCandidate(logutil.NewMemoryLogger(), test.prevPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) + winningTablet, _, err := findMostAdvanced(logutil.NewMemoryLogger(), test.prevPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) if test.err != "" { assert.Error(t, err) assert.Contains(t, err.Error(), test.err) From cc8ad44cedadfb60e5b38d0aeb81c9cea3c12b2e Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Sat, 25 Sep 2021 13:38:15 +0530 Subject: [PATCH 143/176] add few more TODOs Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/topology_recovery.go | 7 ++++--- go/vt/vtctl/reparentutil/util.go | 1 + go/vt/vttablet/tabletmanager/rpc_replication.go | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 28ae219253a..08a75d36890 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -641,6 +641,7 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat } // check if we have received an ERS in progress, if we do, we should not continue with the recovery + // TODO - Mutex instead val = atomic.LoadInt32(&ersInProgress) if val > 0 { AuditTopologyRecovery(topologyRecovery, "an ERS is already in progress, not issuing another") @@ -654,7 +655,7 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat atomic.AddInt32(&shardsLockCounter, 1) defer atomic.AddInt32(&shardsLockCounter, -1) - // TODO: Fix durations + // TODO: Introduce wait replicas timeout in configuration reparentFunctions := reparentutil.NewEmergencyReparentOptions(candidateTabletAlias, nil, 1*time.Second, config.Config.PreventCrossDataCenterPrimaryFailover) ev, err := reparentutil.NewEmergencyReparenter(ts, tmclient.NewTabletManagerClient(), logutil.NewCallbackLogger(func(event *logutilpb.Event) { level := event.GetLevel() @@ -662,9 +663,9 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat // we only log the warnings and errors explicitly, everything gets logged as an information message anyways in auditing topology recovery switch level { case logutilpb.Level_WARNING: - log.Warningf("ERP - %s", value) + log.Warningf("ERS - %s", value) case logutilpb.Level_ERROR: - log.Errorf("ERP - %s", value) + log.Errorf("ERS - %s", value) } AuditTopologyRecovery(topologyRecovery, value) })).ReparentShard(context.Background(), tablet.Keyspace, tablet.Shard, reparentFunctions) diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index e4cb0b224b3..58e984a96c6 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -361,6 +361,7 @@ func waitForCatchingUp(ctx context.Context, tmc tmclient.TabletManagerClient, lo // Wait until the new primary has caught upto that position // TODO - discuss, what happens in case of timeout + // TODO - Subcontext -> timeout for wait for position err = tmc.WaitForPosition(ctx, newPrimary, pos) if err != nil { return err diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index ec16d8003b8..4d26a8aa32c 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -526,6 +526,7 @@ func (tm *TabletManager) SetReplicationSource(ctx context.Context, parentAlias * return err } defer tm.unlock() + // TODO - stop semi-sync over here return tm.setReplicationSourceLocked(ctx, parentAlias, timeCreatedNS, waitPosition, forceStartReplication) } From a02b17344a18881dc1bf16700db1d8c02ab08dfd Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 27 Sep 2021 10:26:40 +0530 Subject: [PATCH 144/176] add timeout to catchup phase of ers Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/emergency_reparenter.go | 2 +- go/vt/vtctl/reparentutil/util.go | 8 ++++---- go/vt/vtctl/reparentutil/util_test.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index ad89453ff7c..f5a8b6c57e1 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -248,7 +248,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // if our better candidate is different from our previous candidate, then we wait for it to catch up to the intermediate primary if !topoproto.TabletAliasEqual(betterCandidate.Alias, intermediateSource.Alias) { - err = waitForCatchingUp(ctx, erp.tmc, erp.logger, intermediateSource, betterCandidate) + err = waitForCatchingUp(ctx, erp.tmc, erp.logger, intermediateSource, betterCandidate, opts.waitReplicasTimeout) if err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 58e984a96c6..1e2a02ae456 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -351,7 +351,7 @@ func FindCurrentPrimary(tabletMap map[string]*topo.TabletInfo, logger logutil.Lo } // waitForCatchingUp promotes the newer candidate over the primary candidate that we have, but it does not set to start accepting writes -func waitForCatchingUp(ctx context.Context, tmc tmclient.TabletManagerClient, logger logutil.Logger, prevPrimary, newPrimary *topodatapb.Tablet) error { +func waitForCatchingUp(ctx context.Context, tmc tmclient.TabletManagerClient, logger logutil.Logger, prevPrimary, newPrimary *topodatapb.Tablet, waitTime time.Duration) error { logger.Infof("waiting for %v to catch up to %v", newPrimary.Alias, prevPrimary.Alias) // Find the primary position of the previous primary pos, err := tmc.MasterPosition(ctx, prevPrimary) @@ -360,9 +360,9 @@ func waitForCatchingUp(ctx context.Context, tmc tmclient.TabletManagerClient, lo } // Wait until the new primary has caught upto that position - // TODO - discuss, what happens in case of timeout - // TODO - Subcontext -> timeout for wait for position - err = tmc.WaitForPosition(ctx, newPrimary, pos) + waitForPosCtx, cancelFunc := context.WithTimeout(ctx, waitTime) + defer cancelFunc() + err = tmc.WaitForPosition(waitForPosCtx, newPrimary, pos) if err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index 78ac667435a..d55c0fd418c 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -1877,7 +1877,7 @@ func TestWaitForCatchingUp(t *testing.T) { t.Run(test.name, func(t *testing.T) { ctx := context.Background() logger := logutil.NewMemoryLogger() - err := waitForCatchingUp(ctx, test.tmc, logger, test.prevPrimary, test.newPrimary) + err := waitForCatchingUp(ctx, test.tmc, logger, test.prevPrimary, test.newPrimary, 2*time.Second) if test.err != "" { assert.EqualError(t, err, test.err) } else { From 23cfb1c6d2a90e9fc8fb9ff705dcd93dcf2cbb67 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 27 Sep 2021 10:53:43 +0530 Subject: [PATCH 145/176] handle todo in setReplicationSource Signed-off-by: Manan Gupta --- go/vt/vttablet/tabletmanager/rpc_replication.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index 4d26a8aa32c..6a73b850860 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -526,8 +526,9 @@ func (tm *TabletManager) SetReplicationSource(ctx context.Context, parentAlias * return err } defer tm.unlock() - // TODO - stop semi-sync over here + // setReplicationSourceLocked also fixes the semi-sync. In case the tablet type is primary it assumes that it will become a replica if SetReplicationSource + // is called, so we always call fixSemiSync with a non-primary tablet type. This will always set the source side replication to false. return tm.setReplicationSourceLocked(ctx, parentAlias, timeCreatedNS, waitPosition, forceStartReplication) } From 9e5bacd3171db13287fc6bc6237f9370ef16a4f0 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 27 Sep 2021 12:39:12 +0530 Subject: [PATCH 146/176] add configuration to set wait time and use mutex to lock the ERS Signed-off-by: Manan Gupta --- go/vt/orchestrator/config/config.go | 2 ++ go/vt/orchestrator/logic/orchestrator.go | 3 +- go/vt/orchestrator/logic/topology_recovery.go | 29 ++++++++++++++----- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/go/vt/orchestrator/config/config.go b/go/vt/orchestrator/config/config.go index 86d6dd2a04e..ee2320a3058 100644 --- a/go/vt/orchestrator/config/config.go +++ b/go/vt/orchestrator/config/config.go @@ -236,6 +236,7 @@ type Configuration struct { MaxConcurrentReplicaOperations int // Maximum number of concurrent operations on replicas InstanceDBExecContextTimeoutSeconds int // Timeout on context used while calling ExecContext on instance database LockShardTimeoutSeconds int // Timeout on context used to lock shard. Should be a small value because we should fail-fast + WaitReplicasTimeoutSeconds int // Timeout on amount of time to wait for the replicas in case of ERS. Should be a small value because we should fail-fast. Should not be larger than LockShardTimeoutSeconds since that is the total time we use for an ERS. } // ToJSONString will marshal this configuration as JSON @@ -396,6 +397,7 @@ func newConfiguration() *Configuration { MaxConcurrentReplicaOperations: 5, InstanceDBExecContextTimeoutSeconds: 30, LockShardTimeoutSeconds: 1, + WaitReplicasTimeoutSeconds: 1, } } diff --git a/go/vt/orchestrator/logic/orchestrator.go b/go/vt/orchestrator/logic/orchestrator.go index fff4ee52a1a..5fab2c0dbe8 100644 --- a/go/vt/orchestrator/logic/orchestrator.go +++ b/go/vt/orchestrator/logic/orchestrator.go @@ -52,7 +52,8 @@ var discoveryQueue *discovery.Queue var snapshotDiscoveryKeys chan inst.InstanceKey var snapshotDiscoveryKeysMutex sync.Mutex var hasReceivedSIGTERM int32 -var ersInProgress int32 +var ersInProgressMutex sync.Mutex +var ersInProgress bool var discoveriesCounter = metrics.NewCounter() var failedDiscoveriesCounter = metrics.NewCounter() diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 08a75d36890..60347545951 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -641,22 +641,17 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat } // check if we have received an ERS in progress, if we do, we should not continue with the recovery - // TODO - Mutex instead - val = atomic.LoadInt32(&ersInProgress) - if val > 0 { + if checkAndSetIfERSInProgress() { AuditTopologyRecovery(topologyRecovery, "an ERS is already in progress, not issuing another") return false, topologyRecovery, nil } - // set the ers in progress - atomic.StoreInt32(&ersInProgress, 1) - defer atomic.StoreInt32(&ersInProgress, 0) + defer setERSCompleted() // add to the shard lock counter since ERS will lock the shard atomic.AddInt32(&shardsLockCounter, 1) defer atomic.AddInt32(&shardsLockCounter, -1) - // TODO: Introduce wait replicas timeout in configuration - reparentFunctions := reparentutil.NewEmergencyReparentOptions(candidateTabletAlias, nil, 1*time.Second, config.Config.PreventCrossDataCenterPrimaryFailover) + reparentFunctions := reparentutil.NewEmergencyReparentOptions(candidateTabletAlias, nil, time.Duration(config.Config.WaitReplicasTimeoutSeconds)*time.Second, config.Config.PreventCrossDataCenterPrimaryFailover) ev, err := reparentutil.NewEmergencyReparenter(ts, tmclient.NewTabletManagerClient(), logutil.NewCallbackLogger(func(event *logutilpb.Event) { level := event.GetLevel() value := event.GetValue() @@ -686,6 +681,24 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat return true, topologyRecovery, err } +// checkAndSetIfERSInProgress checks if an ERS is already in progress. If it is not in progress, then we set it to be in progress. +func checkAndSetIfERSInProgress() bool { + ersInProgressMutex.Lock() + defer ersInProgressMutex.Unlock() + if ersInProgress { + return true + } + ersInProgress = true + return false +} + +// setERSCompleted sets the variable tracking if an ers is in progress to false. +func setERSCompleted() { + ersInProgressMutex.Lock() + defer ersInProgressMutex.Unlock() + ersInProgress = false +} + func postErsCompletion(topologyRecovery *TopologyRecovery, analysisEntry inst.ReplicationAnalysis, skipProcesses bool, promotedReplica *inst.Instance) { if promotedReplica != nil { message := fmt.Sprintf("promoted replica: %+v", promotedReplica.Key) From 27519212524d5ac1756b3c4d148915ad8e52a426 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 27 Sep 2021 14:18:16 +0530 Subject: [PATCH 147/176] bug fix in vtorc Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/tablet_discovery.go | 22 +++++++++---------- go/vt/orchestrator/logic/topology_recovery.go | 11 +++++++++- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/go/vt/orchestrator/logic/tablet_discovery.go b/go/vt/orchestrator/logic/tablet_discovery.go index 08830b67833..25a41729aad 100644 --- a/go/vt/orchestrator/logic/tablet_discovery.go +++ b/go/vt/orchestrator/logic/tablet_discovery.go @@ -61,7 +61,7 @@ func OpenTabletDiscovery() <-chan time.Time { } refreshTabletsUsing(func(instanceKey *inst.InstanceKey) { _ = inst.InjectSeed(instanceKey) - }) + }, false) // TODO(sougou): parameterize poll interval. return time.Tick(15 * time.Second) //nolint SA1015: using time.Tick leaks the underlying ticker } @@ -70,10 +70,10 @@ func OpenTabletDiscovery() <-chan time.Time { func RefreshTablets(forceRefresh bool) { refreshTabletsUsing(func(instanceKey *inst.InstanceKey) { DiscoverInstance(*instanceKey, forceRefresh) - }) + }, forceRefresh) } -func refreshTabletsUsing(loader func(instanceKey *inst.InstanceKey)) { +func refreshTabletsUsing(loader func(instanceKey *inst.InstanceKey), forceRefresh bool) { if !IsLeaderOrActive() { return } @@ -93,7 +93,7 @@ func refreshTabletsUsing(loader func(instanceKey *inst.InstanceKey)) { wg.Add(1) go func(cell string) { defer wg.Done() - refreshTabletsInCell(refreshCtx, cell, loader) + refreshTabletsInCell(refreshCtx, cell, loader, forceRefresh) }(cell) } wg.Wait() @@ -136,14 +136,14 @@ func refreshTabletsUsing(loader func(instanceKey *inst.InstanceKey)) { wg.Add(1) go func(ks *topo.KeyspaceShard) { defer wg.Done() - refreshTabletsInKeyspaceShard(refreshCtx, ks.Keyspace, ks.Shard, loader) + refreshTabletsInKeyspaceShard(refreshCtx, ks.Keyspace, ks.Shard, loader, forceRefresh) }(ks) } wg.Wait() } } -func refreshTabletsInCell(ctx context.Context, cell string, loader func(instanceKey *inst.InstanceKey)) { +func refreshTabletsInCell(ctx context.Context, cell string, loader func(instanceKey *inst.InstanceKey), forceRefresh bool) { tablets, err := topotools.GetTabletMapForCell(ctx, ts, cell) if err != nil { log.Errorf("Error fetching topo info for cell %v: %v", cell, err) @@ -151,10 +151,10 @@ func refreshTabletsInCell(ctx context.Context, cell string, loader func(instance } query := "select hostname, port, info from vitess_tablet where cell = ?" args := sqlutils.Args(cell) - refreshTablets(tablets, query, args, loader) + refreshTablets(tablets, query, args, loader, forceRefresh) } -func refreshTabletsInKeyspaceShard(ctx context.Context, keyspace, shard string, loader func(instanceKey *inst.InstanceKey)) { +func refreshTabletsInKeyspaceShard(ctx context.Context, keyspace, shard string, loader func(instanceKey *inst.InstanceKey), forceRefresh bool) { tablets, err := ts.GetTabletMapForShard(ctx, keyspace, shard) if err != nil { log.Errorf("Error fetching tablets for keyspace/shard %v/%v: %v", keyspace, shard, err) @@ -162,10 +162,10 @@ func refreshTabletsInKeyspaceShard(ctx context.Context, keyspace, shard string, } query := "select hostname, port, info from vitess_tablet where keyspace = ? and shard = ?" args := sqlutils.Args(keyspace, shard) - refreshTablets(tablets, query, args, loader) + refreshTablets(tablets, query, args, loader, forceRefresh) } -func refreshTablets(tablets map[string]*topo.TabletInfo, query string, args []interface{}, loader func(instanceKey *inst.InstanceKey)) { +func refreshTablets(tablets map[string]*topo.TabletInfo, query string, args []interface{}, loader func(instanceKey *inst.InstanceKey), forceRefresh bool) { // Discover new tablets. // TODO(sougou): enhance this to work with multi-schema, // where each instanceKey can have multiple tablets. @@ -188,7 +188,7 @@ func refreshTablets(tablets map[string]*topo.TabletInfo, query string, args []in log.Errore(err) continue } - if proto.Equal(tablet, old) { + if !forceRefresh && proto.Equal(tablet, old) { continue } if err := inst.SaveTablet(tablet); err != nil { diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 60347545951..73375fa05d8 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -614,7 +614,7 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat if !(forceInstanceRecovery || analysisEntry.ClusterDetails.HasAutomatedPrimaryRecovery) { return false, nil, nil } - tablet, err := inst.ReadTablet(analysisEntry.AnalyzedInstanceKey) + tablet, err := TabletRefresh(analysisEntry.AnalyzedInstanceKey) if err != nil { return false, nil, err } @@ -634,6 +634,15 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat } log.Infof("Analysis: %v, deadprimary %+v", analysisEntry.Analysis, analysisEntry.AnalyzedInstanceKey) + // this check is needed because sometimes DeadPrimary code path is forcefully spawned off from other recoveries like PrimaryHasPrimary. + // So we need to check that we only run an ERS if the instance that we analyzed was actually a primary! Otherwise, we would end up running an ERS + // even when the cluster is fine or the problem can be fixed via some other recovery + if tablet.Type != topodatapb.TabletType_PRIMARY { + RefreshTablets(true) + AuditTopologyRecovery(topologyRecovery, "another agent seems to have fixed the problem") + return false, topologyRecovery, nil + } + // check if we have received SIGTERM, if we have, we should not continue with the recovery val := atomic.LoadInt32(&hasReceivedSIGTERM) if val > 0 { From eac76e64ee664fc1efd5e865050c084ae103af5b Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 27 Sep 2021 17:03:50 +0530 Subject: [PATCH 148/176] added for all the durability policies Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/durability_test.go | 125 ++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/go/vt/vtctl/reparentutil/durability_test.go b/go/vt/vtctl/reparentutil/durability_test.go index 7c7649c7056..f6a4c58d68e 100644 --- a/go/vt/vtctl/reparentutil/durability_test.go +++ b/go/vt/vtctl/reparentutil/durability_test.go @@ -19,6 +19,8 @@ package reparentutil import ( "testing" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/vt/topo/topoproto" "github.com/stretchr/testify/assert" @@ -26,6 +28,129 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) +func TestDurabilityNone(t *testing.T) { + err := SetDurabilityPolicy("none", nil) + require.NoError(t, err) + + promoteRule := PromotionRule(&topodatapb.Tablet{ + Type: topodatapb.TabletType_PRIMARY, + }) + require.Equal(t, NeutralPromoteRule, promoteRule) + + promoteRule = PromotionRule(&topodatapb.Tablet{ + Type: topodatapb.TabletType_REPLICA, + }) + require.Equal(t, NeutralPromoteRule, promoteRule) + + promoteRule = PromotionRule(&topodatapb.Tablet{ + Type: topodatapb.TabletType_RDONLY, + }) + require.Equal(t, MustNotPromoteRule, promoteRule) + + promoteRule = PromotionRule(&topodatapb.Tablet{ + Type: topodatapb.TabletType_SPARE, + }) + require.Equal(t, MustNotPromoteRule, promoteRule) + require.Equal(t, 0, PrimarySemiSyncFromTablet(nil)) + require.Equal(t, false, ReplicaSemiSyncFromTablet(nil, nil)) +} + +func TestDurabilitySemiSync(t *testing.T) { + err := SetDurabilityPolicy("semi_sync", nil) + require.NoError(t, err) + + promoteRule := PromotionRule(&topodatapb.Tablet{ + Type: topodatapb.TabletType_PRIMARY, + }) + require.Equal(t, NeutralPromoteRule, promoteRule) + + promoteRule = PromotionRule(&topodatapb.Tablet{ + Type: topodatapb.TabletType_REPLICA, + }) + require.Equal(t, NeutralPromoteRule, promoteRule) + + promoteRule = PromotionRule(&topodatapb.Tablet{ + Type: topodatapb.TabletType_RDONLY, + }) + require.Equal(t, MustNotPromoteRule, promoteRule) + + promoteRule = PromotionRule(&topodatapb.Tablet{ + Type: topodatapb.TabletType_SPARE, + }) + require.Equal(t, MustNotPromoteRule, promoteRule) + require.Equal(t, 1, PrimarySemiSyncFromTablet(nil)) + require.Equal(t, true, ReplicaSemiSyncFromTablet(nil, &topodatapb.Tablet{ + Type: topodatapb.TabletType_REPLICA, + })) + require.Equal(t, false, ReplicaSemiSyncFromTablet(nil, &topodatapb.Tablet{ + Type: topodatapb.TabletType_EXPERIMENTAL, + })) +} + +func TestDurabilityCrossCell(t *testing.T) { + err := SetDurabilityPolicy("cross_cell", nil) + require.NoError(t, err) + + promoteRule := PromotionRule(&topodatapb.Tablet{ + Type: topodatapb.TabletType_PRIMARY, + }) + require.Equal(t, NeutralPromoteRule, promoteRule) + + promoteRule = PromotionRule(&topodatapb.Tablet{ + Type: topodatapb.TabletType_REPLICA, + }) + require.Equal(t, NeutralPromoteRule, promoteRule) + + promoteRule = PromotionRule(&topodatapb.Tablet{ + Type: topodatapb.TabletType_RDONLY, + }) + require.Equal(t, MustNotPromoteRule, promoteRule) + + promoteRule = PromotionRule(&topodatapb.Tablet{ + Type: topodatapb.TabletType_SPARE, + }) + require.Equal(t, MustNotPromoteRule, promoteRule) + require.Equal(t, 1, PrimarySemiSyncFromTablet(nil)) + require.Equal(t, false, ReplicaSemiSyncFromTablet(&topodatapb.Tablet{ + Type: topodatapb.TabletType_PRIMARY, + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + }, + }, &topodatapb.Tablet{ + Type: topodatapb.TabletType_REPLICA, + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + }, + })) + require.Equal(t, true, ReplicaSemiSyncFromTablet(&topodatapb.Tablet{ + Type: topodatapb.TabletType_PRIMARY, + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + }, + }, &topodatapb.Tablet{ + Type: topodatapb.TabletType_REPLICA, + Alias: &topodatapb.TabletAlias{ + Cell: "cell2", + }, + })) + require.Equal(t, false, ReplicaSemiSyncFromTablet(&topodatapb.Tablet{ + Type: topodatapb.TabletType_PRIMARY, + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + }, + }, &topodatapb.Tablet{ + Type: topodatapb.TabletType_EXPERIMENTAL, + Alias: &topodatapb.TabletAlias{ + Cell: "cell2", + }, + })) +} + +func TestError(t *testing.T) { + err := SetDurabilityPolicy("unknown", nil) + require.EqualError(t, err, "durability policy unknown not found") +} + func TestDurabilitySpecified(t *testing.T) { cellName := "cell" durabilityRules := newDurabilitySpecified( From f5d14e0e06c1f631a4d74db462b31030d7bd1327 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 27 Sep 2021 17:20:29 +0530 Subject: [PATCH 149/176] added unit tests for contraint failure to ers tests Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter_test.go | 252 +++++++++++++++++- 1 file changed, 251 insertions(+), 1 deletion(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index e174aebcf57..7de63cfb2e8 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -427,6 +427,12 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { { Keyspace: "testkeyspace", Name: "-", + Shard: &topodatapb.Shard{ + PrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, }, }, tablets: []*topodatapb.Tablet{ @@ -1091,6 +1097,248 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { shouldErr: true, errShouldContain: "failed to be upgraded to primary", }, + { + name: "constraint failure - promotion-rule", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000102": nil, + }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000102": { + Result: "ok", + Error: nil, + }, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000102": { + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000100": nil, + "zone1-0000000101": nil, + }, + StopReplicationAndGetStatusResults: map[string]struct { + Status *replicationdatapb.Status + StopStatus *replicationdatapb.StopReplicationStatus + Error error + }{ + "zone1-0000000100": { + StopStatus: &replicationdatapb.StopReplicationStatus{ + Before: &replicationdatapb.Status{}, + After: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", + }, + }, + }, + "zone1-0000000101": { + StopStatus: &replicationdatapb.StopReplicationStatus{ + Before: &replicationdatapb.Status{}, + After: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", + }, + }, + }, + "zone1-0000000102": { + StopStatus: &replicationdatapb.StopReplicationStatus{ + Before: &replicationdatapb.Status{}, + After: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-26", + }, + }, + }, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000100": { + "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21": nil, + }, + "zone1-0000000101": { + "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21": nil, + }, + "zone1-0000000102": { + "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-26": nil, + }, + }, + }, + shards: []*vtctldatapb.Shard{ + { + Keyspace: "testkeyspace", + Name: "-", + }, + }, + tablets: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + Keyspace: "testkeyspace", + Shard: "-", + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + Type: topodatapb.TabletType_RDONLY, + Keyspace: "testkeyspace", + Shard: "-", + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + Keyspace: "testkeyspace", + Shard: "-", + Hostname: "most up-to-date position, wins election", + }, + }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: "elected primary does not satisfy promotion rule constraint", + }, + { + name: "constraint failure - cross-cell", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, true), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000102": nil, + }, + PromoteReplicaResults: map[string]struct { + Result string + Error error + }{ + "zone1-0000000102": { + Result: "ok", + Error: nil, + }, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000102": { + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000100": nil, + "zone1-0000000101": nil, + }, + StopReplicationAndGetStatusResults: map[string]struct { + Status *replicationdatapb.Status + StopStatus *replicationdatapb.StopReplicationStatus + Error error + }{ + "zone1-0000000100": { + StopStatus: &replicationdatapb.StopReplicationStatus{ + Before: &replicationdatapb.Status{}, + After: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", + }, + }, + }, + "zone1-0000000101": { + StopStatus: &replicationdatapb.StopReplicationStatus{ + Before: &replicationdatapb.Status{}, + After: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", + }, + }, + }, + "zone1-0000000102": { + StopStatus: &replicationdatapb.StopReplicationStatus{ + Before: &replicationdatapb.Status{}, + After: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-26", + }, + }, + }, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000100": { + "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21": nil, + }, + "zone1-0000000101": { + "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21": nil, + }, + "zone1-0000000102": { + "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-26": nil, + }, + }, + }, + shards: []*vtctldatapb.Shard{ + { + Keyspace: "testkeyspace", + Name: "-", + Shard: &topodatapb.Shard{ + PrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 100, + }, + }, + }, + }, + tablets: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Keyspace: "testkeyspace", + Shard: "-", + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + Keyspace: "testkeyspace", + Shard: "-", + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Keyspace: "testkeyspace", + Shard: "-", + Hostname: "most up-to-date position, wins election", + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 100, + }, + Keyspace: "testkeyspace", + Shard: "-", + Hostname: "failed previous primary", + }, + }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1", "zone2"), + shouldErr: true, + errShouldContain: "elected primary does not satisfy geographic constraint", + }, } _ = SetDurabilityPolicy("none", nil) @@ -1105,7 +1353,9 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { ev := &events.Reparent{} for i, tablet := range tt.tablets { - tablet.Type = topodatapb.TabletType_REPLICA + if tablet.Type == topodatapb.TabletType_UNKNOWN { + tablet.Type = topodatapb.TabletType_REPLICA + } tt.tablets[i] = tablet } From aa09afd85ff2a5a57fc533a07436b1bfe0ef5172 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 27 Sep 2021 18:13:55 +0530 Subject: [PATCH 150/176] do not cancel replication context until all the replicas are done Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/util.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 1e2a02ae456..e2eabd7a26d 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -107,7 +107,6 @@ func reparentReplicas(ctx context.Context, ev *events.Reparent, logger logutil.L var replicaMutex sync.Mutex replCtx, replCancel := context.WithTimeout(ctx, opts.waitReplicasTimeout) - defer replCancel() event.DispatchUpdate(ev, "reparenting all tablets") @@ -211,6 +210,11 @@ func reparentReplicas(ctx context.Context, ev *events.Reparent, logger logutil.L return nil, vterrors.Wrapf(primaryErr, "failed to PopulateReparentJournal on primary: %v", primaryErr) } + go func() { + replWg.Wait() + defer replCancel() + }() + select { case <-replSuccessCtx.Done(): // At least one replica was able to SetMaster successfully From 3bf347972ddb2b37673f02819432db0f4543930f Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 10:58:43 +0530 Subject: [PATCH 151/176] rename test package from prs to plannedreparent Signed-off-by: Manan Gupta --- .../endtoend/reparent/{ers => emergencyreparent}/ers_test.go | 2 +- .../endtoend/reparent/{ers => emergencyreparent}/utils_test.go | 2 +- .../{prs => plannedreparent}/reparent_range_based_test.go | 2 +- .../endtoend/reparent/{prs => plannedreparent}/reparent_test.go | 2 +- .../endtoend/reparent/{prs => plannedreparent}/utils_test.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename go/test/endtoend/reparent/{ers => emergencyreparent}/ers_test.go (99%) rename go/test/endtoend/reparent/{ers => emergencyreparent}/utils_test.go (99%) rename go/test/endtoend/reparent/{prs => plannedreparent}/reparent_range_based_test.go (98%) rename go/test/endtoend/reparent/{prs => plannedreparent}/reparent_test.go (99%) rename go/test/endtoend/reparent/{prs => plannedreparent}/utils_test.go (99%) diff --git a/go/test/endtoend/reparent/ers/ers_test.go b/go/test/endtoend/reparent/emergencyreparent/ers_test.go similarity index 99% rename from go/test/endtoend/reparent/ers/ers_test.go rename to go/test/endtoend/reparent/emergencyreparent/ers_test.go index 03345b199cf..e28ca892c40 100644 --- a/go/test/endtoend/reparent/ers/ers_test.go +++ b/go/test/endtoend/reparent/emergencyreparent/ers_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ers +package emergencyreparent import ( "context" diff --git a/go/test/endtoend/reparent/ers/utils_test.go b/go/test/endtoend/reparent/emergencyreparent/utils_test.go similarity index 99% rename from go/test/endtoend/reparent/ers/utils_test.go rename to go/test/endtoend/reparent/emergencyreparent/utils_test.go index 3f96d205031..15761ef6a03 100644 --- a/go/test/endtoend/reparent/ers/utils_test.go +++ b/go/test/endtoend/reparent/emergencyreparent/utils_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ers +package emergencyreparent import ( "context" diff --git a/go/test/endtoend/reparent/prs/reparent_range_based_test.go b/go/test/endtoend/reparent/plannedreparent/reparent_range_based_test.go similarity index 98% rename from go/test/endtoend/reparent/prs/reparent_range_based_test.go rename to go/test/endtoend/reparent/plannedreparent/reparent_range_based_test.go index bd092da6920..b3911de21ba 100644 --- a/go/test/endtoend/reparent/prs/reparent_range_based_test.go +++ b/go/test/endtoend/reparent/plannedreparent/reparent_range_based_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package prs +package plannedreparent import ( "context" diff --git a/go/test/endtoend/reparent/prs/reparent_test.go b/go/test/endtoend/reparent/plannedreparent/reparent_test.go similarity index 99% rename from go/test/endtoend/reparent/prs/reparent_test.go rename to go/test/endtoend/reparent/plannedreparent/reparent_test.go index a7e78f95f8d..f16695fae93 100644 --- a/go/test/endtoend/reparent/prs/reparent_test.go +++ b/go/test/endtoend/reparent/plannedreparent/reparent_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package prs +package plannedreparent import ( "context" diff --git a/go/test/endtoend/reparent/prs/utils_test.go b/go/test/endtoend/reparent/plannedreparent/utils_test.go similarity index 99% rename from go/test/endtoend/reparent/prs/utils_test.go rename to go/test/endtoend/reparent/plannedreparent/utils_test.go index 6c4b8211bda..2c8819fb2c5 100644 --- a/go/test/endtoend/reparent/prs/utils_test.go +++ b/go/test/endtoend/reparent/plannedreparent/utils_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package prs +package plannedreparent import ( "context" From ce8dd2f557123b8eba6932febe40880535f0cdde Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 11:01:09 +0530 Subject: [PATCH 152/176] use assert in tests where we do not want to abort in case of failure Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/durability_test.go | 44 ++++++++++----------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/go/vt/vtctl/reparentutil/durability_test.go b/go/vt/vtctl/reparentutil/durability_test.go index f6a4c58d68e..4dfd09f6aee 100644 --- a/go/vt/vtctl/reparentutil/durability_test.go +++ b/go/vt/vtctl/reparentutil/durability_test.go @@ -35,24 +35,24 @@ func TestDurabilityNone(t *testing.T) { promoteRule := PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, }) - require.Equal(t, NeutralPromoteRule, promoteRule) + assert.Equal(t, NeutralPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_REPLICA, }) - require.Equal(t, NeutralPromoteRule, promoteRule) + assert.Equal(t, NeutralPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_RDONLY, }) - require.Equal(t, MustNotPromoteRule, promoteRule) + assert.Equal(t, MustNotPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_SPARE, }) - require.Equal(t, MustNotPromoteRule, promoteRule) - require.Equal(t, 0, PrimarySemiSyncFromTablet(nil)) - require.Equal(t, false, ReplicaSemiSyncFromTablet(nil, nil)) + assert.Equal(t, MustNotPromoteRule, promoteRule) + assert.Equal(t, 0, PrimarySemiSyncFromTablet(nil)) + assert.Equal(t, false, ReplicaSemiSyncFromTablet(nil, nil)) } func TestDurabilitySemiSync(t *testing.T) { @@ -62,27 +62,27 @@ func TestDurabilitySemiSync(t *testing.T) { promoteRule := PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, }) - require.Equal(t, NeutralPromoteRule, promoteRule) + assert.Equal(t, NeutralPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_REPLICA, }) - require.Equal(t, NeutralPromoteRule, promoteRule) + assert.Equal(t, NeutralPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_RDONLY, }) - require.Equal(t, MustNotPromoteRule, promoteRule) + assert.Equal(t, MustNotPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_SPARE, }) - require.Equal(t, MustNotPromoteRule, promoteRule) - require.Equal(t, 1, PrimarySemiSyncFromTablet(nil)) - require.Equal(t, true, ReplicaSemiSyncFromTablet(nil, &topodatapb.Tablet{ + assert.Equal(t, MustNotPromoteRule, promoteRule) + assert.Equal(t, 1, PrimarySemiSyncFromTablet(nil)) + assert.Equal(t, true, ReplicaSemiSyncFromTablet(nil, &topodatapb.Tablet{ Type: topodatapb.TabletType_REPLICA, })) - require.Equal(t, false, ReplicaSemiSyncFromTablet(nil, &topodatapb.Tablet{ + assert.Equal(t, false, ReplicaSemiSyncFromTablet(nil, &topodatapb.Tablet{ Type: topodatapb.TabletType_EXPERIMENTAL, })) } @@ -94,24 +94,24 @@ func TestDurabilityCrossCell(t *testing.T) { promoteRule := PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, }) - require.Equal(t, NeutralPromoteRule, promoteRule) + assert.Equal(t, NeutralPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_REPLICA, }) - require.Equal(t, NeutralPromoteRule, promoteRule) + assert.Equal(t, NeutralPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_RDONLY, }) - require.Equal(t, MustNotPromoteRule, promoteRule) + assert.Equal(t, MustNotPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_SPARE, }) - require.Equal(t, MustNotPromoteRule, promoteRule) - require.Equal(t, 1, PrimarySemiSyncFromTablet(nil)) - require.Equal(t, false, ReplicaSemiSyncFromTablet(&topodatapb.Tablet{ + assert.Equal(t, MustNotPromoteRule, promoteRule) + assert.Equal(t, 1, PrimarySemiSyncFromTablet(nil)) + assert.Equal(t, false, ReplicaSemiSyncFromTablet(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, Alias: &topodatapb.TabletAlias{ Cell: "cell1", @@ -122,7 +122,7 @@ func TestDurabilityCrossCell(t *testing.T) { Cell: "cell1", }, })) - require.Equal(t, true, ReplicaSemiSyncFromTablet(&topodatapb.Tablet{ + assert.Equal(t, true, ReplicaSemiSyncFromTablet(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, Alias: &topodatapb.TabletAlias{ Cell: "cell1", @@ -133,7 +133,7 @@ func TestDurabilityCrossCell(t *testing.T) { Cell: "cell2", }, })) - require.Equal(t, false, ReplicaSemiSyncFromTablet(&topodatapb.Tablet{ + assert.Equal(t, false, ReplicaSemiSyncFromTablet(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, Alias: &topodatapb.TabletAlias{ Cell: "cell1", @@ -148,7 +148,7 @@ func TestDurabilityCrossCell(t *testing.T) { func TestError(t *testing.T) { err := SetDurabilityPolicy("unknown", nil) - require.EqualError(t, err, "durability policy unknown not found") + assert.EqualError(t, err, "durability policy unknown not found") } func TestDurabilitySpecified(t *testing.T) { From 890349e97b6293cafb37eb258e7752e18e335d44 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 11:07:12 +0530 Subject: [PATCH 153/176] ran make vtctldclient vtadmin_web_proto_types Signed-off-by: Manan Gupta --- web/vtadmin/src/proto/vtadmin.d.ts | 12 ++++++ web/vtadmin/src/proto/vtadmin.js | 60 ++++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/web/vtadmin/src/proto/vtadmin.d.ts b/web/vtadmin/src/proto/vtadmin.d.ts index 6e250c4e9c1..a37a067704c 100644 --- a/web/vtadmin/src/proto/vtadmin.d.ts +++ b/web/vtadmin/src/proto/vtadmin.d.ts @@ -17011,6 +17011,9 @@ export namespace tabletmanagerdata { /** Properties of a RestoreFromBackupRequest. */ interface IRestoreFromBackupRequest { + + /** RestoreFromBackupRequest backup_time */ + backup_time?: (vttime.ITime|null); } /** Represents a RestoreFromBackupRequest. */ @@ -17022,6 +17025,9 @@ export namespace tabletmanagerdata { */ constructor(properties?: tabletmanagerdata.IRestoreFromBackupRequest); + /** RestoreFromBackupRequest backup_time. */ + public backup_time?: (vttime.ITime|null); + /** * Creates a new RestoreFromBackupRequest instance using the specified properties. * @param [properties] Properties to set @@ -29345,6 +29351,9 @@ export namespace vtctldata { /** EmergencyReparentShardRequest wait_replicas_timeout */ wait_replicas_timeout?: (vttime.IDuration|null); + + /** EmergencyReparentShardRequest prevent_cross_cell_promotion */ + prevent_cross_cell_promotion?: (boolean|null); } /** Represents an EmergencyReparentShardRequest. */ @@ -29371,6 +29380,9 @@ export namespace vtctldata { /** EmergencyReparentShardRequest wait_replicas_timeout. */ public wait_replicas_timeout?: (vttime.IDuration|null); + /** EmergencyReparentShardRequest prevent_cross_cell_promotion. */ + public prevent_cross_cell_promotion: boolean; + /** * Creates a new EmergencyReparentShardRequest instance using the specified properties. * @param [properties] Properties to set diff --git a/web/vtadmin/src/proto/vtadmin.js b/web/vtadmin/src/proto/vtadmin.js index 9933ccab90f..1e33d03603d 100644 --- a/web/vtadmin/src/proto/vtadmin.js +++ b/web/vtadmin/src/proto/vtadmin.js @@ -38504,6 +38504,7 @@ $root.tabletmanagerdata = (function() { * Properties of a RestoreFromBackupRequest. * @memberof tabletmanagerdata * @interface IRestoreFromBackupRequest + * @property {vttime.ITime|null} [backup_time] RestoreFromBackupRequest backup_time */ /** @@ -38521,6 +38522,14 @@ $root.tabletmanagerdata = (function() { this[keys[i]] = properties[keys[i]]; } + /** + * RestoreFromBackupRequest backup_time. + * @member {vttime.ITime|null|undefined} backup_time + * @memberof tabletmanagerdata.RestoreFromBackupRequest + * @instance + */ + RestoreFromBackupRequest.prototype.backup_time = null; + /** * Creates a new RestoreFromBackupRequest instance using the specified properties. * @function create @@ -38545,6 +38554,8 @@ $root.tabletmanagerdata = (function() { RestoreFromBackupRequest.encode = function encode(message, writer) { if (!writer) writer = $Writer.create(); + if (message.backup_time != null && Object.hasOwnProperty.call(message, "backup_time")) + $root.vttime.Time.encode(message.backup_time, writer.uint32(/* id 1, wireType 2 =*/10).fork()).ldelim(); return writer; }; @@ -38579,6 +38590,9 @@ $root.tabletmanagerdata = (function() { while (reader.pos < end) { var tag = reader.uint32(); switch (tag >>> 3) { + case 1: + message.backup_time = $root.vttime.Time.decode(reader, reader.uint32()); + break; default: reader.skipType(tag & 7); break; @@ -38614,6 +38628,11 @@ $root.tabletmanagerdata = (function() { RestoreFromBackupRequest.verify = function verify(message) { if (typeof message !== "object" || message === null) return "object expected"; + if (message.backup_time != null && message.hasOwnProperty("backup_time")) { + var error = $root.vttime.Time.verify(message.backup_time); + if (error) + return "backup_time." + error; + } return null; }; @@ -38628,7 +38647,13 @@ $root.tabletmanagerdata = (function() { RestoreFromBackupRequest.fromObject = function fromObject(object) { if (object instanceof $root.tabletmanagerdata.RestoreFromBackupRequest) return object; - return new $root.tabletmanagerdata.RestoreFromBackupRequest(); + var message = new $root.tabletmanagerdata.RestoreFromBackupRequest(); + if (object.backup_time != null) { + if (typeof object.backup_time !== "object") + throw TypeError(".tabletmanagerdata.RestoreFromBackupRequest.backup_time: object expected"); + message.backup_time = $root.vttime.Time.fromObject(object.backup_time); + } + return message; }; /** @@ -38640,8 +38665,15 @@ $root.tabletmanagerdata = (function() { * @param {$protobuf.IConversionOptions} [options] Conversion options * @returns {Object.} Plain object */ - RestoreFromBackupRequest.toObject = function toObject() { - return {}; + RestoreFromBackupRequest.toObject = function toObject(message, options) { + if (!options) + options = {}; + var object = {}; + if (options.defaults) + object.backup_time = null; + if (message.backup_time != null && message.hasOwnProperty("backup_time")) + object.backup_time = $root.vttime.Time.toObject(message.backup_time, options); + return object; }; /** @@ -69762,6 +69794,7 @@ $root.vtctldata = (function() { * @property {topodata.ITabletAlias|null} [new_primary] EmergencyReparentShardRequest new_primary * @property {Array.|null} [ignore_replicas] EmergencyReparentShardRequest ignore_replicas * @property {vttime.IDuration|null} [wait_replicas_timeout] EmergencyReparentShardRequest wait_replicas_timeout + * @property {boolean|null} [prevent_cross_cell_promotion] EmergencyReparentShardRequest prevent_cross_cell_promotion */ /** @@ -69820,6 +69853,14 @@ $root.vtctldata = (function() { */ EmergencyReparentShardRequest.prototype.wait_replicas_timeout = null; + /** + * EmergencyReparentShardRequest prevent_cross_cell_promotion. + * @member {boolean} prevent_cross_cell_promotion + * @memberof vtctldata.EmergencyReparentShardRequest + * @instance + */ + EmergencyReparentShardRequest.prototype.prevent_cross_cell_promotion = false; + /** * Creates a new EmergencyReparentShardRequest instance using the specified properties. * @function create @@ -69855,6 +69896,8 @@ $root.vtctldata = (function() { $root.topodata.TabletAlias.encode(message.ignore_replicas[i], writer.uint32(/* id 4, wireType 2 =*/34).fork()).ldelim(); if (message.wait_replicas_timeout != null && Object.hasOwnProperty.call(message, "wait_replicas_timeout")) $root.vttime.Duration.encode(message.wait_replicas_timeout, writer.uint32(/* id 5, wireType 2 =*/42).fork()).ldelim(); + if (message.prevent_cross_cell_promotion != null && Object.hasOwnProperty.call(message, "prevent_cross_cell_promotion")) + writer.uint32(/* id 6, wireType 0 =*/48).bool(message.prevent_cross_cell_promotion); return writer; }; @@ -69906,6 +69949,9 @@ $root.vtctldata = (function() { case 5: message.wait_replicas_timeout = $root.vttime.Duration.decode(reader, reader.uint32()); break; + case 6: + message.prevent_cross_cell_promotion = reader.bool(); + break; default: reader.skipType(tag & 7); break; @@ -69966,6 +70012,9 @@ $root.vtctldata = (function() { if (error) return "wait_replicas_timeout." + error; } + if (message.prevent_cross_cell_promotion != null && message.hasOwnProperty("prevent_cross_cell_promotion")) + if (typeof message.prevent_cross_cell_promotion !== "boolean") + return "prevent_cross_cell_promotion: boolean expected"; return null; }; @@ -70005,6 +70054,8 @@ $root.vtctldata = (function() { throw TypeError(".vtctldata.EmergencyReparentShardRequest.wait_replicas_timeout: object expected"); message.wait_replicas_timeout = $root.vttime.Duration.fromObject(object.wait_replicas_timeout); } + if (object.prevent_cross_cell_promotion != null) + message.prevent_cross_cell_promotion = Boolean(object.prevent_cross_cell_promotion); return message; }; @@ -70028,6 +70079,7 @@ $root.vtctldata = (function() { object.shard = ""; object.new_primary = null; object.wait_replicas_timeout = null; + object.prevent_cross_cell_promotion = false; } if (message.keyspace != null && message.hasOwnProperty("keyspace")) object.keyspace = message.keyspace; @@ -70042,6 +70094,8 @@ $root.vtctldata = (function() { } if (message.wait_replicas_timeout != null && message.hasOwnProperty("wait_replicas_timeout")) object.wait_replicas_timeout = $root.vttime.Duration.toObject(message.wait_replicas_timeout, options); + if (message.prevent_cross_cell_promotion != null && message.hasOwnProperty("prevent_cross_cell_promotion")) + object.prevent_cross_cell_promotion = message.prevent_cross_cell_promotion; return object; }; From 0d4a6bf2f945c2385a9379ce9f7e36dcfee990ea Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 11:15:34 +0530 Subject: [PATCH 154/176] fix naming for ERSSorter Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/ers_sorter.go | 16 ++++++++-------- .../{ers_sorted_test.go => ers_sorter_test.go} | 0 2 files changed, 8 insertions(+), 8 deletions(-) rename go/vt/vtctl/reparentutil/{ers_sorted_test.go => ers_sorter_test.go} (100%) diff --git a/go/vt/vtctl/reparentutil/ers_sorter.go b/go/vt/vtctl/reparentutil/ers_sorter.go index aabb75ab745..832b4328dea 100644 --- a/go/vt/vtctl/reparentutil/ers_sorter.go +++ b/go/vt/vtctl/reparentutil/ers_sorter.go @@ -25,17 +25,17 @@ import ( "vitess.io/vitess/go/vt/vterrors" ) -// ErsSorter sorts tablets by GTID positions and Promotion rules aimed at finding the best +// ERSSorter sorts tablets by GTID positions and Promotion rules aimed at finding the best // candidate for intermediate promotion in emergency reparent shard -type ErsSorter struct { +type ERSSorter struct { tablets []*topodatapb.Tablet positions []mysql.Position idealCell string } -// NewErsSorter creates a new ErsSorter -func NewErsSorter(tablets []*topodatapb.Tablet, positions []mysql.Position, idealCell string) *ErsSorter { - return &ErsSorter{ +// NewErsSorter creates a new ERSSorter +func NewErsSorter(tablets []*topodatapb.Tablet, positions []mysql.Position, idealCell string) *ERSSorter { + return &ERSSorter{ tablets: tablets, positions: positions, idealCell: idealCell, @@ -43,16 +43,16 @@ func NewErsSorter(tablets []*topodatapb.Tablet, positions []mysql.Position, idea } // Len implements the Interface for sorting -func (ersSorter *ErsSorter) Len() int { return len(ersSorter.tablets) } +func (ersSorter *ERSSorter) Len() int { return len(ersSorter.tablets) } // Swap implements the Interface for sorting -func (ersSorter *ErsSorter) Swap(i, j int) { +func (ersSorter *ERSSorter) Swap(i, j int) { ersSorter.tablets[i], ersSorter.tablets[j] = ersSorter.tablets[j], ersSorter.tablets[i] ersSorter.positions[i], ersSorter.positions[j] = ersSorter.positions[j], ersSorter.positions[i] } // Less implements the Interface for sorting -func (ersSorter *ErsSorter) Less(i, j int) bool { +func (ersSorter *ERSSorter) Less(i, j int) bool { // Returning "true" in this function means [i] is before [j] in the sorting order, // which will lead to [i] be a better candidate for promotion diff --git a/go/vt/vtctl/reparentutil/ers_sorted_test.go b/go/vt/vtctl/reparentutil/ers_sorter_test.go similarity index 100% rename from go/vt/vtctl/reparentutil/ers_sorted_test.go rename to go/vt/vtctl/reparentutil/ers_sorter_test.go From 6b00bccafc96fb3774eaf61d5e28de0d6b41240e Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 11:18:00 +0530 Subject: [PATCH 155/176] fix config test file to reflect change to package name Signed-off-by: Manan Gupta --- test/config.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/config.json b/test/config.json index d3d399e1e56..57bad4b3ecd 100644 --- a/test/config.json +++ b/test/config.json @@ -413,18 +413,18 @@ "RetryMax": 1, "Tags": [] }, - "ers": { + "emergencyreparent": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/reparent/ers"], + "Args": ["vitess.io/vitess/go/test/endtoend/reparent/emergencyreparent"], "Command": [], "Manual": false, "Shard": "14", "RetryMax": 1, "Tags": [] }, - "prs": { + "plannedreparent": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/reparent/prs"], + "Args": ["vitess.io/vitess/go/test/endtoend/reparent/plannedreparent"], "Command": [], "Manual": false, "Shard": "14", From d1201f20f77d4c4d99d7767d5eef082bb7773f66 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 11:25:30 +0530 Subject: [PATCH 156/176] fix change in vtctl file Signed-off-by: Manan Gupta --- go/cmd/vtctl/vtctl.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/go/cmd/vtctl/vtctl.go b/go/cmd/vtctl/vtctl.go index 4e31b76dd79..3e1e5594258 100644 --- a/go/cmd/vtctl/vtctl.go +++ b/go/cmd/vtctl/vtctl.go @@ -93,8 +93,7 @@ func main() { log.Warningf("cannot connect to syslog: %v", err) } - err := reparentutil.SetDurabilityPolicy(*durability, nil) - if err != nil { + if err := reparentutil.SetDurabilityPolicy(*durability, nil); err != nil { log.Errorf("error in setting durability policy: %v", err) exit.Return(1) } @@ -113,7 +112,7 @@ func main() { wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) installSignalHandlers(cancel) - err = vtctl.RunCommand(ctx, wr, args) + err := vtctl.RunCommand(ctx, wr, args) cancel() switch err { case vtctl.ErrUnknownCommand: From 3948d7ec6da49a0b591c4edd2aa5fbe2822f8ad2 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 11:53:31 +0530 Subject: [PATCH 157/176] fix import lines Signed-off-by: Manan Gupta --- go/cmd/vtctl/vtctl.go | 3 +-- go/vt/orchestrator/app/cli.go | 3 +-- go/vt/orchestrator/http/api.go | 3 +-- go/vt/orchestrator/inst/analysis_dao.go | 3 +-- go/vt/orchestrator/inst/candidate_database_instance.go | 3 +-- go/vt/orchestrator/inst/candidate_database_instance_dao.go | 2 +- go/vt/orchestrator/inst/instance.go | 3 +-- go/vt/orchestrator/inst/instance_dao.go | 6 +++--- go/vt/orchestrator/inst/instance_topology.go | 3 +-- go/vt/orchestrator/inst/instance_topology_test.go | 3 +-- go/vt/orchestrator/logic/orchestrator.go | 3 +-- go/vt/orchestrator/logic/topology_recovery.go | 5 ++--- go/vt/vtctl/grpcvtctldserver/server_test.go | 3 +-- go/vt/vtctld/vtctld.go | 3 +-- go/vt/wrangler/testlib/emergency_reparent_shard_test.go | 6 ++---- 15 files changed, 19 insertions(+), 33 deletions(-) diff --git a/go/cmd/vtctl/vtctl.go b/go/cmd/vtctl/vtctl.go index 3e1e5594258..ada00123f6c 100644 --- a/go/cmd/vtctl/vtctl.go +++ b/go/cmd/vtctl/vtctl.go @@ -26,8 +26,6 @@ import ( "syscall" "time" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - "vitess.io/vitess/go/cmd" "context" @@ -39,6 +37,7 @@ import ( "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/vtctl" + "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/vttablet/tmclient" "vitess.io/vitess/go/vt/workflow" "vitess.io/vitess/go/vt/wrangler" diff --git a/go/vt/orchestrator/app/cli.go b/go/vt/orchestrator/app/cli.go index f7e9f0d9b58..9f53992461b 100644 --- a/go/vt/orchestrator/app/cli.go +++ b/go/vt/orchestrator/app/cli.go @@ -26,8 +26,6 @@ import ( "strings" "time" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - "vitess.io/vitess/go/vt/orchestrator/config" "vitess.io/vitess/go/vt/orchestrator/external/golib/log" "vitess.io/vitess/go/vt/orchestrator/external/golib/util" @@ -35,6 +33,7 @@ import ( "vitess.io/vitess/go/vt/orchestrator/kv" "vitess.io/vitess/go/vt/orchestrator/logic" "vitess.io/vitess/go/vt/orchestrator/process" + "vitess.io/vitess/go/vt/vtctl/reparentutil" ) var thisInstanceKey *inst.InstanceKey diff --git a/go/vt/orchestrator/http/api.go b/go/vt/orchestrator/http/api.go index 77e6fe410fc..33318a82edf 100644 --- a/go/vt/orchestrator/http/api.go +++ b/go/vt/orchestrator/http/api.go @@ -25,8 +25,6 @@ import ( "strings" "time" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - "github.com/go-martini/martini" "github.com/martini-contrib/auth" "github.com/martini-contrib/render" @@ -42,6 +40,7 @@ import ( "vitess.io/vitess/go/vt/orchestrator/metrics/query" "vitess.io/vitess/go/vt/orchestrator/process" orcraft "vitess.io/vitess/go/vt/orchestrator/raft" + "vitess.io/vitess/go/vt/vtctl/reparentutil" ) // APIResponseCode is an OK/ERROR response code diff --git a/go/vt/orchestrator/inst/analysis_dao.go b/go/vt/orchestrator/inst/analysis_dao.go index 318a0a51329..57d3cfd86f3 100644 --- a/go/vt/orchestrator/inst/analysis_dao.go +++ b/go/vt/orchestrator/inst/analysis_dao.go @@ -21,8 +21,6 @@ import ( "regexp" "time" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - "google.golang.org/protobuf/encoding/prototext" "vitess.io/vitess/go/vt/orchestrator/config" @@ -31,6 +29,7 @@ import ( "vitess.io/vitess/go/vt/orchestrator/util" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/vtctl/reparentutil" "github.com/patrickmn/go-cache" "github.com/rcrowley/go-metrics" diff --git a/go/vt/orchestrator/inst/candidate_database_instance.go b/go/vt/orchestrator/inst/candidate_database_instance.go index 54ea0c1f8ac..f202c6d6c1e 100644 --- a/go/vt/orchestrator/inst/candidate_database_instance.go +++ b/go/vt/orchestrator/inst/candidate_database_instance.go @@ -19,9 +19,8 @@ package inst import ( "fmt" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - "vitess.io/vitess/go/vt/orchestrator/db" + "vitess.io/vitess/go/vt/vtctl/reparentutil" ) // CandidateDatabaseInstance contains information about explicit promotion rules for an instance diff --git a/go/vt/orchestrator/inst/candidate_database_instance_dao.go b/go/vt/orchestrator/inst/candidate_database_instance_dao.go index 2b228a0de39..7582e76df1a 100644 --- a/go/vt/orchestrator/inst/candidate_database_instance_dao.go +++ b/go/vt/orchestrator/inst/candidate_database_instance_dao.go @@ -19,10 +19,10 @@ package inst import ( "vitess.io/vitess/go/vt/orchestrator/external/golib/log" "vitess.io/vitess/go/vt/orchestrator/external/golib/sqlutils" - "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/orchestrator/config" "vitess.io/vitess/go/vt/orchestrator/db" + "vitess.io/vitess/go/vt/vtctl/reparentutil" ) // RegisterCandidateInstance markes a given instance as suggested for succeeding a primary in the event of failover. diff --git a/go/vt/orchestrator/inst/instance.go b/go/vt/orchestrator/inst/instance.go index a1fc44b6267..6f7184a0fbd 100644 --- a/go/vt/orchestrator/inst/instance.go +++ b/go/vt/orchestrator/inst/instance.go @@ -24,10 +24,9 @@ import ( "strings" "time" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - "vitess.io/vitess/go/vt/orchestrator/config" "vitess.io/vitess/go/vt/orchestrator/external/golib/math" + "vitess.io/vitess/go/vt/vtctl/reparentutil" ) const ReasonableDiscoveryLatency = 500 * time.Millisecond diff --git a/go/vt/orchestrator/inst/instance_dao.go b/go/vt/orchestrator/inst/instance_dao.go index c0d1d0000fc..a582689757d 100644 --- a/go/vt/orchestrator/inst/instance_dao.go +++ b/go/vt/orchestrator/inst/instance_dao.go @@ -39,9 +39,6 @@ import ( "vitess.io/vitess/go/vt/orchestrator/external/golib/log" "vitess.io/vitess/go/vt/orchestrator/external/golib/math" "vitess.io/vitess/go/vt/orchestrator/external/golib/sqlutils" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/orchestrator/attributes" "vitess.io/vitess/go/vt/orchestrator/collection" @@ -50,6 +47,9 @@ import ( "vitess.io/vitess/go/vt/orchestrator/kv" "vitess.io/vitess/go/vt/orchestrator/metrics/query" "vitess.io/vitess/go/vt/orchestrator/util" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/vtctl/reparentutil" ) const ( diff --git a/go/vt/orchestrator/inst/instance_topology.go b/go/vt/orchestrator/inst/instance_topology.go index f8db0e92693..87743a9f66b 100644 --- a/go/vt/orchestrator/inst/instance_topology.go +++ b/go/vt/orchestrator/inst/instance_topology.go @@ -25,13 +25,12 @@ import ( "sync" "time" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - "vitess.io/vitess/go/vt/orchestrator/config" "vitess.io/vitess/go/vt/orchestrator/external/golib/log" "vitess.io/vitess/go/vt/orchestrator/external/golib/math" "vitess.io/vitess/go/vt/orchestrator/external/golib/util" "vitess.io/vitess/go/vt/orchestrator/os" + "vitess.io/vitess/go/vt/vtctl/reparentutil" ) type StopReplicationMethod string diff --git a/go/vt/orchestrator/inst/instance_topology_test.go b/go/vt/orchestrator/inst/instance_topology_test.go index f36b41fdf8b..5d9c138bf4b 100644 --- a/go/vt/orchestrator/inst/instance_topology_test.go +++ b/go/vt/orchestrator/inst/instance_topology_test.go @@ -3,13 +3,12 @@ package inst import ( "math/rand" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - "testing" "vitess.io/vitess/go/vt/orchestrator/config" "vitess.io/vitess/go/vt/orchestrator/external/golib/log" test "vitess.io/vitess/go/vt/orchestrator/external/golib/tests" + "vitess.io/vitess/go/vt/vtctl/reparentutil" ) var ( diff --git a/go/vt/orchestrator/logic/orchestrator.go b/go/vt/orchestrator/logic/orchestrator.go index 5fab2c0dbe8..814b7b5f058 100644 --- a/go/vt/orchestrator/logic/orchestrator.go +++ b/go/vt/orchestrator/logic/orchestrator.go @@ -24,8 +24,6 @@ import ( "syscall" "time" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - "github.com/patrickmn/go-cache" "github.com/rcrowley/go-metrics" "github.com/sjmudd/stopwatch" @@ -39,6 +37,7 @@ import ( ometrics "vitess.io/vitess/go/vt/orchestrator/metrics" "vitess.io/vitess/go/vt/orchestrator/process" "vitess.io/vitess/go/vt/orchestrator/util" + "vitess.io/vitess/go/vt/vtctl/reparentutil" ) const ( diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 73375fa05d8..68cf04a2761 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -34,9 +34,6 @@ import ( "vitess.io/vitess/go/vt/logutil" logutilpb "vitess.io/vitess/go/vt/proto/logutil" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - "vitess.io/vitess/go/vt/vttablet/tmclient" - "github.com/patrickmn/go-cache" "github.com/rcrowley/go-metrics" @@ -48,6 +45,8 @@ import ( "vitess.io/vitess/go/vt/orchestrator/process" "vitess.io/vitess/go/vt/orchestrator/util" topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/vttablet/tmclient" ) var countPendingRecoveries int64 diff --git a/go/vt/vtctl/grpcvtctldserver/server_test.go b/go/vt/vtctl/grpcvtctldserver/server_test.go index ab528ee9b06..537369f905c 100644 --- a/go/vt/vtctl/grpcvtctldserver/server_test.go +++ b/go/vt/vtctl/grpcvtctldserver/server_test.go @@ -24,8 +24,6 @@ import ( "testing" "time" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" @@ -39,6 +37,7 @@ import ( "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" + "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/vttablet/tmclient" mysqlctlpb "vitess.io/vitess/go/vt/proto/mysqlctl" diff --git a/go/vt/vtctld/vtctld.go b/go/vt/vtctld/vtctld.go index 4c82faf37b4..db9159a5511 100644 --- a/go/vt/vtctld/vtctld.go +++ b/go/vt/vtctld/vtctld.go @@ -24,8 +24,6 @@ import ( "strings" "time" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - rice "github.com/GeertJohan/go.rice" "context" @@ -34,6 +32,7 @@ import ( "vitess.io/vitess/go/acl" "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/wrangler" topodatapb "vitess.io/vitess/go/vt/proto/topodata" diff --git a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go index c1491d3b189..3d8b9aa73f9 100644 --- a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go @@ -21,18 +21,16 @@ import ( "testing" "time" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - - "vitess.io/vitess/go/vt/discovery" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/util/sets" "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/vt/discovery" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/vttablet/tmclient" "vitess.io/vitess/go/vt/wrangler" From be76f8c2cdee9d22b8cde2a5023608e831e8a898 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 12:48:48 +0530 Subject: [PATCH 158/176] move functions from util to ers which can be used only in EmergencyReparentShard Signed-off-by: Manan Gupta --- go/cmd/vtctl/vtctl.go | 4 +- .../reparentutil/emergency_reparenter.go | 417 +++- .../reparentutil/emergency_reparenter_test.go | 1478 ++++++++++++- go/vt/vtctl/reparentutil/util.go | 451 +--- go/vt/vtctl/reparentutil/util_test.go | 1852 ++--------------- 5 files changed, 2101 insertions(+), 2101 deletions(-) diff --git a/go/cmd/vtctl/vtctl.go b/go/cmd/vtctl/vtctl.go index ada00123f6c..99d6d64f56b 100644 --- a/go/cmd/vtctl/vtctl.go +++ b/go/cmd/vtctl/vtctl.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "context" "flag" "fmt" "log/syslog" @@ -27,9 +28,6 @@ import ( "time" "vitess.io/vitess/go/cmd" - - "context" - "vitess.io/vitess/go/exit" "vitess.io/vitess/go/trace" "vitess.io/vitess/go/vt/log" diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index f5a8b6c57e1..6481a313c9e 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -18,8 +18,12 @@ package reparentutil import ( "context" + "fmt" + "sync" "time" + "vitess.io/vitess/go/vt/concurrency" + "k8s.io/apimachinery/pkg/util/sets" "vitess.io/vitess/go/mysql" @@ -110,7 +114,7 @@ func NewEmergencyReparenter(ts *topo.Server, tmc tmclient.TabletManagerClient, l // keyspace and shard. func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, shard string, opts EmergencyReparentOptions) (*events.Reparent, error) { // First step is to lock the shard for the given operation - opts.lockAction = getLockAction(opts.newPrimaryAlias) + opts.lockAction = erp.getLockAction(opts.newPrimaryAlias) ctx, unlock, err := erp.ts.LockShard(ctx, keyspace, shard, opts.lockAction) if err != nil { return nil, err @@ -137,6 +141,16 @@ func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, sha return ev, err } +func (erp *EmergencyReparenter) getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { + action := "EmergencyReparentShard" + + if newPrimaryAlias != nil { + action += fmt.Sprintf("(%v)", topoproto.TabletAliasString(newPrimaryAlias)) + } + + return action +} + // reparentShardLocked performs Emergency Reparent Shard operation assuming that the shard is already locked func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *events.Reparent, keyspace, shard string, opts EmergencyReparentOptions) (err error) { // log the starting of the operation and increment the counter @@ -200,7 +214,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // Wait for all candidates to apply relay logs - if err = waitForAllRelayLogsToApply(ctx, erp.logger, erp.tmc, validCandidates, tabletMap, statusMap, opts.waitReplicasTimeout); err != nil { + if err = erp.waitForAllRelayLogsToApply(ctx, erp.logger, erp.tmc, validCandidates, tabletMap, statusMap, opts.waitReplicasTimeout); err != nil { return err } @@ -208,7 +222,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // We let all the other tablets replicate from this tablet. We will then try to choose a better candidate and let it catch up var intermediateSource *topodatapb.Tablet var validCandidateTablets []*topodatapb.Tablet - intermediateSource, validCandidateTablets, err = findMostAdvanced(erp.logger, prevPrimary, validCandidates, tabletMap, opts) + intermediateSource, validCandidateTablets, err = erp.findMostAdvanced(erp.logger, prevPrimary, validCandidates, tabletMap, opts) if err != nil { return err } @@ -216,7 +230,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // check weather the primary candidate selected is ideal or if it can be improved later var isIdeal bool - isIdeal, err = intermediateCandidateIsIdeal(erp.logger, intermediateSource, prevPrimary, validCandidateTablets, tabletMap, opts) + isIdeal, err = erp.intermediateCandidateIsIdeal(erp.logger, intermediateSource, prevPrimary, validCandidateTablets, tabletMap, opts) if err != nil { return err } @@ -234,21 +248,21 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // we do not promote the tablet or change the shard record. We only change the replication for all the other tablets // it also returns the list of the tablets that started replication successfully including itself. These are the candidates that we can use to find a replacement var validReplacementCandidates []*topodatapb.Tablet - validReplacementCandidates, err = promoteIntermediatePrimary(ctx, erp.tmc, ev, erp.logger, intermediateSource, opts.lockAction, tabletMap, statusMap, opts) + validReplacementCandidates, err = erp.promoteIntermediatePrimary(ctx, erp.tmc, ev, erp.logger, intermediateSource, opts.lockAction, tabletMap, statusMap, opts) if err != nil { return err } // try to find a better candidate using the list we got back var betterCandidate *topodatapb.Tablet - betterCandidate, err = identifyPrimaryCandidate(erp.logger, intermediateSource, prevPrimary, validReplacementCandidates, tabletMap, opts) + betterCandidate, err = erp.identifyPrimaryCandidate(erp.logger, intermediateSource, prevPrimary, validReplacementCandidates, tabletMap, opts) if err != nil { return err } // if our better candidate is different from our previous candidate, then we wait for it to catch up to the intermediate primary if !topoproto.TabletAliasEqual(betterCandidate.Alias, intermediateSource.Alias) { - err = waitForCatchingUp(ctx, erp.tmc, erp.logger, intermediateSource, betterCandidate, opts.waitReplicasTimeout) + err = waitForCatchUp(ctx, erp.tmc, erp.logger, intermediateSource, betterCandidate, opts.waitReplicasTimeout) if err != nil { return err } @@ -257,7 +271,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // now we check if all the constraints are satisfied. If they are not, then we should abort - constraintFailure := checkIfConstraintsSatisfied(newPrimary, prevPrimary, opts) + constraintFailure := erp.checkIfConstraintsSatisfied(newPrimary, prevPrimary, opts) if constraintFailure != nil { erp.logger.Errorf("have to override promotion because of constraint failure - %v", constraintFailure) // we want to send both the errors to the user, constraint failure and also any error encountered in undoing the promotion @@ -285,6 +299,391 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } +func (erp *EmergencyReparenter) waitForAllRelayLogsToApply(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, waitReplicasTimeout time.Duration) error { + errCh := make(chan error) + defer close(errCh) + + groupCtx, groupCancel := context.WithTimeout(ctx, waitReplicasTimeout) + defer groupCancel() + + waiterCount := 0 + + for candidate := range validCandidates { + // When we called StopReplicationAndBuildStatusMaps, we got back two + // maps: (1) the StopReplicationStatus of any replicas that actually + // stopped replication; and (2) the MasterStatus of anything that + // returned ErrNotReplica, which is a tablet that is either the current + // primary or is stuck thinking it is a MASTER but is not in actuality. + // + // If we have a tablet in the validCandidates map that does not appear + // in the statusMap, then we have either (a) the current primary, which + // is not replicating, so it is not applying relay logs; or (b) a tablet + // that is stuck thinking it is MASTER but is not in actuality. In that + // second case - (b) - we will most likely find that the stuck MASTER + // does not have a winning position, and fail the ERS. If, on the other + // hand, it does have a winning position, we are trusting the operator + // to know what they are doing by emergency-reparenting onto that + // tablet. In either case, it does not make sense to wait for relay logs + // to apply on a tablet that was never applying relay logs in the first + // place, so we skip it, and log that we did. + status, ok := statusMap[candidate] + if !ok { + logger.Infof("EmergencyReparent candidate %v not in replica status map; this means it was not running replication (because it was formerly PRIMARY), so skipping WaitForRelayLogsToApply step for this candidate", candidate) + continue + } + + go func(alias string, status *replicationdatapb.StopReplicationStatus) { + var err error + defer func() { errCh <- err }() + err = WaitForRelayLogsToApply(groupCtx, tmc, tabletMap[alias], status) + }(candidate, status) + + waiterCount++ + } + + errgroup := concurrency.ErrorGroup{ + NumGoroutines: waiterCount, + NumRequiredSuccesses: waiterCount, + NumAllowedErrors: 0, + } + rec := errgroup.Wait(groupCancel, errCh) + + if len(rec.Errors) != 0 { + return vterrors.Wrapf(rec.Error(), "could not apply all relay logs within the provided waitReplicasTimeout (%s): %v", waitReplicasTimeout, rec.Error()) + } + + return nil +} + +// findMostAdvanced finds the intermediate primary candidate for ERS. We always choose the most advanced one from our valid candidates list +func (erp *EmergencyReparenter) findMostAdvanced(logger logutil.Logger, prevPrimary *topodatapb.Tablet, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (*topodatapb.Tablet, []*topodatapb.Tablet, error) { + logger.Infof("started finding the intermediate primary candidate") + // convert the valid candidates into a list so that we can use it for sorting + validTablets, tabletPositions, err := getValidCandidatesAndPositionsAsList(validCandidates, tabletMap) + if err != nil { + return nil, nil, err + } + + idealCell := "" + if prevPrimary != nil { + idealCell = prevPrimary.Alias.Cell + } + + // sort the tablets for finding the best intermediate primary in ERS + err = sortTabletsForERS(validTablets, tabletPositions, idealCell) + if err != nil { + return nil, nil, err + } + for _, tablet := range validTablets { + logger.Infof("finding intermediate primary - sorted replica: %v", tablet.Alias) + } + + // The first tablet in the sorted list will be the most eligible candidate unless explicitly asked for some other tablet + winningPrimaryTablet := validTablets[0] + winningPosition := tabletPositions[0] + + // We have already removed the tablets with errant GTIDs before calling this function. At this point our winning position must be a + // superset of all the other valid positions. If that is not the case, then we have a split brain scenario, and we should cancel the ERS + for i, position := range tabletPositions { + if !winningPosition.AtLeast(position) { + return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "split brain detected between servers - %v and %v", winningPrimaryTablet.Alias, validTablets[i].Alias) + } + } + + // If we were requested to elect a particular primary, verify it's a valid + // candidate (non-zero position, no errant GTIDs) + // Also, if the candidate is + if opts.newPrimaryAlias != nil { + requestedPrimaryAlias := topoproto.TabletAliasString(opts.newPrimaryAlias) + pos, ok := validCandidates[requestedPrimaryAlias] + if !ok { + return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "requested primary elect %v has errant GTIDs", requestedPrimaryAlias) + } + // if the requested tablet is as advanced as the most advanced tablet, then we can just use it for promotion. + // otherwise, we should let it catchup to the most advanced tablet and let it be the intermediate primary + if pos.AtLeast(winningPosition) { + requestedPrimaryInfo, isFound := tabletMap[requestedPrimaryAlias] + if !isFound { + return nil, nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", requestedPrimaryAlias) + } + winningPrimaryTablet = requestedPrimaryInfo.Tablet + } + } + + return winningPrimaryTablet, validTablets, nil +} + +// promoteIntermediatePrimary promotes the primary candidate that we have, but it does not yet set to start accepting writes +func (erp *EmergencyReparenter) promoteIntermediatePrimary(ctx context.Context, tmc tmclient.TabletManagerClient, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, + lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions) ([]*topodatapb.Tablet, error) { + // we reparent all the other tablets to start replication from our new primary + // we wait for all the replicas so that we can choose a better candidate from the ones that started replication later + validCandidatesForImprovement, err := erp.reparentReplicas(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, opts, true, false) + if err != nil { + return nil, err + } + + // also include the current tablet for being considered as part of valid candidates for ERS promotion + validCandidatesForImprovement = append(validCandidatesForImprovement, newPrimary) + return validCandidatesForImprovement, nil +} + +// reparentReplicas reparents all the replicas provided and populates the reparent journal on the primary. +// Also, it returns the replicas which started replicating only in the case where we wait for all the replicas +func (erp *EmergencyReparenter) reparentReplicas(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, newPrimaryTablet *topodatapb.Tablet, lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions, waitForAllReplicas bool, populateReparentJournal bool) ([]*topodatapb.Tablet, error) { + + var replicasStartedReplication []*topodatapb.Tablet + var replicaMutex sync.Mutex + + replCtx, replCancel := context.WithTimeout(ctx, opts.waitReplicasTimeout) + + event.DispatchUpdate(ev, "reparenting all tablets") + + // Create a context and cancel function to watch for the first successful + // SetMaster call on a replica. We use a background context so that this + // context is only ever Done when its cancel is called by the background + // goroutine we're about to spin up. + // + // Similarly, create a context and cancel for the replica waiter goroutine + // to signal when all replica goroutines have finished. In the case where at + // least one replica succeeds, replSuccessCtx will be canceled first, while + // allReplicasDoneCtx is guaranteed to be canceled within + // opts.WaitReplicasTimeout plus some jitter. + replSuccessCtx, replSuccessCancel := context.WithCancel(context.Background()) + allReplicasDoneCtx, allReplicasDoneCancel := context.WithCancel(context.Background()) + + now := time.Now().UnixNano() + replWg := sync.WaitGroup{} + rec := concurrency.AllErrorRecorder{} + + handlePrimary := func(alias string, tablet *topodatapb.Tablet) error { + position, err := tmc.MasterPosition(replCtx, tablet) + if err != nil { + return err + } + if populateReparentJournal { + logger.Infof("populating reparent journal on new primary %v", alias) + return tmc.PopulateReparentJournal(replCtx, tablet, now, lockAction, newPrimaryTablet.Alias, position) + } + return nil + } + + handleReplica := func(alias string, ti *topo.TabletInfo) { + defer replWg.Done() + logger.Infof("setting new primary on replica %v", alias) + + forceStart := false + if status, ok := statusMap[alias]; ok { + fs, err := ReplicaWasRunning(status) + if err != nil { + err = vterrors.Wrapf(err, "tablet %v could not determine StopReplicationStatus: %v", alias, err) + rec.RecordError(err) + + return + } + + forceStart = fs + } + + err := tmc.SetMaster(replCtx, ti.Tablet, newPrimaryTablet.Alias, 0, "", forceStart) + if err != nil { + err = vterrors.Wrapf(err, "tablet %v SetReplicationSource failed: %v", alias, err) + rec.RecordError(err) + + return + } + + replicaMutex.Lock() + replicasStartedReplication = append(replicasStartedReplication, ti.Tablet) + replicaMutex.Unlock() + + // Signal that at least one goroutine succeeded to SetReplicationSource. + // We do this only when we do not want to wait for all the replicas + if !waitForAllReplicas { + replSuccessCancel() + } + } + + numReplicas := 0 + + for alias, ti := range tabletMap { + switch { + case alias == topoproto.TabletAliasString(newPrimaryTablet.Alias): + continue + case !opts.ignoreReplicas.Has(alias): + replWg.Add(1) + numReplicas++ + go handleReplica(alias, ti) + } + } + + // Spin up a background goroutine to wait until all replica goroutines + // finished. Polling this way allows us to have reparentReplicas return + // success as soon as (a) the primary successfully populates its reparent + // journal and (b) at least one replica successfully begins replicating. + // + // If we were to follow the more common pattern of blocking on replWg.Wait() + // in the main body of promoteNewPrimary, we would be bound to the + // time of slowest replica, instead of the time of the fastest successful + // replica, and we want ERS to be fast. + go func() { + replWg.Wait() + allReplicasDoneCancel() + }() + + primaryErr := handlePrimary(topoproto.TabletAliasString(newPrimaryTablet.Alias), newPrimaryTablet) + if primaryErr != nil { + logger.Warningf("primary failed to PopulateReparentJournal") + replCancel() + + return nil, vterrors.Wrapf(primaryErr, "failed to PopulateReparentJournal on primary: %v", primaryErr) + } + + go func() { + replWg.Wait() + defer replCancel() + }() + + select { + case <-replSuccessCtx.Done(): + // At least one replica was able to SetMaster successfully + // Here we do not need to return the replicas which started replicating + return nil, nil + case <-allReplicasDoneCtx.Done(): + // There are certain timing issues between replSuccessCtx.Done firing + // and allReplicasDoneCtx.Done firing, so we check again if truly all + // replicas failed (where `numReplicas` goroutines recorded an error) or + // one or more actually managed to succeed. + errCount := len(rec.Errors) + + switch { + case errCount > numReplicas: + // Technically, rec.Errors should never be greater than numReplicas, + // but it's better to err on the side of caution here, but also + // we're going to be explicit that this is doubly unexpected. + return nil, vterrors.Wrapf(rec.Error(), "received more errors (= %d) than replicas (= %d), which should be impossible: %v", errCount, numReplicas, rec.Error()) + case errCount == numReplicas: + return nil, vterrors.Wrapf(rec.Error(), "%d replica(s) failed: %v", numReplicas, rec.Error()) + default: + return replicasStartedReplication, nil + } + } + +} + +// intermediateCandidateIsIdeal is used to find whether the intermediate candidate that ERS chose is also the ideal one or not +func (erp *EmergencyReparenter) intermediateCandidateIsIdeal(logger logutil.Logger, newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (bool, error) { + // we try to find a better candidate with the current list of valid candidates, and if it matches our current primary candidate, then we return true + candidate, err := erp.identifyPrimaryCandidate(logger, newPrimary, prevPrimary, validCandidates, tabletMap, opts) + if err != nil { + return false, err + } + return candidate == newPrimary, nil +} + +// identifyPrimaryCandidate is used to find a better candidate for ERS promotion +func (erp *EmergencyReparenter) identifyPrimaryCandidate(logger logutil.Logger, newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (candidate *topodatapb.Tablet, err error) { + defer func() { + if candidate != nil { + logger.Infof("found better candidate - %v", candidate.Alias) + } + }() + + if opts.newPrimaryAlias != nil { + // explicit request to promote a specific tablet + requestedPrimaryAlias := topoproto.TabletAliasString(opts.newPrimaryAlias) + requestedPrimaryInfo, isFound := tabletMap[requestedPrimaryAlias] + if !isFound { + return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", requestedPrimaryAlias) + } + for _, validCandidate := range validCandidates { + if topoproto.TabletAliasEqual(validCandidate.Alias, opts.newPrimaryAlias) { + return requestedPrimaryInfo.Tablet, nil + } + } + return nil, vterrors.Errorf(vtrpc.Code_ABORTED, "requested candidate %v is not in valid candidates list", requestedPrimaryAlias) + } + var preferredCandidates []*topodatapb.Tablet + var neutralReplicas []*topodatapb.Tablet + for _, candidate := range validCandidates { + promotionRule := PromotionRule(candidate) + if promotionRule == MustPromoteRule || promotionRule == PreferPromoteRule { + preferredCandidates = append(preferredCandidates, candidate) + } + if promotionRule == NeutralPromoteRule { + neutralReplicas = append(neutralReplicas, candidate) + } + } + + // So we've already promoted a replica. + // However, can we improve on our choice? Are there any replicas with better promotion rules? + // Maybe we actually promoted such a replica. Does that mean we should keep it? + // Maybe we promoted a "neutral", and some "prefer" server is available. + // Maybe we promoted a "prefer_not" + // Maybe we promoted a server in a different cell than the primary + // There's many options. We may wish to replace the server we promoted with a better one. + + // check whether the one we promoted is in the same cell and belongs to the preferred candidates list + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, true, true) + if candidate != nil { + return candidate, nil + } + // check whether there is some other tablet in the same cell belonging to the preferred candidates list + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, false, true) + if candidate != nil { + return candidate, nil + } + // we do not have a preferred candidate in the same cell + + if !opts.preventCrossCellPromotion { + // check whether the one we promoted belongs to the preferred candidates list + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, true, false) + if candidate != nil { + return candidate, nil + } + // check whether there is some other tablet belonging to the preferred candidates list + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, false, false) + if candidate != nil { + return candidate, nil + } + } + + // repeat the same process for the neutral candidates list + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, true, true) + if candidate != nil { + return candidate, nil + } + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, false, true) + if candidate != nil { + return candidate, nil + } + + if !opts.preventCrossCellPromotion { + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, true, false) + if candidate != nil { + return candidate, nil + } + candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, false, false) + if candidate != nil { + return candidate, nil + } + } + + // return the one that we have if nothing found + return newPrimary, nil +} + +// checkIfConstraintsSatisfied is used to check whether the constraints for ERS are satisfied or not. +func (erp *EmergencyReparenter) checkIfConstraintsSatisfied(newPrimary, prevPrimary *topodatapb.Tablet, opts EmergencyReparentOptions) error { + if opts.preventCrossCellPromotion && prevPrimary != nil && newPrimary.Alias.Cell != prevPrimary.Alias.Cell { + return vterrors.Errorf(vtrpc.Code_ABORTED, "elected primary does not satisfy geographic constraint - %s", topoproto.TabletAliasString(newPrimary.Alias)) + } + if PromotionRule(newPrimary) == MustNotPromoteRule { + return vterrors.Errorf(vtrpc.Code_ABORTED, "elected primary does not satisfy promotion rule constraint - %s", topoproto.TabletAliasString(newPrimary.Alias)) + } + return nil +} + func (erp *EmergencyReparenter) promoteNewPrimary(ctx context.Context, ev *events.Reparent, newPrimary *topodatapb.Tablet, opts EmergencyReparentOptions, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus) error { erp.logger.Infof("starting promotion for the new primary - %v", newPrimary.Alias) // we call PromoteReplica which changes the tablet type, fixes the semi-sync, set the primary to read-write and flushes the binlogs @@ -293,7 +692,7 @@ func (erp *EmergencyReparenter) promoteNewPrimary(ctx context.Context, ev *event return vterrors.Wrapf(err, "primary-elect tablet %v failed to be upgraded to primary: %v", newPrimary.Alias, err) } // we now reparent all the replicas to the new primary we have promoted - _, err = reparentReplicas(ctx, ev, erp.logger, erp.tmc, newPrimary, opts.lockAction, tabletMap, statusMap, opts, false, true) + _, err = erp.reparentReplicas(ctx, ev, erp.logger, erp.tmc, newPrimary, opts.lockAction, tabletMap, statusMap, opts, false, true) if err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 7de63cfb2e8..2bff375ed54 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -22,6 +22,8 @@ import ( "testing" "time" + "vitess.io/vitess/go/vt/topo/topoproto" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/util/sets" @@ -93,13 +95,15 @@ func TestEmergencyReparenter_getLockAction(t *testing.T) { }, } + erp := &EmergencyReparenter{} + for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() - actual := getLockAction(tt.alias) + actual := erp.getLockAction(tt.alias) assert.Equal(t, tt.expected, actual, tt.msg) }) } @@ -2063,7 +2067,8 @@ func TestEmergencyReparenter_waitForAllRelayLogsToApply(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - err := waitForAllRelayLogsToApply(ctx, logger, tt.tmc, tt.candidates, tt.tabletMap, tt.statusMap, waitReplicasTimeout) + erp := NewEmergencyReparenter(nil, tt.tmc, logger) + err := erp.waitForAllRelayLogsToApply(ctx, logger, tt.tmc, tt.candidates, tt.tabletMap, tt.statusMap, waitReplicasTimeout) if tt.shouldErr { assert.Error(t, err) return @@ -2225,3 +2230,1472 @@ func TestEmergencyReparenterCounters(t *testing.T) { require.EqualValues(t, 1, ersSuccessCounter.Get()) require.EqualValues(t, 1, ersFailureCounter.Get()) } + +func TestEmergencyReparenter_findMostAdvanced(t *testing.T) { + sid1 := mysql.SID{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + mysqlGTID1 := mysql.Mysql56GTID{ + Server: sid1, + Sequence: 9, + } + mysqlGTID2 := mysql.Mysql56GTID{ + Server: sid1, + Sequence: 10, + } + mysqlGTID3 := mysql.Mysql56GTID{ + Server: sid1, + Sequence: 11, + } + + positionMostAdvanced := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID1) + positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID2) + positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID3) + + positionIntermediate1 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionIntermediate1.GTIDSet = positionIntermediate1.GTIDSet.AddGTID(mysqlGTID1) + + positionIntermediate2 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionIntermediate2.GTIDSet = positionIntermediate2.GTIDSet.AddGTID(mysqlGTID1) + positionIntermediate2.GTIDSet = positionIntermediate2.GTIDSet.AddGTID(mysqlGTID2) + + positionOnly2 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionOnly2.GTIDSet = positionOnly2.GTIDSet.AddGTID(mysqlGTID2) + + positionEmpty := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + + tests := []struct { + name string + validCandidates map[string]mysql.Position + tabletMap map[string]*topo.TabletInfo + prevPrimary *topodatapb.Tablet + emergencyReparentOps EmergencyReparentOptions + result *topodatapb.Tablet + err string + }{ + { + name: "choose most advanced - with nil previous primary", + validCandidates: map[string]mysql.Position{ + "zone1-0000000100": positionMostAdvanced, + "zone1-0000000101": positionIntermediate1, + "zone1-0000000102": positionIntermediate2, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + "zone1-0000000404": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 404, + }, + Hostname: "ignored tablet", + }, + }, + }, + prevPrimary: nil, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, { + name: "choose most advanced in the same cell of previous primary", + validCandidates: map[string]mysql.Position{ + "zone1-0000000100": positionMostAdvanced, + "zone1-0000000101": positionIntermediate1, + "zone2-0000000100": positionMostAdvanced, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone2-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 100, + }, + }, + }, + "zone1-0000000404": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 404, + }, + Hostname: "ignored tablet", + }, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, { + name: "choose most advanced with the best promotion rule", + validCandidates: map[string]mysql.Position{ + "zone1-0000000100": positionMostAdvanced, + "zone1-0000000101": positionIntermediate1, + "zone1-0000000102": positionMostAdvanced, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, + }, + "zone1-0000000404": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 404, + }, + Hostname: "ignored tablet", + }, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, { + name: "choose most advanced with explicit request", + emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }}, + validCandidates: map[string]mysql.Position{ + "zone1-0000000100": positionMostAdvanced, + "zone1-0000000101": positionIntermediate1, + "zone1-0000000102": positionMostAdvanced, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, + }, + "zone1-0000000404": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 404, + }, + Hostname: "ignored tablet", + }, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, { + name: "split brain detection", + emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }}, + validCandidates: map[string]mysql.Position{ + "zone1-0000000100": positionOnly2, + "zone1-0000000101": positionIntermediate1, + "zone1-0000000102": positionEmpty, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, + }, + "zone1-0000000404": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 404, + }, + Hostname: "ignored tablet", + }, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + err: "split brain detected between servers", + }, + } + + _ = SetDurabilityPolicy("none", nil) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + erp := NewEmergencyReparenter(nil, nil, logutil.NewMemoryLogger()) + + winningTablet, _, err := erp.findMostAdvanced(logutil.NewMemoryLogger(), test.prevPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) + if test.err != "" { + assert.Error(t, err) + assert.Contains(t, err.Error(), test.err) + } else { + assert.NoError(t, err) + assert.True(t, topoproto.TabletAliasEqual(test.result.Alias, winningTablet.Alias)) + } + }) + } +} + +func TestEmergencyReparenter_checkIfConstraintsSatisfied(t *testing.T) { + testcases := []struct { + name string + newPrimary, prevPrimary *topodatapb.Tablet + opts EmergencyReparentOptions + err string + }{ + { + name: "no constraint failure", + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + }, + Type: topodatapb.TabletType_REPLICA, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + }, + }, + opts: EmergencyReparentOptions{preventCrossCellPromotion: true}, + err: "", + }, { + name: "promotion rule constraint failure", + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + }, + }, + opts: EmergencyReparentOptions{preventCrossCellPromotion: true}, + err: "elected primary does not satisfy promotion rule constraint - cell1-0000000100", + }, { + name: "cross cell constraint failure", + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + Uid: 100, + }, + Type: topodatapb.TabletType_REPLICA, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell2", + }, + }, + opts: EmergencyReparentOptions{preventCrossCellPromotion: true}, + err: "elected primary does not satisfy geographic constraint - cell1-0000000100", + }, { + name: "cross cell but no constraint failure", + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell1", + Uid: 100, + }, + Type: topodatapb.TabletType_REPLICA, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "cell2", + }, + }, + opts: EmergencyReparentOptions{preventCrossCellPromotion: false}, + err: "", + }, + } + + _ = SetDurabilityPolicy("none", nil) + erp := NewEmergencyReparenter(nil, nil, nil) + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + err := erp.checkIfConstraintsSatisfied(testcase.newPrimary, testcase.prevPrimary, testcase.opts) + if testcase.err == "" { + require.NoError(t, err) + } else { + require.EqualError(t, err, testcase.err) + } + }) + } +} + +func TestEmergencyReparenter_reparentReplicas(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + emergencyReparentOps EmergencyReparentOptions + tmc *testutil.TabletManagerClient + unlockTopo bool + newPrimaryTabletAlias string + ts *topo.Server + keyspace string + shard string + tablets []*topodatapb.Tablet + tabletMap map[string]*topo.TabletInfo + statusMap map[string]*replicationdatapb.StopReplicationStatus + shouldErr bool + errShouldContain string + }{ + { + name: "success", + emergencyReparentOps: NewEmergencyReparentOptions(nil, sets.NewString("zone1-0000000404"), 0, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000101": nil, + "zone1-0000000102": nil, + "zone1-0000000404": assert.AnError, // okay, because we're ignoring it. + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Hostname: "primary-elect", + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Hostname: "requires force start", + }, + }, + "zone1-0000000404": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 404, + }, + Hostname: "ignored tablet", + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{ + "zone1-0000000101": { // forceStart = false + Before: &replicationdatapb.Status{ + IoThreadRunning: false, + SqlThreadRunning: false, + }, + }, + "zone1-0000000102": { // forceStart = true + Before: &replicationdatapb.Status{ + IoThreadRunning: true, + SqlThreadRunning: true, + }, + }, + }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: false, + }, + { + name: "MasterPosition error", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + tmc: &testutil.TabletManagerClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: fmt.Errorf("primary position error"), + }, + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: "primary position error", + }, + { + name: "cannot repopulate reparent journal on new primary", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": assert.AnError, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: "failed to PopulateReparentJournal on primary", + }, + { + name: "all replicas failing to SetMaster does fail the promotion", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + + SetMasterResults: map[string]error{ + // everyone fails, we all fail + "zone1-0000000101": assert.AnError, + "zone1-0000000102": assert.AnError, + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-00000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: " replica(s) failed", + }, + { + name: "all replicas slow to SetMaster does fail the promotion", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, time.Millisecond*10, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + SetMasterDelays: map[string]time.Duration{ + // nothing is failing, we're just slow + "zone1-0000000101": time.Millisecond * 100, + "zone1-0000000102": time.Millisecond * 75, + }, + SetMasterResults: map[string]error{ + "zone1-0000000101": nil, + "zone1-0000000102": nil, + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + shouldErr: true, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + errShouldContain: "context deadline exceeded", + }, + { + name: "one replica failing to SetMaster does not fail the promotion", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000101": nil, // this one succeeds, so we're good + "zone1-0000000102": assert.AnError, + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: false, + }, + } + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + logger := logutil.NewMemoryLogger() + ev := &events.Reparent{} + + testutil.AddShards(ctx, t, tt.ts, &vtctldatapb.Shard{ + Keyspace: tt.keyspace, + Name: tt.shard, + }) + + if !tt.unlockTopo { + var ( + unlock func(*error) + lerr error + ) + + ctx, unlock, lerr = tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "test lock") + require.NoError(t, lerr, "could not lock %s/%s for test", tt.keyspace, tt.shard) + + defer func() { + unlock(&lerr) + require.NoError(t, lerr, "could not unlock %s/%s after test", tt.keyspace, tt.shard) + }() + } + tabletInfo := tt.tabletMap[tt.newPrimaryTabletAlias] + + erp := NewEmergencyReparenter(tt.ts, tt.tmc, logger) + _, err := erp.reparentReplicas(ctx, ev, logger, tt.tmc, tabletInfo.Tablet, "", tt.tabletMap, tt.statusMap, tt.emergencyReparentOps, false, true) + if tt.shouldErr { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errShouldContain) + return + } + + assert.NoError(t, err) + }) + } +} + +func TestEmergencyReparenter_promoteIntermediatePrimary(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + emergencyReparentOps EmergencyReparentOptions + tmc *testutil.TabletManagerClient + unlockTopo bool + newPrimaryTabletAlias string + ts *topo.Server + keyspace string + shard string + tablets []*topodatapb.Tablet + tabletMap map[string]*topo.TabletInfo + statusMap map[string]*replicationdatapb.StopReplicationStatus + shouldErr bool + errShouldContain string + result []*topodatapb.Tablet + }{ + { + name: "success", + emergencyReparentOps: NewEmergencyReparentOptions(nil, sets.NewString("zone1-0000000404"), 0, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000101": nil, + "zone1-0000000102": nil, + "zone1-0000000404": assert.AnError, // okay, because we're ignoring it. + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Hostname: "primary-elect", + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Hostname: "requires force start", + }, + }, + "zone1-0000000404": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 404, + }, + Hostname: "ignored tablet", + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{ + "zone1-0000000101": { // forceStart = false + Before: &replicationdatapb.Status{ + IoThreadRunning: false, + SqlThreadRunning: false, + }, + }, + "zone1-0000000102": { // forceStart = true + Before: &replicationdatapb.Status{ + IoThreadRunning: true, + SqlThreadRunning: true, + }, + }, + }, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: false, + result: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Hostname: "primary-elect", + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Hostname: "requires force start", + }, + }, + }, + { + name: "all replicas failed", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + + SetMasterResults: map[string]error{ + // everyone fails, we all fail + "zone1-0000000101": assert.AnError, + "zone1-0000000102": assert.AnError, + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-00000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: true, + errShouldContain: " replica(s) failed", + }, + { + name: "one replica failed", + emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + tmc: &testutil.TabletManagerClient{ + PopulateReparentJournalResults: map[string]error{ + "zone1-0000000100": nil, + }, + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Error: nil, + }, + }, + SetMasterResults: map[string]error{ + "zone1-0000000101": nil, // this one succeeds, so we're good + "zone1-0000000102": assert.AnError, + }, + }, + newPrimaryTabletAlias: "zone1-0000000100", + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + }, + }, + }, + statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, + keyspace: "testkeyspace", + shard: "-", + ts: memorytopo.NewServer("zone1"), + shouldErr: false, + result: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + logger := logutil.NewMemoryLogger() + ev := &events.Reparent{} + + testutil.AddShards(ctx, t, tt.ts, &vtctldatapb.Shard{ + Keyspace: tt.keyspace, + Name: tt.shard, + }) + + if !tt.unlockTopo { + var ( + unlock func(*error) + lerr error + ) + + ctx, unlock, lerr = tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "test lock") + require.NoError(t, lerr, "could not lock %s/%s for test", tt.keyspace, tt.shard) + + defer func() { + unlock(&lerr) + require.NoError(t, lerr, "could not unlock %s/%s after test", tt.keyspace, tt.shard) + }() + } + tabletInfo := tt.tabletMap[tt.newPrimaryTabletAlias] + + erp := NewEmergencyReparenter(tt.ts, tt.tmc, logger) + res, err := erp.promoteIntermediatePrimary(ctx, tt.tmc, ev, logger, tabletInfo.Tablet, "", tt.tabletMap, tt.statusMap, tt.emergencyReparentOps) + if tt.shouldErr { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errShouldContain) + return + } + + assert.NoError(t, err) + assert.ElementsMatch(t, tt.result, res) + }) + } +} + +func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { + tests := []struct { + name string + emergencyReparentOps EmergencyReparentOptions + newPrimary *topodatapb.Tablet + prevPrimary *topodatapb.Tablet + validCandidates []*topodatapb.Tablet + tabletMap map[string]*topo.TabletInfo + err string + result *topodatapb.Tablet + }{ + { + name: "explicit request for a primary tablet", + emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }}, + newPrimary: nil, + prevPrimary: nil, + validCandidates: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + }, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, { + name: "explicit request for a primary tablet not in valid list", + emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }}, + newPrimary: nil, + prevPrimary: nil, + validCandidates: nil, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + }, + err: "requested candidate zone1-0000000100 is not in valid candidates list", + }, { + name: "explicit request for a primary tablet not in tablet map", + emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }}, + newPrimary: nil, + prevPrimary: nil, + validCandidates: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + tabletMap: map[string]*topo.TabletInfo{}, + err: "candidate zone1-0000000100 not found in the tablet map; this an impossible situation", + }, { + name: "preferred candidate in the same cell same as our replica", + emergencyReparentOps: EmergencyReparentOptions{}, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + validCandidates: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_REPLICA, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + Type: topodatapb.TabletType_REPLICA, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + tabletMap: nil, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, { + name: "preferred candidate in the same cell different from original replica", + emergencyReparentOps: EmergencyReparentOptions{}, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + validCandidates: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + Type: topodatapb.TabletType_REPLICA, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + tabletMap: nil, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, { + name: "preferred candidate in the different cell same as original replica", + emergencyReparentOps: EmergencyReparentOptions{}, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + validCandidates: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + Type: topodatapb.TabletType_PRIMARY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 102, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + tabletMap: nil, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + }, + }, { + name: "preferred candidate in the different cell different from original replica", + emergencyReparentOps: EmergencyReparentOptions{}, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + validCandidates: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 102, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + tabletMap: nil, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 102, + }, + }, + }, { + name: "prevent cross cell promotion", + emergencyReparentOps: EmergencyReparentOptions{preventCrossCellPromotion: true}, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + }, + }, + validCandidates: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 100, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 101, + }, + Type: topodatapb.TabletType_RDONLY, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone2", + Uid: 102, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + tabletMap: nil, + result: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _ = SetDurabilityPolicy("none", nil) + logger := logutil.NewMemoryLogger() + + erp := NewEmergencyReparenter(nil, nil, logger) + res, err := erp.identifyPrimaryCandidate(logger, test.newPrimary, test.newPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) + if test.err != "" { + assert.EqualError(t, err, test.err) + return + } + assert.NoError(t, err) + assert.True(t, topoproto.TabletAliasEqual(res.Alias, test.result.Alias)) + }) + } +} diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index e2eabd7a26d..1f6b3b014e1 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -18,20 +18,11 @@ package reparentutil import ( "context" - "fmt" "sync" "time" - "vitess.io/vitess/go/event" - - "vitess.io/vitess/go/vt/concurrency" - "vitess.io/vitess/go/mysql" - replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" - - "vitess.io/vitess/go/vt/topotools/events" - "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" @@ -43,204 +34,6 @@ import ( "vitess.io/vitess/go/vt/proto/vtrpc" ) -func waitForAllRelayLogsToApply(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, waitReplicasTimeout time.Duration) error { - errCh := make(chan error) - defer close(errCh) - - groupCtx, groupCancel := context.WithTimeout(ctx, waitReplicasTimeout) - defer groupCancel() - - waiterCount := 0 - - for candidate := range validCandidates { - // When we called StopReplicationAndBuildStatusMaps, we got back two - // maps: (1) the StopReplicationStatus of any replicas that actually - // stopped replication; and (2) the MasterStatus of anything that - // returned ErrNotReplica, which is a tablet that is either the current - // primary or is stuck thinking it is a MASTER but is not in actuality. - // - // If we have a tablet in the validCandidates map that does not appear - // in the statusMap, then we have either (a) the current primary, which - // is not replicating, so it is not applying relay logs; or (b) a tablet - // that is stuck thinking it is MASTER but is not in actuality. In that - // second case - (b) - we will most likely find that the stuck MASTER - // does not have a winning position, and fail the ERS. If, on the other - // hand, it does have a winning position, we are trusting the operator - // to know what they are doing by emergency-reparenting onto that - // tablet. In either case, it does not make sense to wait for relay logs - // to apply on a tablet that was never applying relay logs in the first - // place, so we skip it, and log that we did. - status, ok := statusMap[candidate] - if !ok { - logger.Infof("EmergencyReparent candidate %v not in replica status map; this means it was not running replication (because it was formerly PRIMARY), so skipping WaitForRelayLogsToApply step for this candidate", candidate) - continue - } - - go func(alias string, status *replicationdatapb.StopReplicationStatus) { - var err error - defer func() { errCh <- err }() - err = WaitForRelayLogsToApply(groupCtx, tmc, tabletMap[alias], status) - }(candidate, status) - - waiterCount++ - } - - errgroup := concurrency.ErrorGroup{ - NumGoroutines: waiterCount, - NumRequiredSuccesses: waiterCount, - NumAllowedErrors: 0, - } - rec := errgroup.Wait(groupCancel, errCh) - - if len(rec.Errors) != 0 { - return vterrors.Wrapf(rec.Error(), "could not apply all relay logs within the provided waitReplicasTimeout (%s): %v", waitReplicasTimeout, rec.Error()) - } - - return nil -} - -// reparentReplicas reparents all the replicas provided and populates the reparent journal on the primary. -// Also, it returns the replicas which started replicating only in the case where we wait for all the replicas -func reparentReplicas(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, newPrimaryTablet *topodatapb.Tablet, lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions, waitForAllReplicas bool, populateReparentJournal bool) ([]*topodatapb.Tablet, error) { - - var replicasStartedReplication []*topodatapb.Tablet - var replicaMutex sync.Mutex - - replCtx, replCancel := context.WithTimeout(ctx, opts.waitReplicasTimeout) - - event.DispatchUpdate(ev, "reparenting all tablets") - - // Create a context and cancel function to watch for the first successful - // SetMaster call on a replica. We use a background context so that this - // context is only ever Done when its cancel is called by the background - // goroutine we're about to spin up. - // - // Similarly, create a context and cancel for the replica waiter goroutine - // to signal when all replica goroutines have finished. In the case where at - // least one replica succeeds, replSuccessCtx will be canceled first, while - // allReplicasDoneCtx is guaranteed to be canceled within - // opts.WaitReplicasTimeout plus some jitter. - replSuccessCtx, replSuccessCancel := context.WithCancel(context.Background()) - allReplicasDoneCtx, allReplicasDoneCancel := context.WithCancel(context.Background()) - - now := time.Now().UnixNano() - replWg := sync.WaitGroup{} - rec := concurrency.AllErrorRecorder{} - - handlePrimary := func(alias string, tablet *topodatapb.Tablet) error { - position, err := tmc.MasterPosition(replCtx, tablet) - if err != nil { - return err - } - if populateReparentJournal { - logger.Infof("populating reparent journal on new primary %v", alias) - return tmc.PopulateReparentJournal(replCtx, tablet, now, lockAction, newPrimaryTablet.Alias, position) - } - return nil - } - - handleReplica := func(alias string, ti *topo.TabletInfo) { - defer replWg.Done() - logger.Infof("setting new primary on replica %v", alias) - - forceStart := false - if status, ok := statusMap[alias]; ok { - fs, err := ReplicaWasRunning(status) - if err != nil { - err = vterrors.Wrapf(err, "tablet %v could not determine StopReplicationStatus: %v", alias, err) - rec.RecordError(err) - - return - } - - forceStart = fs - } - - err := tmc.SetMaster(replCtx, ti.Tablet, newPrimaryTablet.Alias, 0, "", forceStart) - if err != nil { - err = vterrors.Wrapf(err, "tablet %v SetReplicationSource failed: %v", alias, err) - rec.RecordError(err) - - return - } - - replicaMutex.Lock() - replicasStartedReplication = append(replicasStartedReplication, ti.Tablet) - replicaMutex.Unlock() - - // Signal that at least one goroutine succeeded to SetReplicationSource. - // We do this only when we do not want to wait for all the replicas - if !waitForAllReplicas { - replSuccessCancel() - } - } - - numReplicas := 0 - - for alias, ti := range tabletMap { - switch { - case alias == topoproto.TabletAliasString(newPrimaryTablet.Alias): - continue - case !opts.ignoreReplicas.Has(alias): - replWg.Add(1) - numReplicas++ - go handleReplica(alias, ti) - } - } - - // Spin up a background goroutine to wait until all replica goroutines - // finished. Polling this way allows us to have reparentReplicas return - // success as soon as (a) the primary successfully populates its reparent - // journal and (b) at least one replica successfully begins replicating. - // - // If we were to follow the more common pattern of blocking on replWg.Wait() - // in the main body of promoteNewPrimary, we would be bound to the - // time of slowest replica, instead of the time of the fastest successful - // replica, and we want ERS to be fast. - go func() { - replWg.Wait() - allReplicasDoneCancel() - }() - - primaryErr := handlePrimary(topoproto.TabletAliasString(newPrimaryTablet.Alias), newPrimaryTablet) - if primaryErr != nil { - logger.Warningf("primary failed to PopulateReparentJournal") - replCancel() - - return nil, vterrors.Wrapf(primaryErr, "failed to PopulateReparentJournal on primary: %v", primaryErr) - } - - go func() { - replWg.Wait() - defer replCancel() - }() - - select { - case <-replSuccessCtx.Done(): - // At least one replica was able to SetMaster successfully - // Here we do not need to return the replicas which started replicating - return nil, nil - case <-allReplicasDoneCtx.Done(): - // There are certain timing issues between replSuccessCtx.Done firing - // and allReplicasDoneCtx.Done firing, so we check again if truly all - // replicas failed (where `numReplicas` goroutines recorded an error) or - // one or more actually managed to succeed. - errCount := len(rec.Errors) - - switch { - case errCount > numReplicas: - // Technically, rec.Errors should never be greater than numReplicas, - // but it's better to err on the side of caution here, but also - // we're going to be explicit that this is doubly unexpected. - return nil, vterrors.Wrapf(rec.Error(), "received more errors (= %d) than replicas (= %d), which should be impossible: %v", errCount, numReplicas, rec.Error()) - case errCount == numReplicas: - return nil, vterrors.Wrapf(rec.Error(), "%d replica(s) failed: %v", numReplicas, rec.Error()) - default: - return replicasStartedReplication, nil - } - } -} - // ChooseNewPrimary finds a tablet that should become a primary after reparent. // The criteria for the new primary-elect are (preferably) to be in the same // cell as the current primary, and to be different from avoidPrimaryAlias. The @@ -354,110 +147,6 @@ func FindCurrentPrimary(tabletMap map[string]*topo.TabletInfo, logger logutil.Lo return currentPrimary } -// waitForCatchingUp promotes the newer candidate over the primary candidate that we have, but it does not set to start accepting writes -func waitForCatchingUp(ctx context.Context, tmc tmclient.TabletManagerClient, logger logutil.Logger, prevPrimary, newPrimary *topodatapb.Tablet, waitTime time.Duration) error { - logger.Infof("waiting for %v to catch up to %v", newPrimary.Alias, prevPrimary.Alias) - // Find the primary position of the previous primary - pos, err := tmc.MasterPosition(ctx, prevPrimary) - if err != nil { - return err - } - - // Wait until the new primary has caught upto that position - waitForPosCtx, cancelFunc := context.WithTimeout(ctx, waitTime) - defer cancelFunc() - err = tmc.WaitForPosition(waitForPosCtx, newPrimary, pos) - if err != nil { - return err - } - return nil -} - -func getLockAction(newPrimaryAlias *topodatapb.TabletAlias) string { - action := "EmergencyReparentShard" - - if newPrimaryAlias != nil { - action += fmt.Sprintf("(%v)", topoproto.TabletAliasString(newPrimaryAlias)) - } - - return action -} - -// restrictValidCandidates is used to restrict some candidates from being considered eligible for becoming the intermediate primary -func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { - restrictedValidCandidates := make(map[string]mysql.Position) - for candidate, position := range validCandidates { - candidateInfo, ok := tabletMap[candidate] - if !ok { - return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) - } - // We do not allow BACKUP, DRAINED or RESTORE type of tablets to be considered for being the replication source or the candidate for primary - if candidateInfo.Type == topodatapb.TabletType_BACKUP || candidateInfo.Type == topodatapb.TabletType_RESTORE || candidateInfo.Type == topodatapb.TabletType_DRAINED { - continue - } - restrictedValidCandidates[candidate] = position - } - return restrictedValidCandidates, nil -} - -// findMostAdvanced finds the intermediate primary candidate for ERS. We always choose the most advanced one from our valid candidates list -func findMostAdvanced(logger logutil.Logger, prevPrimary *topodatapb.Tablet, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (*topodatapb.Tablet, []*topodatapb.Tablet, error) { - logger.Infof("started finding the intermediate primary candidate") - // convert the valid candidates into a list so that we can use it for sorting - validTablets, tabletPositions, err := getValidCandidatesAndPositionsAsList(validCandidates, tabletMap) - if err != nil { - return nil, nil, err - } - - idealCell := "" - if prevPrimary != nil { - idealCell = prevPrimary.Alias.Cell - } - - // sort the tablets for finding the best intermediate primary in ERS - err = sortTabletsForERS(validTablets, tabletPositions, idealCell) - if err != nil { - return nil, nil, err - } - for _, tablet := range validTablets { - logger.Infof("finding intermediate primary - sorted replica: %v", tablet.Alias) - } - - // The first tablet in the sorted list will be the most eligible candidate unless explicitly asked for some other tablet - winningPrimaryTablet := validTablets[0] - winningPosition := tabletPositions[0] - - // We have already removed the tablets with errant GTIDs before calling this function. At this point our winning position must be a - // superset of all the other valid positions. If that is not the case, then we have a split brain scenario, and we should cancel the ERS - for i, position := range tabletPositions { - if !winningPosition.AtLeast(position) { - return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "split brain detected between servers - %v and %v", winningPrimaryTablet.Alias, validTablets[i].Alias) - } - } - - // If we were requested to elect a particular primary, verify it's a valid - // candidate (non-zero position, no errant GTIDs) - // Also, if the candidate is - if opts.newPrimaryAlias != nil { - requestedPrimaryAlias := topoproto.TabletAliasString(opts.newPrimaryAlias) - pos, ok := validCandidates[requestedPrimaryAlias] - if !ok { - return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "requested primary elect %v has errant GTIDs", requestedPrimaryAlias) - } - // if the requested tablet is as advanced as the most advanced tablet, then we can just use it for promotion. - // otherwise, we should let it catchup to the most advanced tablet and let it be the intermediate primary - if pos.AtLeast(winningPosition) { - requestedPrimaryInfo, isFound := tabletMap[requestedPrimaryAlias] - if !isFound { - return nil, nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", requestedPrimaryAlias) - } - winningPrimaryTablet = requestedPrimaryInfo.Tablet - } - } - - return winningPrimaryTablet, validTablets, nil -} - // getValidCandidatesAndPositionsAsList converts the valid candidates from a map to a list of tablets, making it easier to sort func getValidCandidatesAndPositionsAsList(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) ([]*topodatapb.Tablet, []mysql.Position, error) { var validTablets []*topodatapb.Tablet @@ -473,106 +162,21 @@ func getValidCandidatesAndPositionsAsList(validCandidates map[string]mysql.Posit return validTablets, tabletPositions, nil } -// intermediateCandidateIsIdeal is used to find whether the intermediate candidate that ERS chose is also the ideal one or not -func intermediateCandidateIsIdeal(logger logutil.Logger, newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (bool, error) { - // we try to find a better candidate with the current list of valid candidates, and if it matches our current primary candidate, then we return true - candidate, err := identifyPrimaryCandidate(logger, newPrimary, prevPrimary, validCandidates, tabletMap, opts) - if err != nil { - return false, err - } - return candidate == newPrimary, nil -} - -// identifyPrimaryCandidate is used to find a better candidate for ERS promotion -func identifyPrimaryCandidate(logger logutil.Logger, newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (candidate *topodatapb.Tablet, err error) { - defer func() { - if candidate != nil { - logger.Infof("found better candidate - %v", candidate.Alias) - } - }() - - if opts.newPrimaryAlias != nil { - // explicit request to promote a specific tablet - requestedPrimaryAlias := topoproto.TabletAliasString(opts.newPrimaryAlias) - requestedPrimaryInfo, isFound := tabletMap[requestedPrimaryAlias] - if !isFound { - return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", requestedPrimaryAlias) - } - for _, validCandidate := range validCandidates { - if topoproto.TabletAliasEqual(validCandidate.Alias, opts.newPrimaryAlias) { - return requestedPrimaryInfo.Tablet, nil - } - } - return nil, vterrors.Errorf(vtrpc.Code_ABORTED, "requested candidate %v is not in valid candidates list", requestedPrimaryAlias) - } - var preferredCandidates []*topodatapb.Tablet - var neutralReplicas []*topodatapb.Tablet - for _, candidate := range validCandidates { - promotionRule := PromotionRule(candidate) - if promotionRule == MustPromoteRule || promotionRule == PreferPromoteRule { - preferredCandidates = append(preferredCandidates, candidate) - } - if promotionRule == NeutralPromoteRule { - neutralReplicas = append(neutralReplicas, candidate) - } - } - - // So we've already promoted a replica. - // However, can we improve on our choice? Are there any replicas with better promotion rules? - // Maybe we actually promoted such a replica. Does that mean we should keep it? - // Maybe we promoted a "neutral", and some "prefer" server is available. - // Maybe we promoted a "prefer_not" - // Maybe we promoted a server in a different cell than the primary - // There's many options. We may wish to replace the server we promoted with a better one. - - // check whether the one we promoted is in the same cell and belongs to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, true, true) - if candidate != nil { - return candidate, nil - } - // check whether there is some other tablet in the same cell belonging to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, false, true) - if candidate != nil { - return candidate, nil - } - // we do not have a preferred candidate in the same cell - - if !opts.preventCrossCellPromotion { - // check whether the one we promoted belongs to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, true, false) - if candidate != nil { - return candidate, nil - } - // check whether there is some other tablet belonging to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, false, false) - if candidate != nil { - return candidate, nil - } - } - - // repeat the same process for the neutral candidates list - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, true, true) - if candidate != nil { - return candidate, nil - } - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, false, true) - if candidate != nil { - return candidate, nil - } - - if !opts.preventCrossCellPromotion { - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, true, false) - if candidate != nil { - return candidate, nil +// restrictValidCandidates is used to restrict some candidates from being considered eligible for becoming the intermediate primary +func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { + restrictedValidCandidates := make(map[string]mysql.Position) + for candidate, position := range validCandidates { + candidateInfo, ok := tabletMap[candidate] + if !ok { + return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) } - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, false, false) - if candidate != nil { - return candidate, nil + // We do not allow BACKUP, DRAINED or RESTORE type of tablets to be considered for being the replication source or the candidate for primary + if candidateInfo.Type == topodatapb.TabletType_BACKUP || candidateInfo.Type == topodatapb.TabletType_RESTORE || candidateInfo.Type == topodatapb.TabletType_DRAINED { + continue } + restrictedValidCandidates[candidate] = position } - - // return the one that we have if nothing found - return newPrimary, nil + return restrictedValidCandidates, nil } func findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary *topodatapb.Tablet, possibleCandidates []*topodatapb.Tablet, checkEqualPrimary bool, checkSameCell bool) *topodatapb.Tablet { @@ -588,28 +192,21 @@ func findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary *topo return nil } -// promoteIntermediatePrimary promotes the primary candidate that we have, but it does not yet set to start accepting writes -func promoteIntermediatePrimary(ctx context.Context, tmc tmclient.TabletManagerClient, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, - lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions) ([]*topodatapb.Tablet, error) { - // we reparent all the other tablets to start replication from our new primary - // we wait for all the replicas so that we can choose a better candidate from the ones that started replication later - validCandidatesForImprovement, err := reparentReplicas(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, opts, true, false) +// waitForCatchUp promotes the newer candidate over the primary candidate that we have, but it does not set to start accepting writes +func waitForCatchUp(ctx context.Context, tmc tmclient.TabletManagerClient, logger logutil.Logger, prevPrimary, newPrimary *topodatapb.Tablet, waitTime time.Duration) error { + logger.Infof("waiting for %v to catch up to %v", newPrimary.Alias, prevPrimary.Alias) + // Find the primary position of the previous primary + pos, err := tmc.MasterPosition(ctx, prevPrimary) if err != nil { - return nil, err + return err } - // also include the current tablet for being considered as part of valid candidates for ERS promotion - validCandidatesForImprovement = append(validCandidatesForImprovement, newPrimary) - return validCandidatesForImprovement, nil -} - -// checkIfConstraintsSatisfied is used to check whether the constraints for ERS are satisfied or not. -func checkIfConstraintsSatisfied(newPrimary, prevPrimary *topodatapb.Tablet, opts EmergencyReparentOptions) error { - if opts.preventCrossCellPromotion && prevPrimary != nil && newPrimary.Alias.Cell != prevPrimary.Alias.Cell { - return vterrors.Errorf(vtrpc.Code_ABORTED, "elected primary does not satisfy geographic constraint - %s", topoproto.TabletAliasString(newPrimary.Alias)) - } - if PromotionRule(newPrimary) == MustNotPromoteRule { - return vterrors.Errorf(vtrpc.Code_ABORTED, "elected primary does not satisfy promotion rule constraint - %s", topoproto.TabletAliasString(newPrimary.Alias)) + // Wait until the new primary has caught upto that position + waitForPosCtx, cancelFunc := context.WithTimeout(ctx, waitTime) + defer cancelFunc() + err = tmc.WaitForPosition(waitForPosCtx, newPrimary, pos) + if err != nil { + return err } return nil } diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index d55c0fd418c..ec7a7103205 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -24,15 +24,8 @@ import ( "vitess.io/vitess/go/mysql" - "k8s.io/apimachinery/pkg/util/sets" - - vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" - "vitess.io/vitess/go/vt/topo/memorytopo" - "vitess.io/vitess/go/vt/topotools/events" "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" - "github.com/stretchr/testify/require" - "vitess.io/vitess/go/test/utils" "github.com/stretchr/testify/assert" @@ -526,132 +519,46 @@ func TestFindCurrentPrimary(t *testing.T) { } } -func TestCheckIfConstraintsSatisfied(t *testing.T) { - testcases := []struct { - name string - newPrimary, prevPrimary *topodatapb.Tablet - opts EmergencyReparentOptions - err string - }{ - { - name: "no constraint failure", - newPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "cell1", - }, - Type: topodatapb.TabletType_REPLICA, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "cell1", - }, - }, - opts: EmergencyReparentOptions{preventCrossCellPromotion: true}, - err: "", - }, { - name: "promotion rule constraint failure", - newPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "cell1", - Uid: 100, - }, - Type: topodatapb.TabletType_RDONLY, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "cell1", - }, - }, - opts: EmergencyReparentOptions{preventCrossCellPromotion: true}, - err: "elected primary does not satisfy promotion rule constraint - cell1-0000000100", - }, { - name: "cross cell constraint failure", - newPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "cell1", - Uid: 100, - }, - Type: topodatapb.TabletType_REPLICA, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "cell2", - }, - }, - opts: EmergencyReparentOptions{preventCrossCellPromotion: true}, - err: "elected primary does not satisfy geographic constraint - cell1-0000000100", - }, { - name: "cross cell but no constraint failure", - newPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "cell1", - Uid: 100, - }, - Type: topodatapb.TabletType_REPLICA, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "cell2", - }, - }, - opts: EmergencyReparentOptions{preventCrossCellPromotion: false}, - err: "", - }, +func TestGetValidCandidatesAndPositionsAsList(t *testing.T) { + sid1 := mysql.SID{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + mysqlGTID1 := mysql.Mysql56GTID{ + Server: sid1, + Sequence: 9, } - - _ = SetDurabilityPolicy("none", nil) - for _, testcase := range testcases { - t.Run(testcase.name, func(t *testing.T) { - err := checkIfConstraintsSatisfied(testcase.newPrimary, testcase.prevPrimary, testcase.opts) - if testcase.err == "" { - require.NoError(t, err) - } else { - require.EqualError(t, err, testcase.err) - } - }) + mysqlGTID2 := mysql.Mysql56GTID{ + Server: sid1, + Sequence: 10, + } + mysqlGTID3 := mysql.Mysql56GTID{ + Server: sid1, + Sequence: 11, } -} -func TestReparentReplicas(t *testing.T) { - t.Parallel() + positionMostAdvanced := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID1) + positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID2) + positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID3) + + positionIntermediate1 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionIntermediate1.GTIDSet = positionIntermediate1.GTIDSet.AddGTID(mysqlGTID1) + + positionIntermediate2 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} + positionIntermediate2.GTIDSet = positionIntermediate2.GTIDSet.AddGTID(mysqlGTID1) + positionIntermediate2.GTIDSet = positionIntermediate2.GTIDSet.AddGTID(mysqlGTID2) tests := []struct { - name string - emergencyReparentOps EmergencyReparentOptions - tmc *testutil.TabletManagerClient - unlockTopo bool - newPrimaryTabletAlias string - ts *topo.Server - keyspace string - shard string - tablets []*topodatapb.Tablet - tabletMap map[string]*topo.TabletInfo - statusMap map[string]*replicationdatapb.StopReplicationStatus - shouldErr bool - errShouldContain string + name string + validCandidates map[string]mysql.Position + tabletMap map[string]*topo.TabletInfo + tabletRes []*topodatapb.Tablet }{ { - name: "success", - emergencyReparentOps: NewEmergencyReparentOptions(nil, sets.NewString("zone1-0000000404"), 0, false), - tmc: &testutil.TabletManagerClient{ - PopulateReparentJournalResults: map[string]error{ - "zone1-0000000100": nil, - }, - MasterPositionResults: map[string]struct { - Position string - Error error - }{ - "zone1-0000000100": { - Error: nil, - }, - }, - SetMasterResults: map[string]error{ - "zone1-0000000101": nil, - "zone1-0000000102": nil, - "zone1-0000000404": assert.AnError, // okay, because we're ignoring it. - }, + name: "test conversion", + validCandidates: map[string]mysql.Position{ + "zone1-0000000100": positionMostAdvanced, + "zone1-0000000101": positionIntermediate1, + "zone1-0000000102": positionIntermediate2, }, - newPrimaryTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { Tablet: &topodatapb.Tablet{ @@ -689,81 +596,176 @@ func TestReparentReplicas(t *testing.T) { }, }, }, - statusMap: map[string]*replicationdatapb.StopReplicationStatus{ - "zone1-0000000101": { // forceStart = false - Before: &replicationdatapb.Status{ - IoThreadRunning: false, - SqlThreadRunning: false, + tabletRes: []*topodatapb.Tablet{ + { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, }, - }, - "zone1-0000000102": { // forceStart = true - Before: &replicationdatapb.Status{ - IoThreadRunning: true, - SqlThreadRunning: true, + Hostname: "primary-elect", + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, { + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, }, + Hostname: "requires force start", }, }, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: false, }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + tabletRes, posRes, err := getValidCandidatesAndPositionsAsList(test.validCandidates, test.tabletMap) + assert.NoError(t, err) + assert.ElementsMatch(t, test.tabletRes, tabletRes) + assert.Equal(t, len(tabletRes), len(posRes)) + for i, tablet := range tabletRes { + assert.Equal(t, test.validCandidates[topoproto.TabletAliasString(tablet.Alias)], posRes[i]) + } + }) + } +} + +func TestWaitForCatchUp(t *testing.T) { + tests := []struct { + name string + tmc tmclient.TabletManagerClient + prevPrimary *topodatapb.Tablet + newPrimary *topodatapb.Tablet + err string + }{ { - name: "MasterPosition error", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + name: "success", tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string Error error }{ "zone1-0000000100": { - Error: fmt.Errorf("primary position error"), + Position: "abc", + Error: nil, + }, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000101": { + "abc": nil, }, }, }, - newPrimaryTabletAlias: "zone1-0000000100", - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + }, { + name: "error in primary position", + tmc: &testutil.TabletManagerClient{ + MasterPositionResults: map[string]struct { + Position string + Error error + }{ + "zone1-0000000100": { + Position: "abc", + Error: fmt.Errorf("found error in primary position"), }, }, - "zone1-0000000101": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000101": { + "abc": nil, }, }, }, - statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: true, - errShouldContain: "primary position error", - }, - { - name: "cannot repopulate reparent journal on new primary", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), - tmc: &testutil.TabletManagerClient{ - PopulateReparentJournalResults: map[string]error{ - "zone1-0000000100": assert.AnError, + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, }, + }, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + err: "found error in primary position", + }, { + name: "error in waiting for position", + tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string Error error }{ "zone1-0000000100": { - Error: nil, + Position: "abc", + Error: nil, + }, + }, + WaitForPositionResults: map[string]map[string]error{ + "zone1-0000000101": { + "abc": fmt.Errorf("found error in waiting for position"), }, }, }, - newPrimaryTabletAlias: "zone1-0000000100", + prevPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + }, + newPrimary: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + }, + err: "found error in waiting for position", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + logger := logutil.NewMemoryLogger() + err := waitForCatchUp(ctx, test.tmc, logger, test.prevPrimary, test.newPrimary, 2*time.Second) + if test.err != "" { + assert.EqualError(t, err, test.err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestRestrictValidCandidates(t *testing.T) { + tests := []struct { + name string + validCandidates map[string]mysql.Position + tabletMap map[string]*topo.TabletInfo + result map[string]mysql.Position + }{ + { + name: "remove invalid tablets", + validCandidates: map[string]mysql.Position{ + "zone1-0000000100": {}, + "zone1-0000000101": {}, + "zone1-0000000102": {}, + "zone1-0000000103": {}, + "zone1-0000000104": {}, + "zone1-0000000105": {}, + }, tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { Tablet: &topodatapb.Tablet{ @@ -771,6 +773,7 @@ func TestReparentReplicas(t *testing.T) { Cell: "zone1", Uid: 100, }, + Type: topodatapb.TabletType_PRIMARY, }, }, "zone1-0000000101": { @@ -779,1429 +782,7 @@ func TestReparentReplicas(t *testing.T) { Cell: "zone1", Uid: 101, }, - }, - }, - }, - statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: true, - errShouldContain: "failed to PopulateReparentJournal on primary", - }, - { - name: "all replicas failing to SetMaster does fail the promotion", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), - tmc: &testutil.TabletManagerClient{ - PopulateReparentJournalResults: map[string]error{ - "zone1-0000000100": nil, - }, - MasterPositionResults: map[string]struct { - Position string - Error error - }{ - "zone1-0000000100": { - Error: nil, - }, - }, - - SetMasterResults: map[string]error{ - // everyone fails, we all fail - "zone1-0000000101": assert.AnError, - "zone1-0000000102": assert.AnError, - }, - }, - newPrimaryTabletAlias: "zone1-0000000100", - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, - "zone1-0000000101": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - }, - "zone1-00000000102": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - }, - }, - }, - statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: true, - errShouldContain: " replica(s) failed", - }, - { - name: "all replicas slow to SetMaster does fail the promotion", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, time.Millisecond*10, false), - tmc: &testutil.TabletManagerClient{ - PopulateReparentJournalResults: map[string]error{ - "zone1-0000000100": nil, - }, - MasterPositionResults: map[string]struct { - Position string - Error error - }{ - "zone1-0000000100": { - Error: nil, - }, - }, - SetMasterDelays: map[string]time.Duration{ - // nothing is failing, we're just slow - "zone1-0000000101": time.Millisecond * 100, - "zone1-0000000102": time.Millisecond * 75, - }, - SetMasterResults: map[string]error{ - "zone1-0000000101": nil, - "zone1-0000000102": nil, - }, - }, - newPrimaryTabletAlias: "zone1-0000000100", - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, - "zone1-0000000101": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - }, - "zone1-0000000102": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - }, - }, - }, - statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - shouldErr: true, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - errShouldContain: "context deadline exceeded", - }, - { - name: "one replica failing to SetMaster does not fail the promotion", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), - tmc: &testutil.TabletManagerClient{ - PopulateReparentJournalResults: map[string]error{ - "zone1-0000000100": nil, - }, - MasterPositionResults: map[string]struct { - Position string - Error error - }{ - "zone1-0000000100": { - Error: nil, - }, - }, - SetMasterResults: map[string]error{ - "zone1-0000000101": nil, // this one succeeds, so we're good - "zone1-0000000102": assert.AnError, - }, - }, - newPrimaryTabletAlias: "zone1-0000000100", - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, - "zone1-0000000101": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - }, - "zone1-0000000102": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - }, - }, - }, - statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: false, - }, - } - - for _, tt := range tests { - tt := tt - - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - ctx := context.Background() - logger := logutil.NewMemoryLogger() - ev := &events.Reparent{} - - testutil.AddShards(ctx, t, tt.ts, &vtctldatapb.Shard{ - Keyspace: tt.keyspace, - Name: tt.shard, - }) - - if !tt.unlockTopo { - var ( - unlock func(*error) - lerr error - ) - - ctx, unlock, lerr = tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "test lock") - require.NoError(t, lerr, "could not lock %s/%s for test", tt.keyspace, tt.shard) - - defer func() { - unlock(&lerr) - require.NoError(t, lerr, "could not unlock %s/%s after test", tt.keyspace, tt.shard) - }() - } - tabletInfo := tt.tabletMap[tt.newPrimaryTabletAlias] - - _, err := reparentReplicas(ctx, ev, logger, tt.tmc, tabletInfo.Tablet, "", tt.tabletMap, tt.statusMap, tt.emergencyReparentOps, false, true) - if tt.shouldErr { - assert.Error(t, err) - assert.Contains(t, err.Error(), tt.errShouldContain) - return - } - - assert.NoError(t, err) - }) - } -} - -func TestPromoteIntermediatePrimary(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - emergencyReparentOps EmergencyReparentOptions - tmc *testutil.TabletManagerClient - unlockTopo bool - newPrimaryTabletAlias string - ts *topo.Server - keyspace string - shard string - tablets []*topodatapb.Tablet - tabletMap map[string]*topo.TabletInfo - statusMap map[string]*replicationdatapb.StopReplicationStatus - shouldErr bool - errShouldContain string - result []*topodatapb.Tablet - }{ - { - name: "success", - emergencyReparentOps: NewEmergencyReparentOptions(nil, sets.NewString("zone1-0000000404"), 0, false), - tmc: &testutil.TabletManagerClient{ - PopulateReparentJournalResults: map[string]error{ - "zone1-0000000100": nil, - }, - MasterPositionResults: map[string]struct { - Position string - Error error - }{ - "zone1-0000000100": { - Error: nil, - }, - }, - SetMasterResults: map[string]error{ - "zone1-0000000101": nil, - "zone1-0000000102": nil, - "zone1-0000000404": assert.AnError, // okay, because we're ignoring it. - }, - }, - newPrimaryTabletAlias: "zone1-0000000100", - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - Hostname: "primary-elect", - }, - }, - "zone1-0000000101": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - }, - "zone1-0000000102": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - Hostname: "requires force start", - }, - }, - "zone1-0000000404": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 404, - }, - Hostname: "ignored tablet", - }, - }, - }, - statusMap: map[string]*replicationdatapb.StopReplicationStatus{ - "zone1-0000000101": { // forceStart = false - Before: &replicationdatapb.Status{ - IoThreadRunning: false, - SqlThreadRunning: false, - }, - }, - "zone1-0000000102": { // forceStart = true - Before: &replicationdatapb.Status{ - IoThreadRunning: true, - SqlThreadRunning: true, - }, - }, - }, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: false, - result: []*topodatapb.Tablet{ - { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - Hostname: "primary-elect", - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - Hostname: "requires force start", - }, - }, - }, - { - name: "all replicas failed", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), - tmc: &testutil.TabletManagerClient{ - PopulateReparentJournalResults: map[string]error{ - "zone1-0000000100": nil, - }, - MasterPositionResults: map[string]struct { - Position string - Error error - }{ - "zone1-0000000100": { - Error: nil, - }, - }, - - SetMasterResults: map[string]error{ - // everyone fails, we all fail - "zone1-0000000101": assert.AnError, - "zone1-0000000102": assert.AnError, - }, - }, - newPrimaryTabletAlias: "zone1-0000000100", - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, - "zone1-0000000101": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - }, - "zone1-00000000102": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - }, - }, - }, - statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: true, - errShouldContain: " replica(s) failed", - }, - { - name: "one replica failed", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), - tmc: &testutil.TabletManagerClient{ - PopulateReparentJournalResults: map[string]error{ - "zone1-0000000100": nil, - }, - MasterPositionResults: map[string]struct { - Position string - Error error - }{ - "zone1-0000000100": { - Error: nil, - }, - }, - SetMasterResults: map[string]error{ - "zone1-0000000101": nil, // this one succeeds, so we're good - "zone1-0000000102": assert.AnError, - }, - }, - newPrimaryTabletAlias: "zone1-0000000100", - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, - "zone1-0000000101": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - }, - "zone1-0000000102": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - }, - }, - }, - statusMap: map[string]*replicationdatapb.StopReplicationStatus{}, - keyspace: "testkeyspace", - shard: "-", - ts: memorytopo.NewServer("zone1"), - shouldErr: false, - result: []*topodatapb.Tablet{ - { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - }, - }, - } - - for _, tt := range tests { - tt := tt - - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - ctx := context.Background() - logger := logutil.NewMemoryLogger() - ev := &events.Reparent{} - - testutil.AddShards(ctx, t, tt.ts, &vtctldatapb.Shard{ - Keyspace: tt.keyspace, - Name: tt.shard, - }) - - if !tt.unlockTopo { - var ( - unlock func(*error) - lerr error - ) - - ctx, unlock, lerr = tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "test lock") - require.NoError(t, lerr, "could not lock %s/%s for test", tt.keyspace, tt.shard) - - defer func() { - unlock(&lerr) - require.NoError(t, lerr, "could not unlock %s/%s after test", tt.keyspace, tt.shard) - }() - } - tabletInfo := tt.tabletMap[tt.newPrimaryTabletAlias] - - res, err := promoteIntermediatePrimary(ctx, tt.tmc, ev, logger, tabletInfo.Tablet, "", tt.tabletMap, tt.statusMap, tt.emergencyReparentOps) - if tt.shouldErr { - assert.Error(t, err) - assert.Contains(t, err.Error(), tt.errShouldContain) - return - } - - assert.NoError(t, err) - assert.ElementsMatch(t, tt.result, res) - }) - } -} - -func TestIdentifyPrimaryCandidate(t *testing.T) { - tests := []struct { - name string - emergencyReparentOps EmergencyReparentOptions - newPrimary *topodatapb.Tablet - prevPrimary *topodatapb.Tablet - validCandidates []*topodatapb.Tablet - tabletMap map[string]*topo.TabletInfo - err string - result *topodatapb.Tablet - }{ - { - name: "explicit request for a primary tablet", - emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }}, - newPrimary: nil, - prevPrimary: nil, - validCandidates: []*topodatapb.Tablet{ - { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, - }, - result: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, { - name: "explicit request for a primary tablet not in valid list", - emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }}, - newPrimary: nil, - prevPrimary: nil, - validCandidates: nil, - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, - }, - err: "requested candidate zone1-0000000100 is not in valid candidates list", - }, { - name: "explicit request for a primary tablet not in tablet map", - emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }}, - newPrimary: nil, - prevPrimary: nil, - validCandidates: []*topodatapb.Tablet{ - { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, - tabletMap: map[string]*topo.TabletInfo{}, - err: "candidate zone1-0000000100 not found in the tablet map; this an impossible situation", - }, { - name: "preferred candidate in the same cell same as our replica", - emergencyReparentOps: EmergencyReparentOptions{}, - newPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - }, - }, - validCandidates: []*topodatapb.Tablet{ - { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - Type: topodatapb.TabletType_REPLICA, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - Type: topodatapb.TabletType_REPLICA, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 100, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 101, - }, - Type: topodatapb.TabletType_PRIMARY, - }, - }, - tabletMap: nil, - result: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, { - name: "preferred candidate in the same cell different from original replica", - emergencyReparentOps: EmergencyReparentOptions{}, - newPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - }, - }, - validCandidates: []*topodatapb.Tablet{ - { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - Type: topodatapb.TabletType_REPLICA, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 100, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 101, - }, - Type: topodatapb.TabletType_PRIMARY, - }, - }, - tabletMap: nil, - result: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - }, { - name: "preferred candidate in the different cell same as original replica", - emergencyReparentOps: EmergencyReparentOptions{}, - newPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 101, - }, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - }, - }, - validCandidates: []*topodatapb.Tablet{ - { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 100, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 101, - }, - Type: topodatapb.TabletType_PRIMARY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 102, - }, - Type: topodatapb.TabletType_PRIMARY, - }, - }, - tabletMap: nil, - result: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 101, - }, - }, - }, { - name: "preferred candidate in the different cell different from original replica", - emergencyReparentOps: EmergencyReparentOptions{}, - newPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 101, - }, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - }, - }, - validCandidates: []*topodatapb.Tablet{ - { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 100, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 101, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 102, - }, - Type: topodatapb.TabletType_PRIMARY, - }, - }, - tabletMap: nil, - result: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 102, - }, - }, - }, { - name: "prevent cross cell promotion", - emergencyReparentOps: EmergencyReparentOptions{preventCrossCellPromotion: true}, - newPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - }, - }, - validCandidates: []*topodatapb.Tablet{ - { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 100, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 101, - }, - Type: topodatapb.TabletType_RDONLY, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 102, - }, - Type: topodatapb.TabletType_PRIMARY, - }, - }, - tabletMap: nil, - result: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - _ = SetDurabilityPolicy("none", nil) - logger := logutil.NewMemoryLogger() - res, err := identifyPrimaryCandidate(logger, test.newPrimary, test.newPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) - if test.err != "" { - assert.EqualError(t, err, test.err) - return - } - assert.NoError(t, err) - assert.True(t, topoproto.TabletAliasEqual(res.Alias, test.result.Alias)) - }) - } -} - -func TestGetValidCandidatesAndPositionsAsList(t *testing.T) { - sid1 := mysql.SID{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} - mysqlGTID1 := mysql.Mysql56GTID{ - Server: sid1, - Sequence: 9, - } - mysqlGTID2 := mysql.Mysql56GTID{ - Server: sid1, - Sequence: 10, - } - mysqlGTID3 := mysql.Mysql56GTID{ - Server: sid1, - Sequence: 11, - } - - positionMostAdvanced := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} - positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID1) - positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID2) - positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID3) - - positionIntermediate1 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} - positionIntermediate1.GTIDSet = positionIntermediate1.GTIDSet.AddGTID(mysqlGTID1) - - positionIntermediate2 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} - positionIntermediate2.GTIDSet = positionIntermediate2.GTIDSet.AddGTID(mysqlGTID1) - positionIntermediate2.GTIDSet = positionIntermediate2.GTIDSet.AddGTID(mysqlGTID2) - - tests := []struct { - name string - validCandidates map[string]mysql.Position - tabletMap map[string]*topo.TabletInfo - tabletRes []*topodatapb.Tablet - }{ - { - name: "test conversion", - validCandidates: map[string]mysql.Position{ - "zone1-0000000100": positionMostAdvanced, - "zone1-0000000101": positionIntermediate1, - "zone1-0000000102": positionIntermediate2, - }, - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - Hostname: "primary-elect", - }, - }, - "zone1-0000000101": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - }, - "zone1-0000000102": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - Hostname: "requires force start", - }, - }, - "zone1-0000000404": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 404, - }, - Hostname: "ignored tablet", - }, - }, - }, - tabletRes: []*topodatapb.Tablet{ - { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - Hostname: "primary-elect", - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, { - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - Hostname: "requires force start", - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - tabletRes, posRes, err := getValidCandidatesAndPositionsAsList(test.validCandidates, test.tabletMap) - assert.NoError(t, err) - assert.ElementsMatch(t, test.tabletRes, tabletRes) - assert.Equal(t, len(tabletRes), len(posRes)) - for i, tablet := range tabletRes { - assert.Equal(t, test.validCandidates[topoproto.TabletAliasString(tablet.Alias)], posRes[i]) - } - }) - } -} - -func TestWaitForCatchingUp(t *testing.T) { - tests := []struct { - name string - tmc tmclient.TabletManagerClient - prevPrimary *topodatapb.Tablet - newPrimary *topodatapb.Tablet - err string - }{ - { - name: "success", - tmc: &testutil.TabletManagerClient{ - MasterPositionResults: map[string]struct { - Position string - Error error - }{ - "zone1-0000000100": { - Position: "abc", - Error: nil, - }, - }, - WaitForPositionResults: map[string]map[string]error{ - "zone1-0000000101": { - "abc": nil, - }, - }, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - newPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - }, { - name: "error in primary position", - tmc: &testutil.TabletManagerClient{ - MasterPositionResults: map[string]struct { - Position string - Error error - }{ - "zone1-0000000100": { - Position: "abc", - Error: fmt.Errorf("found error in primary position"), - }, - }, - WaitForPositionResults: map[string]map[string]error{ - "zone1-0000000101": { - "abc": nil, - }, - }, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - newPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - err: "found error in primary position", - }, { - name: "error in waiting for position", - tmc: &testutil.TabletManagerClient{ - MasterPositionResults: map[string]struct { - Position string - Error error - }{ - "zone1-0000000100": { - Position: "abc", - Error: nil, - }, - }, - WaitForPositionResults: map[string]map[string]error{ - "zone1-0000000101": { - "abc": fmt.Errorf("found error in waiting for position"), - }, - }, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - newPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - err: "found error in waiting for position", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ctx := context.Background() - logger := logutil.NewMemoryLogger() - err := waitForCatchingUp(ctx, test.tmc, logger, test.prevPrimary, test.newPrimary, 2*time.Second) - if test.err != "" { - assert.EqualError(t, err, test.err) - } else { - assert.NoError(t, err) - } - }) - } -} - -func TestRestrictValidCandidates(t *testing.T) { - tests := []struct { - name string - validCandidates map[string]mysql.Position - tabletMap map[string]*topo.TabletInfo - result map[string]mysql.Position - }{ - { - name: "remove invalid tablets", - validCandidates: map[string]mysql.Position{ - "zone1-0000000100": {}, - "zone1-0000000101": {}, - "zone1-0000000102": {}, - "zone1-0000000103": {}, - "zone1-0000000104": {}, - "zone1-0000000105": {}, - }, - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - Type: topodatapb.TabletType_PRIMARY, - }, - }, - "zone1-0000000101": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - Type: topodatapb.TabletType_RDONLY, - }, - }, - "zone1-0000000102": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - Type: topodatapb.TabletType_RESTORE, - }, - }, - "zone1-0000000103": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 103, - }, - Type: topodatapb.TabletType_DRAINED, - }, - }, - "zone1-0000000104": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 104, - }, - Type: topodatapb.TabletType_SPARE, - }, - }, - "zone1-0000000105": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 103, - }, - Type: topodatapb.TabletType_BACKUP, - }, - }, - }, - result: map[string]mysql.Position{ - "zone1-0000000100": {}, - "zone1-0000000101": {}, - "zone1-0000000104": {}, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - res, err := restrictValidCandidates(test.validCandidates, test.tabletMap) - assert.NoError(t, err) - assert.Equal(t, res, test.result) - }) - } -} - -func TestFindMostAdvanced(t *testing.T) { - sid1 := mysql.SID{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} - mysqlGTID1 := mysql.Mysql56GTID{ - Server: sid1, - Sequence: 9, - } - mysqlGTID2 := mysql.Mysql56GTID{ - Server: sid1, - Sequence: 10, - } - mysqlGTID3 := mysql.Mysql56GTID{ - Server: sid1, - Sequence: 11, - } - - positionMostAdvanced := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} - positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID1) - positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID2) - positionMostAdvanced.GTIDSet = positionMostAdvanced.GTIDSet.AddGTID(mysqlGTID3) - - positionIntermediate1 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} - positionIntermediate1.GTIDSet = positionIntermediate1.GTIDSet.AddGTID(mysqlGTID1) - - positionIntermediate2 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} - positionIntermediate2.GTIDSet = positionIntermediate2.GTIDSet.AddGTID(mysqlGTID1) - positionIntermediate2.GTIDSet = positionIntermediate2.GTIDSet.AddGTID(mysqlGTID2) - - positionOnly2 := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} - positionOnly2.GTIDSet = positionOnly2.GTIDSet.AddGTID(mysqlGTID2) - - positionEmpty := mysql.Position{GTIDSet: mysql.Mysql56GTIDSet{}} - - tests := []struct { - name string - validCandidates map[string]mysql.Position - tabletMap map[string]*topo.TabletInfo - prevPrimary *topodatapb.Tablet - emergencyReparentOps EmergencyReparentOptions - result *topodatapb.Tablet - err string - }{ - { - name: "choose most advanced - with nil previous primary", - validCandidates: map[string]mysql.Position{ - "zone1-0000000100": positionMostAdvanced, - "zone1-0000000101": positionIntermediate1, - "zone1-0000000102": positionIntermediate2, - }, - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, - "zone1-0000000101": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - }, - "zone1-0000000102": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - }, - }, - "zone1-0000000404": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 404, - }, - Hostname: "ignored tablet", - }, - }, - }, - prevPrimary: nil, - result: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, { - name: "choose most advanced in the same cell of previous primary", - validCandidates: map[string]mysql.Position{ - "zone1-0000000100": positionMostAdvanced, - "zone1-0000000101": positionIntermediate1, - "zone2-0000000100": positionMostAdvanced, - }, - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, - "zone1-0000000101": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - }, - "zone2-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 100, - }, - }, - }, - "zone1-0000000404": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 404, - }, - Hostname: "ignored tablet", - }, - }, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - }, - }, - result: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, { - name: "choose most advanced with the best promotion rule", - validCandidates: map[string]mysql.Position{ - "zone1-0000000100": positionMostAdvanced, - "zone1-0000000101": positionIntermediate1, - "zone1-0000000102": positionMostAdvanced, - }, - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - Type: topodatapb.TabletType_PRIMARY, - }, - }, - "zone1-0000000101": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - }, - "zone1-0000000102": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - Type: topodatapb.TabletType_RDONLY, - }, - }, - "zone1-0000000404": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 404, - }, - Hostname: "ignored tablet", - }, - }, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - }, - }, - result: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, { - name: "choose most advanced with explicit request", - emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }}, - validCandidates: map[string]mysql.Position{ - "zone1-0000000100": positionMostAdvanced, - "zone1-0000000101": positionIntermediate1, - "zone1-0000000102": positionMostAdvanced, - }, - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - Type: topodatapb.TabletType_PRIMARY, - }, - }, - "zone1-0000000101": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, + Type: topodatapb.TabletType_RDONLY, }, }, "zone1-0000000102": { @@ -2210,99 +791,50 @@ func TestFindMostAdvanced(t *testing.T) { Cell: "zone1", Uid: 102, }, - Type: topodatapb.TabletType_RDONLY, - }, - }, - "zone1-0000000404": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 404, - }, - Hostname: "ignored tablet", - }, - }, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - }, - }, - result: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }, - }, - }, { - name: "split brain detection", - emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 102, - }}, - validCandidates: map[string]mysql.Position{ - "zone1-0000000100": positionOnly2, - "zone1-0000000101": positionIntermediate1, - "zone1-0000000102": positionEmpty, - }, - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - Type: topodatapb.TabletType_PRIMARY, + Type: topodatapb.TabletType_RESTORE, }, }, - "zone1-0000000101": { + "zone1-0000000103": { Tablet: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", - Uid: 101, + Uid: 103, }, + Type: topodatapb.TabletType_DRAINED, }, }, - "zone1-0000000102": { + "zone1-0000000104": { Tablet: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", - Uid: 102, + Uid: 104, }, - Type: topodatapb.TabletType_RDONLY, + Type: topodatapb.TabletType_SPARE, }, }, - "zone1-0000000404": { + "zone1-0000000105": { Tablet: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", - Uid: 404, + Uid: 103, }, - Hostname: "ignored tablet", + Type: topodatapb.TabletType_BACKUP, }, }, }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - }, + result: map[string]mysql.Position{ + "zone1-0000000100": {}, + "zone1-0000000101": {}, + "zone1-0000000104": {}, }, - err: "split brain detected between servers", }, } - _ = SetDurabilityPolicy("none", nil) - for _, test := range tests { t.Run(test.name, func(t *testing.T) { - winningTablet, _, err := findMostAdvanced(logutil.NewMemoryLogger(), test.prevPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) - if test.err != "" { - assert.Error(t, err) - assert.Contains(t, err.Error(), test.err) - } else { - assert.NoError(t, err) - assert.True(t, topoproto.TabletAliasEqual(test.result.Alias, winningTablet.Alias)) - } + res, err := restrictValidCandidates(test.validCandidates, test.tabletMap) + assert.NoError(t, err) + assert.Equal(t, res, test.result) }) } } From 3559ce1b570e906ceef2bcf5abbb97a65459a1d6 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 12:55:59 +0530 Subject: [PATCH 159/176] removed arguments available in the method Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter.go | 51 +++++++++---------- .../reparentutil/emergency_reparenter_test.go | 10 ++-- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 6481a313c9e..b919c28fd4f 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -214,7 +214,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // Wait for all candidates to apply relay logs - if err = erp.waitForAllRelayLogsToApply(ctx, erp.logger, erp.tmc, validCandidates, tabletMap, statusMap, opts.waitReplicasTimeout); err != nil { + if err = erp.waitForAllRelayLogsToApply(ctx, validCandidates, tabletMap, statusMap, opts.waitReplicasTimeout); err != nil { return err } @@ -222,7 +222,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // We let all the other tablets replicate from this tablet. We will then try to choose a better candidate and let it catch up var intermediateSource *topodatapb.Tablet var validCandidateTablets []*topodatapb.Tablet - intermediateSource, validCandidateTablets, err = erp.findMostAdvanced(erp.logger, prevPrimary, validCandidates, tabletMap, opts) + intermediateSource, validCandidateTablets, err = erp.findMostAdvanced(prevPrimary, validCandidates, tabletMap, opts) if err != nil { return err } @@ -230,7 +230,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // check weather the primary candidate selected is ideal or if it can be improved later var isIdeal bool - isIdeal, err = erp.intermediateCandidateIsIdeal(erp.logger, intermediateSource, prevPrimary, validCandidateTablets, tabletMap, opts) + isIdeal, err = erp.intermediateCandidateIsIdeal(intermediateSource, prevPrimary, validCandidateTablets, tabletMap, opts) if err != nil { return err } @@ -248,14 +248,14 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // we do not promote the tablet or change the shard record. We only change the replication for all the other tablets // it also returns the list of the tablets that started replication successfully including itself. These are the candidates that we can use to find a replacement var validReplacementCandidates []*topodatapb.Tablet - validReplacementCandidates, err = erp.promoteIntermediatePrimary(ctx, erp.tmc, ev, erp.logger, intermediateSource, opts.lockAction, tabletMap, statusMap, opts) + validReplacementCandidates, err = erp.promoteIntermediatePrimary(ctx, ev, intermediateSource, tabletMap, statusMap, opts) if err != nil { return err } // try to find a better candidate using the list we got back var betterCandidate *topodatapb.Tablet - betterCandidate, err = erp.identifyPrimaryCandidate(erp.logger, intermediateSource, prevPrimary, validReplacementCandidates, tabletMap, opts) + betterCandidate, err = erp.identifyPrimaryCandidate(intermediateSource, prevPrimary, validReplacementCandidates, tabletMap, opts) if err != nil { return err } @@ -299,7 +299,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } -func (erp *EmergencyReparenter) waitForAllRelayLogsToApply(ctx context.Context, logger logutil.Logger, tmc tmclient.TabletManagerClient, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, waitReplicasTimeout time.Duration) error { +func (erp *EmergencyReparenter) waitForAllRelayLogsToApply(ctx context.Context, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, waitReplicasTimeout time.Duration) error { errCh := make(chan error) defer close(errCh) @@ -328,14 +328,14 @@ func (erp *EmergencyReparenter) waitForAllRelayLogsToApply(ctx context.Context, // place, so we skip it, and log that we did. status, ok := statusMap[candidate] if !ok { - logger.Infof("EmergencyReparent candidate %v not in replica status map; this means it was not running replication (because it was formerly PRIMARY), so skipping WaitForRelayLogsToApply step for this candidate", candidate) + erp.logger.Infof("EmergencyReparent candidate %v not in replica status map; this means it was not running replication (because it was formerly PRIMARY), so skipping WaitForRelayLogsToApply step for this candidate", candidate) continue } go func(alias string, status *replicationdatapb.StopReplicationStatus) { var err error defer func() { errCh <- err }() - err = WaitForRelayLogsToApply(groupCtx, tmc, tabletMap[alias], status) + err = WaitForRelayLogsToApply(groupCtx, erp.tmc, tabletMap[alias], status) }(candidate, status) waiterCount++ @@ -356,8 +356,8 @@ func (erp *EmergencyReparenter) waitForAllRelayLogsToApply(ctx context.Context, } // findMostAdvanced finds the intermediate primary candidate for ERS. We always choose the most advanced one from our valid candidates list -func (erp *EmergencyReparenter) findMostAdvanced(logger logutil.Logger, prevPrimary *topodatapb.Tablet, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (*topodatapb.Tablet, []*topodatapb.Tablet, error) { - logger.Infof("started finding the intermediate primary candidate") +func (erp *EmergencyReparenter) findMostAdvanced(prevPrimary *topodatapb.Tablet, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (*topodatapb.Tablet, []*topodatapb.Tablet, error) { + erp.logger.Infof("started finding the intermediate primary candidate") // convert the valid candidates into a list so that we can use it for sorting validTablets, tabletPositions, err := getValidCandidatesAndPositionsAsList(validCandidates, tabletMap) if err != nil { @@ -375,7 +375,7 @@ func (erp *EmergencyReparenter) findMostAdvanced(logger logutil.Logger, prevPrim return nil, nil, err } for _, tablet := range validTablets { - logger.Infof("finding intermediate primary - sorted replica: %v", tablet.Alias) + erp.logger.Infof("finding intermediate primary - sorted replica: %v", tablet.Alias) } // The first tablet in the sorted list will be the most eligible candidate unless explicitly asked for some other tablet @@ -414,11 +414,10 @@ func (erp *EmergencyReparenter) findMostAdvanced(logger logutil.Logger, prevPrim } // promoteIntermediatePrimary promotes the primary candidate that we have, but it does not yet set to start accepting writes -func (erp *EmergencyReparenter) promoteIntermediatePrimary(ctx context.Context, tmc tmclient.TabletManagerClient, ev *events.Reparent, logger logutil.Logger, newPrimary *topodatapb.Tablet, - lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions) ([]*topodatapb.Tablet, error) { +func (erp *EmergencyReparenter) promoteIntermediatePrimary(ctx context.Context, ev *events.Reparent, newPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions) ([]*topodatapb.Tablet, error) { // we reparent all the other tablets to start replication from our new primary // we wait for all the replicas so that we can choose a better candidate from the ones that started replication later - validCandidatesForImprovement, err := erp.reparentReplicas(ctx, ev, logger, tmc, newPrimary, lockAction, tabletMap, statusMap, opts, true, false) + validCandidatesForImprovement, err := erp.reparentReplicas(ctx, ev, newPrimary, tabletMap, statusMap, opts, true, false) if err != nil { return nil, err } @@ -430,7 +429,7 @@ func (erp *EmergencyReparenter) promoteIntermediatePrimary(ctx context.Context, // reparentReplicas reparents all the replicas provided and populates the reparent journal on the primary. // Also, it returns the replicas which started replicating only in the case where we wait for all the replicas -func (erp *EmergencyReparenter) reparentReplicas(ctx context.Context, ev *events.Reparent, logger logutil.Logger, tmc tmclient.TabletManagerClient, newPrimaryTablet *topodatapb.Tablet, lockAction string, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions, waitForAllReplicas bool, populateReparentJournal bool) ([]*topodatapb.Tablet, error) { +func (erp *EmergencyReparenter) reparentReplicas(ctx context.Context, ev *events.Reparent, newPrimaryTablet *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions, waitForAllReplicas bool, populateReparentJournal bool) ([]*topodatapb.Tablet, error) { var replicasStartedReplication []*topodatapb.Tablet var replicaMutex sync.Mutex @@ -457,20 +456,20 @@ func (erp *EmergencyReparenter) reparentReplicas(ctx context.Context, ev *events rec := concurrency.AllErrorRecorder{} handlePrimary := func(alias string, tablet *topodatapb.Tablet) error { - position, err := tmc.MasterPosition(replCtx, tablet) + position, err := erp.tmc.MasterPosition(replCtx, tablet) if err != nil { return err } if populateReparentJournal { - logger.Infof("populating reparent journal on new primary %v", alias) - return tmc.PopulateReparentJournal(replCtx, tablet, now, lockAction, newPrimaryTablet.Alias, position) + erp.logger.Infof("populating reparent journal on new primary %v", alias) + return erp.tmc.PopulateReparentJournal(replCtx, tablet, now, opts.lockAction, newPrimaryTablet.Alias, position) } return nil } handleReplica := func(alias string, ti *topo.TabletInfo) { defer replWg.Done() - logger.Infof("setting new primary on replica %v", alias) + erp.logger.Infof("setting new primary on replica %v", alias) forceStart := false if status, ok := statusMap[alias]; ok { @@ -485,7 +484,7 @@ func (erp *EmergencyReparenter) reparentReplicas(ctx context.Context, ev *events forceStart = fs } - err := tmc.SetMaster(replCtx, ti.Tablet, newPrimaryTablet.Alias, 0, "", forceStart) + err := erp.tmc.SetMaster(replCtx, ti.Tablet, newPrimaryTablet.Alias, 0, "", forceStart) if err != nil { err = vterrors.Wrapf(err, "tablet %v SetReplicationSource failed: %v", alias, err) rec.RecordError(err) @@ -533,7 +532,7 @@ func (erp *EmergencyReparenter) reparentReplicas(ctx context.Context, ev *events primaryErr := handlePrimary(topoproto.TabletAliasString(newPrimaryTablet.Alias), newPrimaryTablet) if primaryErr != nil { - logger.Warningf("primary failed to PopulateReparentJournal") + erp.logger.Warningf("primary failed to PopulateReparentJournal") replCancel() return nil, vterrors.Wrapf(primaryErr, "failed to PopulateReparentJournal on primary: %v", primaryErr) @@ -572,9 +571,9 @@ func (erp *EmergencyReparenter) reparentReplicas(ctx context.Context, ev *events } // intermediateCandidateIsIdeal is used to find whether the intermediate candidate that ERS chose is also the ideal one or not -func (erp *EmergencyReparenter) intermediateCandidateIsIdeal(logger logutil.Logger, newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (bool, error) { +func (erp *EmergencyReparenter) intermediateCandidateIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (bool, error) { // we try to find a better candidate with the current list of valid candidates, and if it matches our current primary candidate, then we return true - candidate, err := erp.identifyPrimaryCandidate(logger, newPrimary, prevPrimary, validCandidates, tabletMap, opts) + candidate, err := erp.identifyPrimaryCandidate(newPrimary, prevPrimary, validCandidates, tabletMap, opts) if err != nil { return false, err } @@ -582,10 +581,10 @@ func (erp *EmergencyReparenter) intermediateCandidateIsIdeal(logger logutil.Logg } // identifyPrimaryCandidate is used to find a better candidate for ERS promotion -func (erp *EmergencyReparenter) identifyPrimaryCandidate(logger logutil.Logger, newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (candidate *topodatapb.Tablet, err error) { +func (erp *EmergencyReparenter) identifyPrimaryCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (candidate *topodatapb.Tablet, err error) { defer func() { if candidate != nil { - logger.Infof("found better candidate - %v", candidate.Alias) + erp.logger.Infof("found better candidate - %v", candidate.Alias) } }() @@ -692,7 +691,7 @@ func (erp *EmergencyReparenter) promoteNewPrimary(ctx context.Context, ev *event return vterrors.Wrapf(err, "primary-elect tablet %v failed to be upgraded to primary: %v", newPrimary.Alias, err) } // we now reparent all the replicas to the new primary we have promoted - _, err = erp.reparentReplicas(ctx, ev, erp.logger, erp.tmc, newPrimary, opts.lockAction, tabletMap, statusMap, opts, false, true) + _, err = erp.reparentReplicas(ctx, ev, newPrimary, tabletMap, statusMap, opts, false, true) if err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 2bff375ed54..d9cfa1ff2ba 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -2068,7 +2068,7 @@ func TestEmergencyReparenter_waitForAllRelayLogsToApply(t *testing.T) { t.Parallel() erp := NewEmergencyReparenter(nil, tt.tmc, logger) - err := erp.waitForAllRelayLogsToApply(ctx, logger, tt.tmc, tt.candidates, tt.tabletMap, tt.statusMap, waitReplicasTimeout) + err := erp.waitForAllRelayLogsToApply(ctx, tt.candidates, tt.tabletMap, tt.statusMap, waitReplicasTimeout) if tt.shouldErr { assert.Error(t, err) return @@ -2551,7 +2551,7 @@ func TestEmergencyReparenter_findMostAdvanced(t *testing.T) { t.Run(test.name, func(t *testing.T) { erp := NewEmergencyReparenter(nil, nil, logutil.NewMemoryLogger()) - winningTablet, _, err := erp.findMostAdvanced(logutil.NewMemoryLogger(), test.prevPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) + winningTablet, _, err := erp.findMostAdvanced(test.prevPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) if test.err != "" { assert.Error(t, err) assert.Contains(t, err.Error(), test.err) @@ -3030,7 +3030,7 @@ func TestEmergencyReparenter_reparentReplicas(t *testing.T) { tabletInfo := tt.tabletMap[tt.newPrimaryTabletAlias] erp := NewEmergencyReparenter(tt.ts, tt.tmc, logger) - _, err := erp.reparentReplicas(ctx, ev, logger, tt.tmc, tabletInfo.Tablet, "", tt.tabletMap, tt.statusMap, tt.emergencyReparentOps, false, true) + _, err := erp.reparentReplicas(ctx, ev, tabletInfo.Tablet, tt.tabletMap, tt.statusMap, tt.emergencyReparentOps, false, true) if tt.shouldErr { assert.Error(t, err) assert.Contains(t, err.Error(), tt.errShouldContain) @@ -3316,7 +3316,7 @@ func TestEmergencyReparenter_promoteIntermediatePrimary(t *testing.T) { tabletInfo := tt.tabletMap[tt.newPrimaryTabletAlias] erp := NewEmergencyReparenter(tt.ts, tt.tmc, logger) - res, err := erp.promoteIntermediatePrimary(ctx, tt.tmc, ev, logger, tabletInfo.Tablet, "", tt.tabletMap, tt.statusMap, tt.emergencyReparentOps) + res, err := erp.promoteIntermediatePrimary(ctx, ev, tabletInfo.Tablet, tt.tabletMap, tt.statusMap, tt.emergencyReparentOps) if tt.shouldErr { assert.Error(t, err) assert.Contains(t, err.Error(), tt.errShouldContain) @@ -3689,7 +3689,7 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { logger := logutil.NewMemoryLogger() erp := NewEmergencyReparenter(nil, nil, logger) - res, err := erp.identifyPrimaryCandidate(logger, test.newPrimary, test.newPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) + res, err := erp.identifyPrimaryCandidate(test.newPrimary, test.newPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) if test.err != "" { assert.EqualError(t, err, test.err) return From 533bc115329550aab40b57b917b45bb7b026fbc5 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 13:08:10 +0530 Subject: [PATCH 160/176] fix more imports Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/durability_test.go | 6 ++--- .../reparentutil/emergency_reparenter.go | 27 +++++++------------ .../reparentutil/emergency_reparenter_test.go | 3 +-- go/vt/vtctl/reparentutil/util.go | 1 - go/vt/vtctl/reparentutil/util_test.go | 16 +++++------ 5 files changed, 18 insertions(+), 35 deletions(-) diff --git a/go/vt/vtctl/reparentutil/durability_test.go b/go/vt/vtctl/reparentutil/durability_test.go index 4dfd09f6aee..dc65cccc293 100644 --- a/go/vt/vtctl/reparentutil/durability_test.go +++ b/go/vt/vtctl/reparentutil/durability_test.go @@ -19,13 +19,11 @@ package reparentutil import ( "testing" - "github.com/stretchr/testify/require" - - "vitess.io/vitess/go/vt/topo/topoproto" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo/topoproto" ) func TestDurabilityNone(t *testing.T) { diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index b919c28fd4f..cea435a02cc 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -22,32 +22,23 @@ import ( "sync" "time" - "vitess.io/vitess/go/vt/concurrency" + "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/util/sets" + "vitess.io/vitess/go/event" "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/stats" - + "vitess.io/vitess/go/vt/concurrency" + "vitess.io/vitess/go/vt/logutil" + logutilpb "vitess.io/vitess/go/vt/proto/logutil" replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" - - "vitess.io/vitess/go/vt/topo/topoproto" - - "vitess.io/vitess/go/vt/proto/vtrpc" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - - "google.golang.org/protobuf/proto" - + "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/topo" - "vitess.io/vitess/go/vt/vterrors" - - "vitess.io/vitess/go/event" - - "vitess.io/vitess/go/vt/logutil" - logutilpb "vitess.io/vitess/go/vt/proto/logutil" + "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/topotools/events" + "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vttablet/tmclient" ) @@ -112,7 +103,7 @@ func NewEmergencyReparenter(ts *topo.Server, tmc tmclient.TabletManagerClient, l // ReparentShard performs the EmergencyReparentShard operation on the given // keyspace and shard. -func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace, shard string, opts EmergencyReparentOptions) (*events.Reparent, error) { +func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace string, shard string, opts EmergencyReparentOptions) (*events.Reparent, error) { // First step is to lock the shard for the given operation opts.lockAction = erp.getLockAction(opts.newPrimaryAlias) ctx, unlock, err := erp.ts.LockShard(ctx, keyspace, shard, opts.lockAction) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index d9cfa1ff2ba..7105f8029d5 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -22,8 +22,6 @@ import ( "testing" "time" - "vitess.io/vitess/go/vt/topo/topoproto" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/util/sets" @@ -32,6 +30,7 @@ import ( "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/memorytopo" + "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/topotools/events" "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 1f6b3b014e1..50eaecd0a32 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -22,7 +22,6 @@ import ( "time" "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index ec7a7103205..cf3fa17ae5f 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -22,22 +22,18 @@ import ( "testing" "time" - "vitess.io/vitess/go/mysql" - - "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" - - "vitess.io/vitess/go/test/utils" - "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/utils" "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/topo" - "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/vttablet/tmclient" - replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/proto/vttime" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" + "vitess.io/vitess/go/vt/vttablet/tmclient" ) type chooseNewPrimaryTestTMClient struct { From 243673a05e7e21733bf7d47508b168e2cc41ba0a Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 13:21:33 +0530 Subject: [PATCH 161/176] removed the NewEmergencyReparentOptions function Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/topology_recovery.go | 12 ++- go/vt/vtctl/grpcvtctldserver/server.go | 8 +- .../reparentutil/emergency_reparenter.go | 62 ++++++------- .../reparentutil/emergency_reparenter_test.go | 92 ++++++++++--------- go/vt/wrangler/reparent.go | 8 +- 5 files changed, 97 insertions(+), 85 deletions(-) diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 68cf04a2761..51b45962cb8 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -659,7 +659,6 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat atomic.AddInt32(&shardsLockCounter, 1) defer atomic.AddInt32(&shardsLockCounter, -1) - reparentFunctions := reparentutil.NewEmergencyReparentOptions(candidateTabletAlias, nil, time.Duration(config.Config.WaitReplicasTimeoutSeconds)*time.Second, config.Config.PreventCrossDataCenterPrimaryFailover) ev, err := reparentutil.NewEmergencyReparenter(ts, tmclient.NewTabletManagerClient(), logutil.NewCallbackLogger(func(event *logutilpb.Event) { level := event.GetLevel() value := event.GetValue() @@ -671,7 +670,16 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat log.Errorf("ERS - %s", value) } AuditTopologyRecovery(topologyRecovery, value) - })).ReparentShard(context.Background(), tablet.Keyspace, tablet.Shard, reparentFunctions) + })).ReparentShard(context.Background(), + tablet.Keyspace, + tablet.Shard, + reparentutil.EmergencyReparentOptions{ + NewPrimaryAlias: candidateTabletAlias, + IgnoreReplicas: nil, + WaitReplicasTimeout: time.Duration(config.Config.WaitReplicasTimeoutSeconds) * time.Second, + PreventCrossCellPromotion: config.Config.PreventCrossDataCenterPrimaryFailover, + }, + ) // here we need to forcefully refresh all the tablets otherwise old information is used and failover scenarios are spawned off which are not required // For example, if we do not refresh the tablets forcefully and the new primary is found in the cache then its source key is not updated and this spawns off diff --git a/go/vt/vtctl/grpcvtctldserver/server.go b/go/vt/vtctl/grpcvtctldserver/server.go index 608df83ee5f..df81810c2a3 100644 --- a/go/vt/vtctl/grpcvtctldserver/server.go +++ b/go/vt/vtctl/grpcvtctldserver/server.go @@ -609,7 +609,13 @@ func (s *VtctldServer) EmergencyReparentShard(ctx context.Context, req *vtctldat ev, err := reparentutil.NewEmergencyReparenter(s.ts, s.tmc, logger).ReparentShard(ctx, req.Keyspace, req.Shard, - reparentutil.NewEmergencyReparentOptions(req.NewPrimary, sets.NewString(ignoreReplicaAliases...), waitReplicasTimeout, req.PreventCrossCellPromotion)) + reparentutil.EmergencyReparentOptions{ + NewPrimaryAlias: req.NewPrimary, + IgnoreReplicas: sets.NewString(ignoreReplicaAliases...), + WaitReplicasTimeout: waitReplicasTimeout, + PreventCrossCellPromotion: req.PreventCrossCellPromotion, + }, + ) resp := &vtctldatapb.EmergencyReparentShardResponse{ Keyspace: req.Keyspace, diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index cea435a02cc..aa623b1de7b 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -49,28 +49,18 @@ type EmergencyReparenter struct { logger logutil.Logger } -type ( - // EmergencyReparentOptions provides optional parameters to - // EmergencyReparentShard operations. Options are passed by value, so it is safe - // for callers to mutate and reuse options structs for multiple calls. - EmergencyReparentOptions struct { - newPrimaryAlias *topodatapb.TabletAlias - ignoreReplicas sets.String - waitReplicasTimeout time.Duration - preventCrossCellPromotion bool - - lockAction string - } -) - -// NewEmergencyReparentOptions creates a new EmergencyReparentOptions which is used in ERS -func NewEmergencyReparentOptions(newPrimaryAlias *topodatapb.TabletAlias, ignoreReplicas sets.String, waitReplicasTimeout time.Duration, preventCrossCellPromotion bool) EmergencyReparentOptions { - return EmergencyReparentOptions{ - newPrimaryAlias: newPrimaryAlias, - ignoreReplicas: ignoreReplicas, - waitReplicasTimeout: waitReplicasTimeout, - preventCrossCellPromotion: preventCrossCellPromotion, - } +// EmergencyReparentOptions provides optional parameters to +// EmergencyReparentShard operations. Options are passed by value, so it is safe +// for callers to mutate and reuse options structs for multiple calls. +type EmergencyReparentOptions struct { + NewPrimaryAlias *topodatapb.TabletAlias + IgnoreReplicas sets.String + WaitReplicasTimeout time.Duration + PreventCrossCellPromotion bool + + // Private options managed internally. We use value passing to avoid leaking + // these details back out. + lockAction string } // counters for Emergency Reparent Shard @@ -105,7 +95,7 @@ func NewEmergencyReparenter(ts *topo.Server, tmc tmclient.TabletManagerClient, l // keyspace and shard. func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace string, shard string, opts EmergencyReparentOptions) (*events.Reparent, error) { // First step is to lock the shard for the given operation - opts.lockAction = erp.getLockAction(opts.newPrimaryAlias) + opts.lockAction = erp.getLockAction(opts.NewPrimaryAlias) ctx, unlock, err := erp.ts.LockShard(ctx, keyspace, shard, opts.lockAction) if err != nil { return nil, err @@ -179,7 +169,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // Stop replication on all the tablets and build their status map var statusMap map[string]*replicationdatapb.StopReplicationStatus var primaryStatusMap map[string]*replicationdatapb.PrimaryStatus - statusMap, primaryStatusMap, err = StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, tabletMap, opts.waitReplicasTimeout, opts.ignoreReplicas, erp.logger) + statusMap, primaryStatusMap, err = StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, tabletMap, opts.WaitReplicasTimeout, opts.IgnoreReplicas, erp.logger) if err != nil { return vterrors.Wrapf(err, "failed to stop replication and build status maps: %v", err) } @@ -205,7 +195,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // Wait for all candidates to apply relay logs - if err = erp.waitForAllRelayLogsToApply(ctx, validCandidates, tabletMap, statusMap, opts.waitReplicasTimeout); err != nil { + if err = erp.waitForAllRelayLogsToApply(ctx, validCandidates, tabletMap, statusMap, opts.WaitReplicasTimeout); err != nil { return err } @@ -253,7 +243,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // if our better candidate is different from our previous candidate, then we wait for it to catch up to the intermediate primary if !topoproto.TabletAliasEqual(betterCandidate.Alias, intermediateSource.Alias) { - err = waitForCatchUp(ctx, erp.tmc, erp.logger, intermediateSource, betterCandidate, opts.waitReplicasTimeout) + err = waitForCatchUp(ctx, erp.tmc, erp.logger, intermediateSource, betterCandidate, opts.WaitReplicasTimeout) if err != nil { return err } @@ -384,8 +374,8 @@ func (erp *EmergencyReparenter) findMostAdvanced(prevPrimary *topodatapb.Tablet, // If we were requested to elect a particular primary, verify it's a valid // candidate (non-zero position, no errant GTIDs) // Also, if the candidate is - if opts.newPrimaryAlias != nil { - requestedPrimaryAlias := topoproto.TabletAliasString(opts.newPrimaryAlias) + if opts.NewPrimaryAlias != nil { + requestedPrimaryAlias := topoproto.TabletAliasString(opts.NewPrimaryAlias) pos, ok := validCandidates[requestedPrimaryAlias] if !ok { return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "requested primary elect %v has errant GTIDs", requestedPrimaryAlias) @@ -425,7 +415,7 @@ func (erp *EmergencyReparenter) reparentReplicas(ctx context.Context, ev *events var replicasStartedReplication []*topodatapb.Tablet var replicaMutex sync.Mutex - replCtx, replCancel := context.WithTimeout(ctx, opts.waitReplicasTimeout) + replCtx, replCancel := context.WithTimeout(ctx, opts.WaitReplicasTimeout) event.DispatchUpdate(ev, "reparenting all tablets") @@ -500,7 +490,7 @@ func (erp *EmergencyReparenter) reparentReplicas(ctx context.Context, ev *events switch { case alias == topoproto.TabletAliasString(newPrimaryTablet.Alias): continue - case !opts.ignoreReplicas.Has(alias): + case !opts.IgnoreReplicas.Has(alias): replWg.Add(1) numReplicas++ go handleReplica(alias, ti) @@ -579,15 +569,15 @@ func (erp *EmergencyReparenter) identifyPrimaryCandidate(newPrimary, prevPrimary } }() - if opts.newPrimaryAlias != nil { + if opts.NewPrimaryAlias != nil { // explicit request to promote a specific tablet - requestedPrimaryAlias := topoproto.TabletAliasString(opts.newPrimaryAlias) + requestedPrimaryAlias := topoproto.TabletAliasString(opts.NewPrimaryAlias) requestedPrimaryInfo, isFound := tabletMap[requestedPrimaryAlias] if !isFound { return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", requestedPrimaryAlias) } for _, validCandidate := range validCandidates { - if topoproto.TabletAliasEqual(validCandidate.Alias, opts.newPrimaryAlias) { + if topoproto.TabletAliasEqual(validCandidate.Alias, opts.NewPrimaryAlias) { return requestedPrimaryInfo.Tablet, nil } } @@ -625,7 +615,7 @@ func (erp *EmergencyReparenter) identifyPrimaryCandidate(newPrimary, prevPrimary } // we do not have a preferred candidate in the same cell - if !opts.preventCrossCellPromotion { + if !opts.PreventCrossCellPromotion { // check whether the one we promoted belongs to the preferred candidates list candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, true, false) if candidate != nil { @@ -648,7 +638,7 @@ func (erp *EmergencyReparenter) identifyPrimaryCandidate(newPrimary, prevPrimary return candidate, nil } - if !opts.preventCrossCellPromotion { + if !opts.PreventCrossCellPromotion { candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, true, false) if candidate != nil { return candidate, nil @@ -665,7 +655,7 @@ func (erp *EmergencyReparenter) identifyPrimaryCandidate(newPrimary, prevPrimary // checkIfConstraintsSatisfied is used to check whether the constraints for ERS are satisfied or not. func (erp *EmergencyReparenter) checkIfConstraintsSatisfied(newPrimary, prevPrimary *topodatapb.Tablet, opts EmergencyReparentOptions) error { - if opts.preventCrossCellPromotion && prevPrimary != nil && newPrimary.Alias.Cell != prevPrimary.Alias.Cell { + if opts.PreventCrossCellPromotion && prevPrimary != nil && newPrimary.Alias.Cell != prevPrimary.Alias.Cell { return vterrors.Errorf(vtrpc.Code_ABORTED, "elected primary does not satisfy geographic constraint - %s", topoproto.TabletAliasString(newPrimary.Alias)) } if PromotionRule(newPrimary) == MustNotPromoteRule { diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 7105f8029d5..344f8601beb 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -128,7 +128,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }{ { name: "success", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000102": nil, @@ -241,10 +241,10 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { // Here, all our tablets are tied, so we're going to explicitly pick // zone1-101. name: "success with requested primary-elect", - emergencyReparentOps: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ + emergencyReparentOps: EmergencyReparentOptions{NewPrimaryAlias: &topodatapb.TabletAlias{ Cell: "zone1", Uid: 101, - }, nil, 0, false), + }}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000101": nil, @@ -354,7 +354,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "success with existing primary", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ DemoteMasterResults: map[string]struct { Status *replicationdatapb.PrimaryStatus @@ -473,7 +473,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "shard not found", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{}, unlockTopo: true, // we shouldn't try to lock the nonexistent shard shards: nil, @@ -485,7 +485,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "cannot stop replication", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -544,7 +544,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "lost topo lock", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -603,7 +603,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "cannot get reparent candidates", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -674,7 +674,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "zero valid reparent candidates", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{}, shards: []*vtctldatapb.Shard{ { @@ -691,7 +691,9 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { { name: "error waiting for relay logs to apply", // one replica is going to take a minute to apply relay logs - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, time.Millisecond*50, false), + emergencyReparentOps: EmergencyReparentOptions{ + WaitReplicasTimeout: time.Millisecond * 50, + }, tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -780,10 +782,10 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "requested primary-elect is not in tablet map", - emergencyReparentOps: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ + emergencyReparentOps: EmergencyReparentOptions{NewPrimaryAlias: &topodatapb.TabletAlias{ Cell: "zone1", Uid: 200, - }, nil, 0, false), + }}, tmc: &testutil.TabletManagerClient{ StopReplicationAndGetStatusResults: map[string]struct { Status *replicationdatapb.Status @@ -867,10 +869,10 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "requested primary-elect is not winning primary-elect", - emergencyReparentOps: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ // we're requesting a tablet that's behind in replication + emergencyReparentOps: EmergencyReparentOptions{NewPrimaryAlias: &topodatapb.TabletAlias{ // we're requesting a tablet that's behind in replication Cell: "zone1", Uid: 102, - }, nil, 0, false), + }}, keyspace: "testkeyspace", shard: "-", ts: memorytopo.NewServer("zone1"), @@ -987,10 +989,10 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "cannot promote new primary", - emergencyReparentOps: NewEmergencyReparentOptions(&topodatapb.TabletAlias{ + emergencyReparentOps: EmergencyReparentOptions{NewPrimaryAlias: &topodatapb.TabletAlias{ Cell: "zone1", Uid: 102, - }, nil, 0, false), + }}, tmc: &testutil.TabletManagerClient{ PromoteReplicaResults: map[string]struct { Result string @@ -1102,7 +1104,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "constraint failure - promotion-rule", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000102": nil, @@ -1217,7 +1219,7 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, { name: "constraint failure - cross-cell", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, true), + emergencyReparentOps: EmergencyReparentOptions{PreventCrossCellPromotion: true}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000102": nil, @@ -1411,7 +1413,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }{ { name: "success", - emergencyReparentOps: NewEmergencyReparentOptions(nil, sets.NewString("zone1-0000000404"), 0, false), + emergencyReparentOps: EmergencyReparentOptions{IgnoreReplicas: sets.NewString("zone1-0000000404")}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1497,7 +1499,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, { name: "MasterPosition error", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string @@ -1544,7 +1546,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, { name: "cannot repopulate reparent journal on new primary", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": assert.AnError, @@ -1594,7 +1596,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, { name: "all replicas failing to SetMaster does fail the promotion", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1658,7 +1660,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, { name: "all replicas slow to SetMaster does fail the promotion", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, time.Millisecond*10, false), + emergencyReparentOps: EmergencyReparentOptions{WaitReplicasTimeout: time.Millisecond * 10}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -1726,7 +1728,7 @@ func TestEmergencyReparenter_promoteNewPrimary(t *testing.T) { }, { name: "one replica failing to SetMaster does not fail the promotion", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -2084,7 +2086,7 @@ func TestEmergencyReparenterCounters(t *testing.T) { ersFailureCounter.Set(0) _ = SetDurabilityPolicy("none", nil) - emergencyReparentOps := NewEmergencyReparentOptions(nil, nil, 0, false) + emergencyReparentOps := EmergencyReparentOptions{} tmc := &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000102": nil, @@ -2215,7 +2217,7 @@ func TestEmergencyReparenterCounters(t *testing.T) { require.EqualValues(t, 0, ersFailureCounter.Get()) // set emergencyReparentOps to request a non existent tablet - emergencyReparentOps.newPrimaryAlias = &topodatapb.TabletAlias{ + emergencyReparentOps.NewPrimaryAlias = &topodatapb.TabletAlias{ Cell: "bogus", Uid: 100, } @@ -2430,7 +2432,7 @@ func TestEmergencyReparenter_findMostAdvanced(t *testing.T) { }, }, { name: "choose most advanced with explicit request", - emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ + emergencyReparentOps: EmergencyReparentOptions{NewPrimaryAlias: &topodatapb.TabletAlias{ Cell: "zone1", Uid: 102, }}, @@ -2489,7 +2491,7 @@ func TestEmergencyReparenter_findMostAdvanced(t *testing.T) { }, }, { name: "split brain detection", - emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ + emergencyReparentOps: EmergencyReparentOptions{NewPrimaryAlias: &topodatapb.TabletAlias{ Cell: "zone1", Uid: 102, }}, @@ -2582,7 +2584,7 @@ func TestEmergencyReparenter_checkIfConstraintsSatisfied(t *testing.T) { Cell: "cell1", }, }, - opts: EmergencyReparentOptions{preventCrossCellPromotion: true}, + opts: EmergencyReparentOptions{PreventCrossCellPromotion: true}, err: "", }, { name: "promotion rule constraint failure", @@ -2598,7 +2600,7 @@ func TestEmergencyReparenter_checkIfConstraintsSatisfied(t *testing.T) { Cell: "cell1", }, }, - opts: EmergencyReparentOptions{preventCrossCellPromotion: true}, + opts: EmergencyReparentOptions{PreventCrossCellPromotion: true}, err: "elected primary does not satisfy promotion rule constraint - cell1-0000000100", }, { name: "cross cell constraint failure", @@ -2614,7 +2616,7 @@ func TestEmergencyReparenter_checkIfConstraintsSatisfied(t *testing.T) { Cell: "cell2", }, }, - opts: EmergencyReparentOptions{preventCrossCellPromotion: true}, + opts: EmergencyReparentOptions{PreventCrossCellPromotion: true}, err: "elected primary does not satisfy geographic constraint - cell1-0000000100", }, { name: "cross cell but no constraint failure", @@ -2630,7 +2632,7 @@ func TestEmergencyReparenter_checkIfConstraintsSatisfied(t *testing.T) { Cell: "cell2", }, }, - opts: EmergencyReparentOptions{preventCrossCellPromotion: false}, + opts: EmergencyReparentOptions{PreventCrossCellPromotion: false}, err: "", }, } @@ -2670,7 +2672,7 @@ func TestEmergencyReparenter_reparentReplicas(t *testing.T) { }{ { name: "success", - emergencyReparentOps: NewEmergencyReparentOptions(nil, sets.NewString("zone1-0000000404"), 0, false), + emergencyReparentOps: EmergencyReparentOptions{IgnoreReplicas: sets.NewString("zone1-0000000404")}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -2748,7 +2750,7 @@ func TestEmergencyReparenter_reparentReplicas(t *testing.T) { }, { name: "MasterPosition error", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ MasterPositionResults: map[string]struct { Position string @@ -2787,7 +2789,7 @@ func TestEmergencyReparenter_reparentReplicas(t *testing.T) { }, { name: "cannot repopulate reparent journal on new primary", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": assert.AnError, @@ -2829,7 +2831,7 @@ func TestEmergencyReparenter_reparentReplicas(t *testing.T) { }, { name: "all replicas failing to SetMaster does fail the promotion", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -2885,7 +2887,7 @@ func TestEmergencyReparenter_reparentReplicas(t *testing.T) { }, { name: "all replicas slow to SetMaster does fail the promotion", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, time.Millisecond*10, false), + emergencyReparentOps: EmergencyReparentOptions{WaitReplicasTimeout: time.Millisecond * 10}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -2944,7 +2946,7 @@ func TestEmergencyReparenter_reparentReplicas(t *testing.T) { }, { name: "one replica failing to SetMaster does not fail the promotion", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -3062,7 +3064,7 @@ func TestEmergencyReparenter_promoteIntermediatePrimary(t *testing.T) { }{ { name: "success", - emergencyReparentOps: NewEmergencyReparentOptions(nil, sets.NewString("zone1-0000000404"), 0, false), + emergencyReparentOps: EmergencyReparentOptions{IgnoreReplicas: sets.NewString("zone1-0000000404")}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -3161,7 +3163,7 @@ func TestEmergencyReparenter_promoteIntermediatePrimary(t *testing.T) { }, { name: "all replicas failed", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -3217,7 +3219,7 @@ func TestEmergencyReparenter_promoteIntermediatePrimary(t *testing.T) { }, { name: "one replica failed", - emergencyReparentOps: NewEmergencyReparentOptions(nil, nil, 0, false), + emergencyReparentOps: EmergencyReparentOptions{}, tmc: &testutil.TabletManagerClient{ PopulateReparentJournalResults: map[string]error{ "zone1-0000000100": nil, @@ -3341,7 +3343,7 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { }{ { name: "explicit request for a primary tablet", - emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ + emergencyReparentOps: EmergencyReparentOptions{NewPrimaryAlias: &topodatapb.TabletAlias{ Cell: "zone1", Uid: 100, }}, @@ -3373,7 +3375,7 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { }, }, { name: "explicit request for a primary tablet not in valid list", - emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ + emergencyReparentOps: EmergencyReparentOptions{NewPrimaryAlias: &topodatapb.TabletAlias{ Cell: "zone1", Uid: 100, }}, @@ -3393,7 +3395,7 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { err: "requested candidate zone1-0000000100 is not in valid candidates list", }, { name: "explicit request for a primary tablet not in tablet map", - emergencyReparentOps: EmergencyReparentOptions{newPrimaryAlias: &topodatapb.TabletAlias{ + emergencyReparentOps: EmergencyReparentOptions{NewPrimaryAlias: &topodatapb.TabletAlias{ Cell: "zone1", Uid: 100, }}, @@ -3627,7 +3629,7 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { }, }, { name: "prevent cross cell promotion", - emergencyReparentOps: EmergencyReparentOptions{preventCrossCellPromotion: true}, + emergencyReparentOps: EmergencyReparentOptions{PreventCrossCellPromotion: true}, newPrimary: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", diff --git a/go/vt/wrangler/reparent.go b/go/vt/wrangler/reparent.go index 3493aec6ead..48ffe0d6f20 100644 --- a/go/vt/wrangler/reparent.go +++ b/go/vt/wrangler/reparent.go @@ -150,7 +150,13 @@ func (wr *Wrangler) EmergencyReparentShard(ctx context.Context, keyspace, shard ctx, keyspace, shard, - reparentutil.NewEmergencyReparentOptions(primaryElectTabletAlias, ignoredTablets, waitReplicasTimeout, preventCrossCellPromotion)) + reparentutil.EmergencyReparentOptions{ + NewPrimaryAlias: primaryElectTabletAlias, + WaitReplicasTimeout: waitReplicasTimeout, + IgnoreReplicas: ignoredTablets, + PreventCrossCellPromotion: preventCrossCellPromotion, + }, + ) return err } From 54bbcb9335e81f60a7b3ae28364f3ea417a47e49 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 13:33:37 +0530 Subject: [PATCH 162/176] collect all variables into a single place Signed-off-by: Manan Gupta --- .../reparentutil/emergency_reparenter.go | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index aa623b1de7b..b6a6b4e0d2f 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -138,8 +138,22 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve erp.logger.Infof("will initiate emergency reparent shard in keyspace - %s, shard - %s", keyspace, shard) ersCounter.Add(1) + // variables used by the ERS functions are declared here + var ( + shardInfo *topo.ShardInfo + prevPrimary *topodatapb.Tablet + tabletMap map[string]*topo.TabletInfo + statusMap map[string]*replicationdatapb.StopReplicationStatus + primaryStatusMap map[string]*replicationdatapb.PrimaryStatus + validCandidates map[string]mysql.Position + intermediateSource *topodatapb.Tablet + validCandidateTablets []*topodatapb.Tablet + validReplacementCandidates []*topodatapb.Tablet + betterCandidate *topodatapb.Tablet + isIdeal bool + ) + // get the shard information from the topology server - var shardInfo *topo.ShardInfo shardInfo, err = erp.ts.GetShard(ctx, keyspace, shard) if err != nil { return err @@ -149,7 +163,6 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // get the previous primary according to the topology server, // we use this information to choose the best candidate in the same cell // and to undo promotion in case of failure - var prevPrimary *topodatapb.Tablet if shardInfo.PrimaryAlias != nil { prevPrimaryInfo, err := erp.ts.GetTablet(ctx, shardInfo.PrimaryAlias) if err != nil { @@ -160,15 +173,12 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // read all the tablets and there information event.DispatchUpdate(ev, "reading all tablets") - var tabletMap map[string]*topo.TabletInfo tabletMap, err = erp.ts.GetTabletMapForShard(ctx, keyspace, shard) if err != nil { return vterrors.Wrapf(err, "failed to get tablet map for %v/%v: %v", keyspace, shard, err) } // Stop replication on all the tablets and build their status map - var statusMap map[string]*replicationdatapb.StopReplicationStatus - var primaryStatusMap map[string]*replicationdatapb.PrimaryStatus statusMap, primaryStatusMap, err = StopReplicationAndBuildStatusMaps(ctx, erp.tmc, ev, tabletMap, opts.WaitReplicasTimeout, opts.IgnoreReplicas, erp.logger) if err != nil { return vterrors.Wrapf(err, "failed to stop replication and build status maps: %v", err) @@ -181,7 +191,6 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // find the valid candidates for becoming the primary // this is where we check for errant GTIDs and remove the tablets that have them from consideration - var validCandidates map[string]mysql.Position validCandidates, err = FindValidEmergencyReparentCandidates(statusMap, primaryStatusMap) if err != nil { return err @@ -201,8 +210,6 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // find the intermediate replication source that we want to replicate from. This will always be the most advanced tablet that we have // We let all the other tablets replicate from this tablet. We will then try to choose a better candidate and let it catch up - var intermediateSource *topodatapb.Tablet - var validCandidateTablets []*topodatapb.Tablet intermediateSource, validCandidateTablets, err = erp.findMostAdvanced(prevPrimary, validCandidates, tabletMap, opts) if err != nil { return err @@ -210,7 +217,6 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve erp.logger.Infof("intermediate primary selected - %v", intermediateSource.Alias) // check weather the primary candidate selected is ideal or if it can be improved later - var isIdeal bool isIdeal, err = erp.intermediateCandidateIsIdeal(intermediateSource, prevPrimary, validCandidateTablets, tabletMap, opts) if err != nil { return err @@ -228,14 +234,12 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve // we now promote our intermediate primary candidate and also reparent all the other tablets to start replicating from this candidate // we do not promote the tablet or change the shard record. We only change the replication for all the other tablets // it also returns the list of the tablets that started replication successfully including itself. These are the candidates that we can use to find a replacement - var validReplacementCandidates []*topodatapb.Tablet validReplacementCandidates, err = erp.promoteIntermediatePrimary(ctx, ev, intermediateSource, tabletMap, statusMap, opts) if err != nil { return err } // try to find a better candidate using the list we got back - var betterCandidate *topodatapb.Tablet betterCandidate, err = erp.identifyPrimaryCandidate(intermediateSource, prevPrimary, validReplacementCandidates, tabletMap, opts) if err != nil { return err @@ -412,8 +416,10 @@ func (erp *EmergencyReparenter) promoteIntermediatePrimary(ctx context.Context, // Also, it returns the replicas which started replicating only in the case where we wait for all the replicas func (erp *EmergencyReparenter) reparentReplicas(ctx context.Context, ev *events.Reparent, newPrimaryTablet *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions, waitForAllReplicas bool, populateReparentJournal bool) ([]*topodatapb.Tablet, error) { - var replicasStartedReplication []*topodatapb.Tablet - var replicaMutex sync.Mutex + var ( + replicasStartedReplication []*topodatapb.Tablet + replicaMutex sync.Mutex + ) replCtx, replCancel := context.WithTimeout(ctx, opts.WaitReplicasTimeout) @@ -583,8 +589,10 @@ func (erp *EmergencyReparenter) identifyPrimaryCandidate(newPrimary, prevPrimary } return nil, vterrors.Errorf(vtrpc.Code_ABORTED, "requested candidate %v is not in valid candidates list", requestedPrimaryAlias) } - var preferredCandidates []*topodatapb.Tablet - var neutralReplicas []*topodatapb.Tablet + var ( + preferredCandidates []*topodatapb.Tablet + neutralReplicas []*topodatapb.Tablet + ) for _, candidate := range validCandidates { promotionRule := PromotionRule(candidate) if promotionRule == MustPromoteRule || promotionRule == PreferPromoteRule { From 18ab55b2be850bae8eb919986b47cf3ea31fee10 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 14:16:22 +0530 Subject: [PATCH 163/176] moved promotionRules to its own package Signed-off-by: Manan Gupta --- go/vt/orchestrator/app/cli.go | 4 +- go/vt/orchestrator/http/api.go | 4 +- .../inst/candidate_database_instance.go | 6 +-- .../inst/candidate_database_instance_dao.go | 4 +- go/vt/orchestrator/inst/instance.go | 4 +- go/vt/orchestrator/inst/instance_dao.go | 9 ++-- go/vt/orchestrator/inst/instance_topology.go | 4 +- .../inst/instance_topology_test.go | 30 ++++++------ go/vt/orchestrator/logic/topology_recovery.go | 3 +- go/vt/vtctl/reparentutil/durability.go | 39 ++++++++------- go/vt/vtctl/reparentutil/durability_test.go | 47 ++++++++++--------- .../reparentutil/emergency_reparenter.go | 7 +-- .../{ => promotionrule}/promotion_rule.go | 2 +- 13 files changed, 83 insertions(+), 80 deletions(-) rename go/vt/vtctl/reparentutil/{ => promotionrule}/promotion_rule.go (98%) diff --git a/go/vt/orchestrator/app/cli.go b/go/vt/orchestrator/app/cli.go index 9f53992461b..8b107b484db 100644 --- a/go/vt/orchestrator/app/cli.go +++ b/go/vt/orchestrator/app/cli.go @@ -33,7 +33,7 @@ import ( "vitess.io/vitess/go/vt/orchestrator/kv" "vitess.io/vitess/go/vt/orchestrator/logic" "vitess.io/vitess/go/vt/orchestrator/process" - "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" ) var thisInstanceKey *inst.InstanceKey @@ -1274,7 +1274,7 @@ func Cli(command string, strict bool, instance string, destination string, owner case registerCliCommand("register-candidate", "Instance, meta", `Indicate that a specific instance is a preferred candidate for primary promotion`): { instanceKey, _ = inst.FigureInstanceKey(instanceKey, thisInstanceKey) - promotionRule, err := reparentutil.ParseCandidatePromotionRule(*config.RuntimeCLIFlags.PromotionRule) + promotionRule, err := promotionrule.ParseCandidatePromotionRule(*config.RuntimeCLIFlags.PromotionRule) if err != nil { log.Fatale(err) } diff --git a/go/vt/orchestrator/http/api.go b/go/vt/orchestrator/http/api.go index 33318a82edf..d16e19f6b9b 100644 --- a/go/vt/orchestrator/http/api.go +++ b/go/vt/orchestrator/http/api.go @@ -40,7 +40,7 @@ import ( "vitess.io/vitess/go/vt/orchestrator/metrics/query" "vitess.io/vitess/go/vt/orchestrator/process" orcraft "vitess.io/vitess/go/vt/orchestrator/raft" - "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" ) // APIResponseCode is an OK/ERROR response code @@ -2387,7 +2387,7 @@ func (this *HttpAPI) RegisterCandidate(params martini.Params, r render.Render, r Respond(r, &APIResponse{Code: ERROR, Message: err.Error()}) return } - promotionRule, err := reparentutil.ParseCandidatePromotionRule(params["promotionRule"]) + promotionRule, err := promotionrule.ParseCandidatePromotionRule(params["promotionRule"]) if err != nil { Respond(r, &APIResponse{Code: ERROR, Message: err.Error()}) return diff --git a/go/vt/orchestrator/inst/candidate_database_instance.go b/go/vt/orchestrator/inst/candidate_database_instance.go index f202c6d6c1e..5851273b53e 100644 --- a/go/vt/orchestrator/inst/candidate_database_instance.go +++ b/go/vt/orchestrator/inst/candidate_database_instance.go @@ -20,19 +20,19 @@ import ( "fmt" "vitess.io/vitess/go/vt/orchestrator/db" - "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" ) // CandidateDatabaseInstance contains information about explicit promotion rules for an instance type CandidateDatabaseInstance struct { Hostname string Port int - PromotionRule reparentutil.CandidatePromotionRule + PromotionRule promotionrule.CandidatePromotionRule LastSuggestedString string PromotionRuleExpiry string // generated when retrieved from database for consistency reasons } -func NewCandidateDatabaseInstance(instanceKey *InstanceKey, promotionRule reparentutil.CandidatePromotionRule) *CandidateDatabaseInstance { +func NewCandidateDatabaseInstance(instanceKey *InstanceKey, promotionRule promotionrule.CandidatePromotionRule) *CandidateDatabaseInstance { return &CandidateDatabaseInstance{ Hostname: instanceKey.Hostname, Port: instanceKey.Port, diff --git a/go/vt/orchestrator/inst/candidate_database_instance_dao.go b/go/vt/orchestrator/inst/candidate_database_instance_dao.go index 7582e76df1a..2479cb56a2b 100644 --- a/go/vt/orchestrator/inst/candidate_database_instance_dao.go +++ b/go/vt/orchestrator/inst/candidate_database_instance_dao.go @@ -22,7 +22,7 @@ import ( "vitess.io/vitess/go/vt/orchestrator/config" "vitess.io/vitess/go/vt/orchestrator/db" - "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" ) // RegisterCandidateInstance markes a given instance as suggested for succeeding a primary in the event of failover. @@ -94,7 +94,7 @@ func BulkReadCandidateDatabaseInstance() ([]CandidateDatabaseInstance, error) { cdi := CandidateDatabaseInstance{ Hostname: m.GetString("hostname"), Port: m.GetInt("port"), - PromotionRule: reparentutil.CandidatePromotionRule(m.GetString("promotion_rule")), + PromotionRule: promotionrule.CandidatePromotionRule(m.GetString("promotion_rule")), LastSuggestedString: m.GetString("last_suggested"), PromotionRuleExpiry: m.GetString("promotion_rule_expiry"), } diff --git a/go/vt/orchestrator/inst/instance.go b/go/vt/orchestrator/inst/instance.go index 6f7184a0fbd..8a5095c97a3 100644 --- a/go/vt/orchestrator/inst/instance.go +++ b/go/vt/orchestrator/inst/instance.go @@ -26,7 +26,7 @@ import ( "vitess.io/vitess/go/vt/orchestrator/config" "vitess.io/vitess/go/vt/orchestrator/external/golib/math" - "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" ) const ReasonableDiscoveryLatency = 500 * time.Millisecond @@ -111,7 +111,7 @@ type Instance struct { // be picked up from daabase_candidate_instance's value when // reading an instance from the db. IsCandidate bool - PromotionRule reparentutil.CandidatePromotionRule + PromotionRule promotionrule.CandidatePromotionRule IsDowntimed bool DowntimeReason string DowntimeOwner string diff --git a/go/vt/orchestrator/inst/instance_dao.go b/go/vt/orchestrator/inst/instance_dao.go index a582689757d..bc3997322eb 100644 --- a/go/vt/orchestrator/inst/instance_dao.go +++ b/go/vt/orchestrator/inst/instance_dao.go @@ -50,6 +50,7 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" ) const ( @@ -933,7 +934,7 @@ func BulkReadInstance() ([](*InstanceKey), error) { } func ReadInstancePromotionRule(instance *Instance) (err error) { - var promotionRule reparentutil.CandidatePromotionRule = reparentutil.NeutralPromoteRule + var promotionRule promotionrule.CandidatePromotionRule = promotionrule.NeutralPromoteRule query := ` select ifnull(nullif(promotion_rule, ''), 'neutral') as promotion_rule @@ -943,7 +944,7 @@ func ReadInstancePromotionRule(instance *Instance) (err error) { args := sqlutils.Args(instance.Key.Hostname, instance.Key.Port) err = db.QueryOrchestrator(query, args, func(m sqlutils.RowMap) error { - promotionRule = reparentutil.CandidatePromotionRule(m.GetString("promotion_rule")) + promotionRule = promotionrule.CandidatePromotionRule(m.GetString("promotion_rule")) return nil }) instance.PromotionRule = promotionRule @@ -1023,7 +1024,7 @@ func readInstanceRow(m sqlutils.RowMap) *Instance { instance.IsLastCheckValid = m.GetBool("is_last_check_valid") instance.SecondsSinceLastSeen = m.GetNullInt64("seconds_since_last_seen") instance.IsCandidate = m.GetBool("is_candidate") - instance.PromotionRule = reparentutil.CandidatePromotionRule(m.GetString("promotion_rule")) + instance.PromotionRule = promotionrule.CandidatePromotionRule(m.GetString("promotion_rule")) instance.IsDowntimed = m.GetBool("is_downtimed") instance.DowntimeReason = m.GetString("downtime_reason") instance.DowntimeOwner = m.GetString("downtime_owner") @@ -1427,7 +1428,7 @@ func ReadClusterNeutralPromotionRuleInstances(clusterName string) (neutralInstan return neutralInstances, err } for _, instance := range instances { - if instance.PromotionRule == reparentutil.NeutralPromoteRule { + if instance.PromotionRule == promotionrule.NeutralPromoteRule { neutralInstances = append(neutralInstances, instance) } } diff --git a/go/vt/orchestrator/inst/instance_topology.go b/go/vt/orchestrator/inst/instance_topology.go index 87743a9f66b..fe08e6f62dc 100644 --- a/go/vt/orchestrator/inst/instance_topology.go +++ b/go/vt/orchestrator/inst/instance_topology.go @@ -30,7 +30,7 @@ import ( "vitess.io/vitess/go/vt/orchestrator/external/golib/math" "vitess.io/vitess/go/vt/orchestrator/external/golib/util" "vitess.io/vitess/go/vt/orchestrator/os" - "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" ) type StopReplicationMethod string @@ -1610,7 +1610,7 @@ func isValidAsCandidatePrimaryInBinlogServerTopology(replica *Instance) bool { } func IsBannedFromBeingCandidateReplica(replica *Instance) bool { - if replica.PromotionRule == reparentutil.MustNotPromoteRule { + if replica.PromotionRule == promotionrule.MustNotPromoteRule { log.Debugf("instance %+v is banned because of promotion rule", replica.Key) return true } diff --git a/go/vt/orchestrator/inst/instance_topology_test.go b/go/vt/orchestrator/inst/instance_topology_test.go index 5d9c138bf4b..d3380f5367d 100644 --- a/go/vt/orchestrator/inst/instance_topology_test.go +++ b/go/vt/orchestrator/inst/instance_topology_test.go @@ -8,7 +8,7 @@ import ( "vitess.io/vitess/go/vt/orchestrator/config" "vitess.io/vitess/go/vt/orchestrator/external/golib/log" test "vitess.io/vitess/go/vt/orchestrator/external/golib/tests" - "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" ) var ( @@ -252,7 +252,7 @@ func TestIsBannedFromBeingCandidateReplica(t *testing.T) { { instances, _ := generateTestInstances() for _, instance := range instances { - instance.PromotionRule = reparentutil.MustNotPromoteRule + instance.PromotionRule = promotionrule.MustNotPromoteRule } for _, instance := range instances { test.S(t).ExpectTrue(IsBannedFromBeingCandidateReplica(instance)) @@ -472,7 +472,7 @@ func TestChooseCandidateReplicaPriorityBinlogFormatRowOverrides(t *testing.T) { func TestChooseCandidateReplicaMustNotPromoteRule(t *testing.T) { instances, instancesMap := generateTestInstances() applyGeneralGoodToGoReplicationParams(instances) - instancesMap[i830Key.StringCode()].PromotionRule = reparentutil.MustNotPromoteRule + instancesMap[i830Key.StringCode()].PromotionRule = promotionrule.MustNotPromoteRule instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) @@ -486,8 +486,8 @@ func TestChooseCandidateReplicaMustNotPromoteRule(t *testing.T) { func TestChooseCandidateReplicaPreferNotPromoteRule(t *testing.T) { instances, instancesMap := generateTestInstances() applyGeneralGoodToGoReplicationParams(instances) - instancesMap[i830Key.StringCode()].PromotionRule = reparentutil.MustNotPromoteRule - instancesMap[i820Key.StringCode()].PromotionRule = reparentutil.PreferNotPromoteRule + instancesMap[i830Key.StringCode()].PromotionRule = promotionrule.MustNotPromoteRule + instancesMap[i820Key.StringCode()].PromotionRule = promotionrule.PreferNotPromoteRule instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) @@ -502,9 +502,9 @@ func TestChooseCandidateReplicaPreferNotPromoteRule2(t *testing.T) { instances, instancesMap := generateTestInstances() applyGeneralGoodToGoReplicationParams(instances) for _, instance := range instances { - instance.PromotionRule = reparentutil.PreferNotPromoteRule + instance.PromotionRule = promotionrule.PreferNotPromoteRule } - instancesMap[i830Key.StringCode()].PromotionRule = reparentutil.MustNotPromoteRule + instancesMap[i830Key.StringCode()].PromotionRule = promotionrule.MustNotPromoteRule instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) @@ -520,9 +520,9 @@ func TestChooseCandidateReplicaPromoteRuleOrdering(t *testing.T) { applyGeneralGoodToGoReplicationParams(instances) for _, instance := range instances { instance.ExecBinlogCoordinates = instancesMap[i710Key.StringCode()].ExecBinlogCoordinates - instance.PromotionRule = reparentutil.NeutralPromoteRule + instance.PromotionRule = promotionrule.NeutralPromoteRule } - instancesMap[i830Key.StringCode()].PromotionRule = reparentutil.PreferPromoteRule + instancesMap[i830Key.StringCode()].PromotionRule = promotionrule.PreferPromoteRule instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) @@ -538,9 +538,9 @@ func TestChooseCandidateReplicaPromoteRuleOrdering2(t *testing.T) { applyGeneralGoodToGoReplicationParams(instances) for _, instance := range instances { instance.ExecBinlogCoordinates = instancesMap[i710Key.StringCode()].ExecBinlogCoordinates - instance.PromotionRule = reparentutil.PreferPromoteRule + instance.PromotionRule = promotionrule.PreferPromoteRule } - instancesMap[i820Key.StringCode()].PromotionRule = reparentutil.MustPromoteRule + instancesMap[i820Key.StringCode()].PromotionRule = promotionrule.MustPromoteRule instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) @@ -556,11 +556,11 @@ func TestChooseCandidateReplicaPromoteRuleOrdering3(t *testing.T) { applyGeneralGoodToGoReplicationParams(instances) for _, instance := range instances { instance.ExecBinlogCoordinates = instancesMap[i710Key.StringCode()].ExecBinlogCoordinates - instance.PromotionRule = reparentutil.NeutralPromoteRule + instance.PromotionRule = promotionrule.NeutralPromoteRule } - instancesMap[i730Key.StringCode()].PromotionRule = reparentutil.MustPromoteRule - instancesMap[i810Key.StringCode()].PromotionRule = reparentutil.PreferPromoteRule - instancesMap[i830Key.StringCode()].PromotionRule = reparentutil.PreferNotPromoteRule + instancesMap[i730Key.StringCode()].PromotionRule = promotionrule.MustPromoteRule + instancesMap[i810Key.StringCode()].PromotionRule = promotionrule.PreferPromoteRule + instancesMap[i830Key.StringCode()].PromotionRule = promotionrule.PreferNotPromoteRule instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 51b45962cb8..10a6e52e51b 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -46,6 +46,7 @@ import ( "vitess.io/vitess/go/vt/orchestrator/util" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" "vitess.io/vitess/go/vt/vttablet/tmclient" ) @@ -495,7 +496,7 @@ func SuggestReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, de keepSearchingHint := "" if satisfied, reason := PrimaryFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, promotedReplica); !satisfied { keepSearchingHint = fmt.Sprintf("Will keep searching; %s", reason) - } else if promotedReplica.PromotionRule == reparentutil.PreferNotPromoteRule { + } else if promotedReplica.PromotionRule == promotionrule.PreferNotPromoteRule { keepSearchingHint = fmt.Sprintf("Will keep searching because we have promoted a server with prefer_not rule: %+v", promotedReplica.Key) } if keepSearchingHint != "" { diff --git a/go/vt/vtctl/reparentutil/durability.go b/go/vt/vtctl/reparentutil/durability.go index 073c55684d3..904ca818a09 100644 --- a/go/vt/vtctl/reparentutil/durability.go +++ b/go/vt/vtctl/reparentutil/durability.go @@ -20,10 +20,9 @@ import ( "fmt" "vitess.io/vitess/go/vt/log" - - "vitess.io/vitess/go/vt/topo/topoproto" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" ) //======================================================================= @@ -56,7 +55,7 @@ func init() { // durabler is the interface which is used to get the promotion rules for candidates and the semi sync setup type durabler interface { - promotionRule(*topodatapb.Tablet) CandidatePromotionRule + promotionRule(*topodatapb.Tablet) promotionrule.CandidatePromotionRule primarySemiSync(*topodatapb.Tablet) int replicaSemiSync(primary, replica *topodatapb.Tablet) bool } @@ -82,7 +81,7 @@ func SetDurabilityPolicy(name string, durabilityParams map[string]string) error } // PromotionRule returns the promotion rule for the instance. -func PromotionRule(tablet *topodatapb.Tablet) CandidatePromotionRule { +func PromotionRule(tablet *topodatapb.Tablet) promotionrule.CandidatePromotionRule { return curDurabilityPolicy.promotionRule(tablet) } @@ -103,12 +102,12 @@ func ReplicaSemiSyncFromTablet(primary, replica *topodatapb.Tablet) bool { // durabilityNone has no semi-sync and returns NeutralPromoteRule for Primary and Replica tablet types, MustNotPromoteRule for everything else type durabilityNone struct{} -func (d *durabilityNone) promotionRule(tablet *topodatapb.Tablet) CandidatePromotionRule { +func (d *durabilityNone) promotionRule(tablet *topodatapb.Tablet) promotionrule.CandidatePromotionRule { switch tablet.Type { case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: - return NeutralPromoteRule + return promotionrule.NeutralPromoteRule } - return MustNotPromoteRule + return promotionrule.MustNotPromoteRule } func (d *durabilityNone) primarySemiSync(tablet *topodatapb.Tablet) int { @@ -125,12 +124,12 @@ func (d *durabilityNone) replicaSemiSync(primary, replica *topodatapb.Tablet) bo // It returns NeutralPromoteRule for Primary and Replica tablet types, MustNotPromoteRule for everything else type durabilitySemiSync struct{} -func (d *durabilitySemiSync) promotionRule(tablet *topodatapb.Tablet) CandidatePromotionRule { +func (d *durabilitySemiSync) promotionRule(tablet *topodatapb.Tablet) promotionrule.CandidatePromotionRule { switch tablet.Type { case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: - return NeutralPromoteRule + return promotionrule.NeutralPromoteRule } - return MustNotPromoteRule + return promotionrule.MustNotPromoteRule } func (d *durabilitySemiSync) primarySemiSync(tablet *topodatapb.Tablet) int { @@ -152,12 +151,12 @@ func (d *durabilitySemiSync) replicaSemiSync(primary, replica *topodatapb.Tablet // It returns NeutralPromoteRule for Primary and Replica tablet types, MustNotPromoteRule for everything else type durabilityCrossCell struct{} -func (d *durabilityCrossCell) promotionRule(tablet *topodatapb.Tablet) CandidatePromotionRule { +func (d *durabilityCrossCell) promotionRule(tablet *topodatapb.Tablet) promotionrule.CandidatePromotionRule { switch tablet.Type { case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: - return NeutralPromoteRule + return promotionrule.NeutralPromoteRule } - return MustNotPromoteRule + return promotionrule.MustNotPromoteRule } func (d *durabilityCrossCell) primarySemiSync(tablet *topodatapb.Tablet) int { @@ -181,10 +180,10 @@ func (d *durabilityCrossCell) replicaSemiSync(primary, replica *topodatapb.Table // durabilitySpecified is like durabilityNone. It has an additional map which it first queries with the tablet alias as the key // If a CandidatePromotionRule is found in that map, then that is used as the promotion rule. Otherwise, it reverts to the same logic as durabilityNone type durabilitySpecified struct { - promotionRules map[string]CandidatePromotionRule + promotionRules map[string]promotionrule.CandidatePromotionRule } -func (d *durabilitySpecified) promotionRule(tablet *topodatapb.Tablet) CandidatePromotionRule { +func (d *durabilitySpecified) promotionRule(tablet *topodatapb.Tablet) promotionrule.CandidatePromotionRule { promoteRule, isFound := d.promotionRules[topoproto.TabletAliasString(tablet.Alias)] if isFound { return promoteRule @@ -192,9 +191,9 @@ func (d *durabilitySpecified) promotionRule(tablet *topodatapb.Tablet) Candidate switch tablet.Type { case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: - return NeutralPromoteRule + return promotionrule.NeutralPromoteRule } - return MustNotPromoteRule + return promotionrule.MustNotPromoteRule } func (d *durabilitySpecified) primarySemiSync(tablet *topodatapb.Tablet) int { @@ -207,11 +206,11 @@ func (d *durabilitySpecified) replicaSemiSync(primary, replica *topodatapb.Table // newDurabilitySpecified is a function that is used to create a new durabilitySpecified struct func newDurabilitySpecified(m map[string]string) durabler { - promotionRules := map[string]CandidatePromotionRule{} + promotionRules := map[string]promotionrule.CandidatePromotionRule{} // range over the map given by the user for tabletAliasStr, promotionRuleStr := range m { // parse the promotion rule - promotionRule, err := ParseCandidatePromotionRule(promotionRuleStr) + promotionRule, err := promotionrule.ParseCandidatePromotionRule(promotionRuleStr) // if parsing is not successful, skip over this rule if err != nil { log.Errorf("invalid promotion rule %s found, received error - %v", promotionRuleStr, err) diff --git a/go/vt/vtctl/reparentutil/durability_test.go b/go/vt/vtctl/reparentutil/durability_test.go index dc65cccc293..6a96d5d4d5c 100644 --- a/go/vt/vtctl/reparentutil/durability_test.go +++ b/go/vt/vtctl/reparentutil/durability_test.go @@ -24,6 +24,7 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" ) func TestDurabilityNone(t *testing.T) { @@ -33,22 +34,22 @@ func TestDurabilityNone(t *testing.T) { promoteRule := PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, }) - assert.Equal(t, NeutralPromoteRule, promoteRule) + assert.Equal(t, promotionrule.NeutralPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_REPLICA, }) - assert.Equal(t, NeutralPromoteRule, promoteRule) + assert.Equal(t, promotionrule.NeutralPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_RDONLY, }) - assert.Equal(t, MustNotPromoteRule, promoteRule) + assert.Equal(t, promotionrule.MustNotPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_SPARE, }) - assert.Equal(t, MustNotPromoteRule, promoteRule) + assert.Equal(t, promotionrule.MustNotPromoteRule, promoteRule) assert.Equal(t, 0, PrimarySemiSyncFromTablet(nil)) assert.Equal(t, false, ReplicaSemiSyncFromTablet(nil, nil)) } @@ -60,22 +61,22 @@ func TestDurabilitySemiSync(t *testing.T) { promoteRule := PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, }) - assert.Equal(t, NeutralPromoteRule, promoteRule) + assert.Equal(t, promotionrule.NeutralPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_REPLICA, }) - assert.Equal(t, NeutralPromoteRule, promoteRule) + assert.Equal(t, promotionrule.NeutralPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_RDONLY, }) - assert.Equal(t, MustNotPromoteRule, promoteRule) + assert.Equal(t, promotionrule.MustNotPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_SPARE, }) - assert.Equal(t, MustNotPromoteRule, promoteRule) + assert.Equal(t, promotionrule.MustNotPromoteRule, promoteRule) assert.Equal(t, 1, PrimarySemiSyncFromTablet(nil)) assert.Equal(t, true, ReplicaSemiSyncFromTablet(nil, &topodatapb.Tablet{ Type: topodatapb.TabletType_REPLICA, @@ -92,22 +93,22 @@ func TestDurabilityCrossCell(t *testing.T) { promoteRule := PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, }) - assert.Equal(t, NeutralPromoteRule, promoteRule) + assert.Equal(t, promotionrule.NeutralPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_REPLICA, }) - assert.Equal(t, NeutralPromoteRule, promoteRule) + assert.Equal(t, promotionrule.NeutralPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_RDONLY, }) - assert.Equal(t, MustNotPromoteRule, promoteRule) + assert.Equal(t, promotionrule.MustNotPromoteRule, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_SPARE, }) - assert.Equal(t, MustNotPromoteRule, promoteRule) + assert.Equal(t, promotionrule.MustNotPromoteRule, promoteRule) assert.Equal(t, 1, PrimarySemiSyncFromTablet(nil)) assert.Equal(t, false, ReplicaSemiSyncFromTablet(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, @@ -153,16 +154,16 @@ func TestDurabilitySpecified(t *testing.T) { cellName := "cell" durabilityRules := newDurabilitySpecified( map[string]string{ - "cell-0000000000": string(MustPromoteRule), - "cell-0000000001": string(PreferPromoteRule), - "cell-0000000002": string(NeutralPromoteRule), - "cell-0000000003": string(PreferNotPromoteRule), - "cell-0000000004": string(MustNotPromoteRule), + "cell-0000000000": string(promotionrule.MustPromoteRule), + "cell-0000000001": string(promotionrule.PreferPromoteRule), + "cell-0000000002": string(promotionrule.NeutralPromoteRule), + "cell-0000000003": string(promotionrule.PreferNotPromoteRule), + "cell-0000000004": string(promotionrule.MustNotPromoteRule), }) testcases := []struct { tablet *topodatapb.Tablet - promotionRule CandidatePromotionRule + promotionRule promotionrule.CandidatePromotionRule }{ { tablet: &topodatapb.Tablet{ @@ -171,7 +172,7 @@ func TestDurabilitySpecified(t *testing.T) { Uid: 0, }, }, - promotionRule: MustNotPromoteRule, + promotionRule: promotionrule.MustNotPromoteRule, }, { tablet: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ @@ -179,7 +180,7 @@ func TestDurabilitySpecified(t *testing.T) { Uid: 1, }, }, - promotionRule: PreferPromoteRule, + promotionRule: promotionrule.PreferPromoteRule, }, { tablet: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ @@ -187,7 +188,7 @@ func TestDurabilitySpecified(t *testing.T) { Uid: 2, }, }, - promotionRule: NeutralPromoteRule, + promotionRule: promotionrule.NeutralPromoteRule, }, { tablet: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ @@ -195,7 +196,7 @@ func TestDurabilitySpecified(t *testing.T) { Uid: 3, }, }, - promotionRule: PreferNotPromoteRule, + promotionRule: promotionrule.PreferNotPromoteRule, }, { tablet: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ @@ -203,7 +204,7 @@ func TestDurabilitySpecified(t *testing.T) { Uid: 4, }, }, - promotionRule: MustNotPromoteRule, + promotionRule: promotionrule.MustNotPromoteRule, }, } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index b6a6b4e0d2f..ccd9ea53b87 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -38,6 +38,7 @@ import ( "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/topotools/events" + "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vttablet/tmclient" ) @@ -595,10 +596,10 @@ func (erp *EmergencyReparenter) identifyPrimaryCandidate(newPrimary, prevPrimary ) for _, candidate := range validCandidates { promotionRule := PromotionRule(candidate) - if promotionRule == MustPromoteRule || promotionRule == PreferPromoteRule { + if promotionRule == promotionrule.MustPromoteRule || promotionRule == promotionrule.PreferPromoteRule { preferredCandidates = append(preferredCandidates, candidate) } - if promotionRule == NeutralPromoteRule { + if promotionRule == promotionrule.NeutralPromoteRule { neutralReplicas = append(neutralReplicas, candidate) } } @@ -666,7 +667,7 @@ func (erp *EmergencyReparenter) checkIfConstraintsSatisfied(newPrimary, prevPrim if opts.PreventCrossCellPromotion && prevPrimary != nil && newPrimary.Alias.Cell != prevPrimary.Alias.Cell { return vterrors.Errorf(vtrpc.Code_ABORTED, "elected primary does not satisfy geographic constraint - %s", topoproto.TabletAliasString(newPrimary.Alias)) } - if PromotionRule(newPrimary) == MustNotPromoteRule { + if PromotionRule(newPrimary) == promotionrule.MustNotPromoteRule { return vterrors.Errorf(vtrpc.Code_ABORTED, "elected primary does not satisfy promotion rule constraint - %s", topoproto.TabletAliasString(newPrimary.Alias)) } return nil diff --git a/go/vt/vtctl/reparentutil/promotion_rule.go b/go/vt/vtctl/reparentutil/promotionrule/promotion_rule.go similarity index 98% rename from go/vt/vtctl/reparentutil/promotion_rule.go rename to go/vt/vtctl/reparentutil/promotionrule/promotion_rule.go index 077eff7f518..0039c2d70bd 100644 --- a/go/vt/vtctl/reparentutil/promotion_rule.go +++ b/go/vt/vtctl/reparentutil/promotionrule/promotion_rule.go @@ -14,7 +14,7 @@ limitations under the License. */ -package reparentutil +package promotionrule import ( "fmt" From 0570edf3f8927507149cbbbd46331925d76624c1 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 14:18:40 +0530 Subject: [PATCH 164/176] rename constants and functions in promotionrule package Signed-off-by: Manan Gupta --- go/vt/orchestrator/app/cli.go | 2 +- go/vt/orchestrator/http/api.go | 2 +- go/vt/orchestrator/inst/instance_dao.go | 4 +- go/vt/orchestrator/inst/instance_topology.go | 2 +- .../inst/instance_topology_test.go | 28 ++++++------ go/vt/orchestrator/logic/topology_recovery.go | 2 +- go/vt/vtctl/reparentutil/durability.go | 18 ++++---- go/vt/vtctl/reparentutil/durability_test.go | 44 +++++++++---------- .../reparentutil/emergency_reparenter.go | 6 +-- .../promotionrule/promotion_rule.go | 24 +++++----- 10 files changed, 66 insertions(+), 66 deletions(-) diff --git a/go/vt/orchestrator/app/cli.go b/go/vt/orchestrator/app/cli.go index 8b107b484db..0dc179af848 100644 --- a/go/vt/orchestrator/app/cli.go +++ b/go/vt/orchestrator/app/cli.go @@ -1274,7 +1274,7 @@ func Cli(command string, strict bool, instance string, destination string, owner case registerCliCommand("register-candidate", "Instance, meta", `Indicate that a specific instance is a preferred candidate for primary promotion`): { instanceKey, _ = inst.FigureInstanceKey(instanceKey, thisInstanceKey) - promotionRule, err := promotionrule.ParseCandidatePromotionRule(*config.RuntimeCLIFlags.PromotionRule) + promotionRule, err := promotionrule.Parse(*config.RuntimeCLIFlags.PromotionRule) if err != nil { log.Fatale(err) } diff --git a/go/vt/orchestrator/http/api.go b/go/vt/orchestrator/http/api.go index d16e19f6b9b..b8c884c18fb 100644 --- a/go/vt/orchestrator/http/api.go +++ b/go/vt/orchestrator/http/api.go @@ -2387,7 +2387,7 @@ func (this *HttpAPI) RegisterCandidate(params martini.Params, r render.Render, r Respond(r, &APIResponse{Code: ERROR, Message: err.Error()}) return } - promotionRule, err := promotionrule.ParseCandidatePromotionRule(params["promotionRule"]) + promotionRule, err := promotionrule.Parse(params["promotionRule"]) if err != nil { Respond(r, &APIResponse{Code: ERROR, Message: err.Error()}) return diff --git a/go/vt/orchestrator/inst/instance_dao.go b/go/vt/orchestrator/inst/instance_dao.go index bc3997322eb..a909c438a16 100644 --- a/go/vt/orchestrator/inst/instance_dao.go +++ b/go/vt/orchestrator/inst/instance_dao.go @@ -934,7 +934,7 @@ func BulkReadInstance() ([](*InstanceKey), error) { } func ReadInstancePromotionRule(instance *Instance) (err error) { - var promotionRule promotionrule.CandidatePromotionRule = promotionrule.NeutralPromoteRule + var promotionRule promotionrule.CandidatePromotionRule = promotionrule.Neutral query := ` select ifnull(nullif(promotion_rule, ''), 'neutral') as promotion_rule @@ -1428,7 +1428,7 @@ func ReadClusterNeutralPromotionRuleInstances(clusterName string) (neutralInstan return neutralInstances, err } for _, instance := range instances { - if instance.PromotionRule == promotionrule.NeutralPromoteRule { + if instance.PromotionRule == promotionrule.Neutral { neutralInstances = append(neutralInstances, instance) } } diff --git a/go/vt/orchestrator/inst/instance_topology.go b/go/vt/orchestrator/inst/instance_topology.go index fe08e6f62dc..c5fbad0f444 100644 --- a/go/vt/orchestrator/inst/instance_topology.go +++ b/go/vt/orchestrator/inst/instance_topology.go @@ -1610,7 +1610,7 @@ func isValidAsCandidatePrimaryInBinlogServerTopology(replica *Instance) bool { } func IsBannedFromBeingCandidateReplica(replica *Instance) bool { - if replica.PromotionRule == promotionrule.MustNotPromoteRule { + if replica.PromotionRule == promotionrule.MustNot { log.Debugf("instance %+v is banned because of promotion rule", replica.Key) return true } diff --git a/go/vt/orchestrator/inst/instance_topology_test.go b/go/vt/orchestrator/inst/instance_topology_test.go index d3380f5367d..477cae9c098 100644 --- a/go/vt/orchestrator/inst/instance_topology_test.go +++ b/go/vt/orchestrator/inst/instance_topology_test.go @@ -252,7 +252,7 @@ func TestIsBannedFromBeingCandidateReplica(t *testing.T) { { instances, _ := generateTestInstances() for _, instance := range instances { - instance.PromotionRule = promotionrule.MustNotPromoteRule + instance.PromotionRule = promotionrule.MustNot } for _, instance := range instances { test.S(t).ExpectTrue(IsBannedFromBeingCandidateReplica(instance)) @@ -472,7 +472,7 @@ func TestChooseCandidateReplicaPriorityBinlogFormatRowOverrides(t *testing.T) { func TestChooseCandidateReplicaMustNotPromoteRule(t *testing.T) { instances, instancesMap := generateTestInstances() applyGeneralGoodToGoReplicationParams(instances) - instancesMap[i830Key.StringCode()].PromotionRule = promotionrule.MustNotPromoteRule + instancesMap[i830Key.StringCode()].PromotionRule = promotionrule.MustNot instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) @@ -486,8 +486,8 @@ func TestChooseCandidateReplicaMustNotPromoteRule(t *testing.T) { func TestChooseCandidateReplicaPreferNotPromoteRule(t *testing.T) { instances, instancesMap := generateTestInstances() applyGeneralGoodToGoReplicationParams(instances) - instancesMap[i830Key.StringCode()].PromotionRule = promotionrule.MustNotPromoteRule - instancesMap[i820Key.StringCode()].PromotionRule = promotionrule.PreferNotPromoteRule + instancesMap[i830Key.StringCode()].PromotionRule = promotionrule.MustNot + instancesMap[i820Key.StringCode()].PromotionRule = promotionrule.PreferNot instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) @@ -502,9 +502,9 @@ func TestChooseCandidateReplicaPreferNotPromoteRule2(t *testing.T) { instances, instancesMap := generateTestInstances() applyGeneralGoodToGoReplicationParams(instances) for _, instance := range instances { - instance.PromotionRule = promotionrule.PreferNotPromoteRule + instance.PromotionRule = promotionrule.PreferNot } - instancesMap[i830Key.StringCode()].PromotionRule = promotionrule.MustNotPromoteRule + instancesMap[i830Key.StringCode()].PromotionRule = promotionrule.MustNot instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) @@ -520,9 +520,9 @@ func TestChooseCandidateReplicaPromoteRuleOrdering(t *testing.T) { applyGeneralGoodToGoReplicationParams(instances) for _, instance := range instances { instance.ExecBinlogCoordinates = instancesMap[i710Key.StringCode()].ExecBinlogCoordinates - instance.PromotionRule = promotionrule.NeutralPromoteRule + instance.PromotionRule = promotionrule.Neutral } - instancesMap[i830Key.StringCode()].PromotionRule = promotionrule.PreferPromoteRule + instancesMap[i830Key.StringCode()].PromotionRule = promotionrule.Prefer instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) @@ -538,9 +538,9 @@ func TestChooseCandidateReplicaPromoteRuleOrdering2(t *testing.T) { applyGeneralGoodToGoReplicationParams(instances) for _, instance := range instances { instance.ExecBinlogCoordinates = instancesMap[i710Key.StringCode()].ExecBinlogCoordinates - instance.PromotionRule = promotionrule.PreferPromoteRule + instance.PromotionRule = promotionrule.Prefer } - instancesMap[i820Key.StringCode()].PromotionRule = promotionrule.MustPromoteRule + instancesMap[i820Key.StringCode()].PromotionRule = promotionrule.Must instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) @@ -556,11 +556,11 @@ func TestChooseCandidateReplicaPromoteRuleOrdering3(t *testing.T) { applyGeneralGoodToGoReplicationParams(instances) for _, instance := range instances { instance.ExecBinlogCoordinates = instancesMap[i710Key.StringCode()].ExecBinlogCoordinates - instance.PromotionRule = promotionrule.NeutralPromoteRule + instance.PromotionRule = promotionrule.Neutral } - instancesMap[i730Key.StringCode()].PromotionRule = promotionrule.MustPromoteRule - instancesMap[i810Key.StringCode()].PromotionRule = promotionrule.PreferPromoteRule - instancesMap[i830Key.StringCode()].PromotionRule = promotionrule.PreferNotPromoteRule + instancesMap[i730Key.StringCode()].PromotionRule = promotionrule.Must + instancesMap[i810Key.StringCode()].PromotionRule = promotionrule.Prefer + instancesMap[i830Key.StringCode()].PromotionRule = promotionrule.PreferNot instances = sortedReplicas(instances, NoStopReplication) candidate, aheadReplicas, equalReplicas, laterReplicas, cannotReplicateReplicas, err := ChooseCandidateReplica(instances) test.S(t).ExpectNil(err) diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 10a6e52e51b..21e2110cc33 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -496,7 +496,7 @@ func SuggestReplacementForPromotedReplica(topologyRecovery *TopologyRecovery, de keepSearchingHint := "" if satisfied, reason := PrimaryFailoverGeographicConstraintSatisfied(&topologyRecovery.AnalysisEntry, promotedReplica); !satisfied { keepSearchingHint = fmt.Sprintf("Will keep searching; %s", reason) - } else if promotedReplica.PromotionRule == promotionrule.PreferNotPromoteRule { + } else if promotedReplica.PromotionRule == promotionrule.PreferNot { keepSearchingHint = fmt.Sprintf("Will keep searching because we have promoted a server with prefer_not rule: %+v", promotedReplica.Key) } if keepSearchingHint != "" { diff --git a/go/vt/vtctl/reparentutil/durability.go b/go/vt/vtctl/reparentutil/durability.go index 904ca818a09..715ba335a93 100644 --- a/go/vt/vtctl/reparentutil/durability.go +++ b/go/vt/vtctl/reparentutil/durability.go @@ -105,9 +105,9 @@ type durabilityNone struct{} func (d *durabilityNone) promotionRule(tablet *topodatapb.Tablet) promotionrule.CandidatePromotionRule { switch tablet.Type { case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: - return promotionrule.NeutralPromoteRule + return promotionrule.Neutral } - return promotionrule.MustNotPromoteRule + return promotionrule.MustNot } func (d *durabilityNone) primarySemiSync(tablet *topodatapb.Tablet) int { @@ -127,9 +127,9 @@ type durabilitySemiSync struct{} func (d *durabilitySemiSync) promotionRule(tablet *topodatapb.Tablet) promotionrule.CandidatePromotionRule { switch tablet.Type { case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: - return promotionrule.NeutralPromoteRule + return promotionrule.Neutral } - return promotionrule.MustNotPromoteRule + return promotionrule.MustNot } func (d *durabilitySemiSync) primarySemiSync(tablet *topodatapb.Tablet) int { @@ -154,9 +154,9 @@ type durabilityCrossCell struct{} func (d *durabilityCrossCell) promotionRule(tablet *topodatapb.Tablet) promotionrule.CandidatePromotionRule { switch tablet.Type { case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: - return promotionrule.NeutralPromoteRule + return promotionrule.Neutral } - return promotionrule.MustNotPromoteRule + return promotionrule.MustNot } func (d *durabilityCrossCell) primarySemiSync(tablet *topodatapb.Tablet) int { @@ -191,9 +191,9 @@ func (d *durabilitySpecified) promotionRule(tablet *topodatapb.Tablet) promotion switch tablet.Type { case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: - return promotionrule.NeutralPromoteRule + return promotionrule.Neutral } - return promotionrule.MustNotPromoteRule + return promotionrule.MustNot } func (d *durabilitySpecified) primarySemiSync(tablet *topodatapb.Tablet) int { @@ -210,7 +210,7 @@ func newDurabilitySpecified(m map[string]string) durabler { // range over the map given by the user for tabletAliasStr, promotionRuleStr := range m { // parse the promotion rule - promotionRule, err := promotionrule.ParseCandidatePromotionRule(promotionRuleStr) + promotionRule, err := promotionrule.Parse(promotionRuleStr) // if parsing is not successful, skip over this rule if err != nil { log.Errorf("invalid promotion rule %s found, received error - %v", promotionRuleStr, err) diff --git a/go/vt/vtctl/reparentutil/durability_test.go b/go/vt/vtctl/reparentutil/durability_test.go index 6a96d5d4d5c..d962b7f1f80 100644 --- a/go/vt/vtctl/reparentutil/durability_test.go +++ b/go/vt/vtctl/reparentutil/durability_test.go @@ -34,22 +34,22 @@ func TestDurabilityNone(t *testing.T) { promoteRule := PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, }) - assert.Equal(t, promotionrule.NeutralPromoteRule, promoteRule) + assert.Equal(t, promotionrule.Neutral, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_REPLICA, }) - assert.Equal(t, promotionrule.NeutralPromoteRule, promoteRule) + assert.Equal(t, promotionrule.Neutral, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_RDONLY, }) - assert.Equal(t, promotionrule.MustNotPromoteRule, promoteRule) + assert.Equal(t, promotionrule.MustNot, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_SPARE, }) - assert.Equal(t, promotionrule.MustNotPromoteRule, promoteRule) + assert.Equal(t, promotionrule.MustNot, promoteRule) assert.Equal(t, 0, PrimarySemiSyncFromTablet(nil)) assert.Equal(t, false, ReplicaSemiSyncFromTablet(nil, nil)) } @@ -61,22 +61,22 @@ func TestDurabilitySemiSync(t *testing.T) { promoteRule := PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, }) - assert.Equal(t, promotionrule.NeutralPromoteRule, promoteRule) + assert.Equal(t, promotionrule.Neutral, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_REPLICA, }) - assert.Equal(t, promotionrule.NeutralPromoteRule, promoteRule) + assert.Equal(t, promotionrule.Neutral, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_RDONLY, }) - assert.Equal(t, promotionrule.MustNotPromoteRule, promoteRule) + assert.Equal(t, promotionrule.MustNot, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_SPARE, }) - assert.Equal(t, promotionrule.MustNotPromoteRule, promoteRule) + assert.Equal(t, promotionrule.MustNot, promoteRule) assert.Equal(t, 1, PrimarySemiSyncFromTablet(nil)) assert.Equal(t, true, ReplicaSemiSyncFromTablet(nil, &topodatapb.Tablet{ Type: topodatapb.TabletType_REPLICA, @@ -93,22 +93,22 @@ func TestDurabilityCrossCell(t *testing.T) { promoteRule := PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, }) - assert.Equal(t, promotionrule.NeutralPromoteRule, promoteRule) + assert.Equal(t, promotionrule.Neutral, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_REPLICA, }) - assert.Equal(t, promotionrule.NeutralPromoteRule, promoteRule) + assert.Equal(t, promotionrule.Neutral, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_RDONLY, }) - assert.Equal(t, promotionrule.MustNotPromoteRule, promoteRule) + assert.Equal(t, promotionrule.MustNot, promoteRule) promoteRule = PromotionRule(&topodatapb.Tablet{ Type: topodatapb.TabletType_SPARE, }) - assert.Equal(t, promotionrule.MustNotPromoteRule, promoteRule) + assert.Equal(t, promotionrule.MustNot, promoteRule) assert.Equal(t, 1, PrimarySemiSyncFromTablet(nil)) assert.Equal(t, false, ReplicaSemiSyncFromTablet(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, @@ -154,11 +154,11 @@ func TestDurabilitySpecified(t *testing.T) { cellName := "cell" durabilityRules := newDurabilitySpecified( map[string]string{ - "cell-0000000000": string(promotionrule.MustPromoteRule), - "cell-0000000001": string(promotionrule.PreferPromoteRule), - "cell-0000000002": string(promotionrule.NeutralPromoteRule), - "cell-0000000003": string(promotionrule.PreferNotPromoteRule), - "cell-0000000004": string(promotionrule.MustNotPromoteRule), + "cell-0000000000": string(promotionrule.Must), + "cell-0000000001": string(promotionrule.Prefer), + "cell-0000000002": string(promotionrule.Neutral), + "cell-0000000003": string(promotionrule.PreferNot), + "cell-0000000004": string(promotionrule.MustNot), }) testcases := []struct { @@ -172,7 +172,7 @@ func TestDurabilitySpecified(t *testing.T) { Uid: 0, }, }, - promotionRule: promotionrule.MustNotPromoteRule, + promotionRule: promotionrule.MustNot, }, { tablet: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ @@ -180,7 +180,7 @@ func TestDurabilitySpecified(t *testing.T) { Uid: 1, }, }, - promotionRule: promotionrule.PreferPromoteRule, + promotionRule: promotionrule.Prefer, }, { tablet: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ @@ -188,7 +188,7 @@ func TestDurabilitySpecified(t *testing.T) { Uid: 2, }, }, - promotionRule: promotionrule.NeutralPromoteRule, + promotionRule: promotionrule.Neutral, }, { tablet: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ @@ -196,7 +196,7 @@ func TestDurabilitySpecified(t *testing.T) { Uid: 3, }, }, - promotionRule: promotionrule.PreferNotPromoteRule, + promotionRule: promotionrule.PreferNot, }, { tablet: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ @@ -204,7 +204,7 @@ func TestDurabilitySpecified(t *testing.T) { Uid: 4, }, }, - promotionRule: promotionrule.MustNotPromoteRule, + promotionRule: promotionrule.MustNot, }, } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index ccd9ea53b87..dae1f5d6930 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -596,10 +596,10 @@ func (erp *EmergencyReparenter) identifyPrimaryCandidate(newPrimary, prevPrimary ) for _, candidate := range validCandidates { promotionRule := PromotionRule(candidate) - if promotionRule == promotionrule.MustPromoteRule || promotionRule == promotionrule.PreferPromoteRule { + if promotionRule == promotionrule.Must || promotionRule == promotionrule.Prefer { preferredCandidates = append(preferredCandidates, candidate) } - if promotionRule == promotionrule.NeutralPromoteRule { + if promotionRule == promotionrule.Neutral { neutralReplicas = append(neutralReplicas, candidate) } } @@ -667,7 +667,7 @@ func (erp *EmergencyReparenter) checkIfConstraintsSatisfied(newPrimary, prevPrim if opts.PreventCrossCellPromotion && prevPrimary != nil && newPrimary.Alias.Cell != prevPrimary.Alias.Cell { return vterrors.Errorf(vtrpc.Code_ABORTED, "elected primary does not satisfy geographic constraint - %s", topoproto.TabletAliasString(newPrimary.Alias)) } - if PromotionRule(newPrimary) == promotionrule.MustNotPromoteRule { + if PromotionRule(newPrimary) == promotionrule.MustNot { return vterrors.Errorf(vtrpc.Code_ABORTED, "elected primary does not satisfy promotion rule constraint - %s", topoproto.TabletAliasString(newPrimary.Alias)) } return nil diff --git a/go/vt/vtctl/reparentutil/promotionrule/promotion_rule.go b/go/vt/vtctl/reparentutil/promotionrule/promotion_rule.go index 0039c2d70bd..06f9ad0dec6 100644 --- a/go/vt/vtctl/reparentutil/promotionrule/promotion_rule.go +++ b/go/vt/vtctl/reparentutil/promotionrule/promotion_rule.go @@ -25,19 +25,19 @@ import ( type CandidatePromotionRule string const ( - MustPromoteRule CandidatePromotionRule = "must" - PreferPromoteRule CandidatePromotionRule = "prefer" - NeutralPromoteRule CandidatePromotionRule = "neutral" - PreferNotPromoteRule CandidatePromotionRule = "prefer_not" - MustNotPromoteRule CandidatePromotionRule = "must_not" + Must CandidatePromotionRule = "must" + Prefer CandidatePromotionRule = "prefer" + Neutral CandidatePromotionRule = "neutral" + PreferNot CandidatePromotionRule = "prefer_not" + MustNot CandidatePromotionRule = "must_not" ) var promotionRuleOrderMap = map[CandidatePromotionRule]int{ - MustPromoteRule: 0, - PreferPromoteRule: 1, - NeutralPromoteRule: 2, - PreferNotPromoteRule: 3, - MustNotPromoteRule: 4, + Must: 0, + Prefer: 1, + Neutral: 2, + PreferNot: 3, + MustNot: 4, } func (this *CandidatePromotionRule) BetterThan(other CandidatePromotionRule) bool { @@ -48,9 +48,9 @@ func (this *CandidatePromotionRule) BetterThan(other CandidatePromotionRule) boo return promotionRuleOrderMap[*this] < otherOrder } -// ParseCandidatePromotionRule returns a CandidatePromotionRule by name. +// Parse returns a CandidatePromotionRule by name. // It returns an error if there is no known rule by the given name. -func ParseCandidatePromotionRule(ruleName string) (CandidatePromotionRule, error) { +func Parse(ruleName string) (CandidatePromotionRule, error) { switch ruleName { case "prefer", "neutral", "prefer_not", "must_not": return CandidatePromotionRule(ruleName), nil From 2276ae3c1e46a62d345d2f47874c1b67164c2ba3 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 14:21:24 +0530 Subject: [PATCH 165/176] remove dead code from main_test.go file in vtorc Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/main_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/go/test/endtoend/vtorc/main_test.go b/go/test/endtoend/vtorc/main_test.go index 77ba9425172..7e4852aef59 100644 --- a/go/test/endtoend/vtorc/main_test.go +++ b/go/test/endtoend/vtorc/main_test.go @@ -420,9 +420,6 @@ func checkPrimaryTablet(t *testing.T, cluster *cluster.LocalProcessCluster, tabl err = json2.Unmarshal([]byte(result), &streamHealthResponse) require.NoError(t, err) - //if !streamHealthResponse.GetServing() { - // log.Exitf("stream health not updated") - //} if checkServing && !streamHealthResponse.GetServing() { log.Warningf("Tablet %v is not serving in health stream yet, sleep for 1 second\n", tablet.Alias) time.Sleep(time.Second) From beae8392b4314b1696a511eed2b581080e06e7fc Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 14:22:26 +0530 Subject: [PATCH 166/176] renamed function in vtorc tests Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/main_test.go | 2 +- .../endtoend/vtorc/primary_failure_test.go | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/go/test/endtoend/vtorc/main_test.go b/go/test/endtoend/vtorc/main_test.go index 7e4852aef59..4aa202db9c6 100644 --- a/go/test/endtoend/vtorc/main_test.go +++ b/go/test/endtoend/vtorc/main_test.go @@ -468,7 +468,7 @@ func checkReplication(t *testing.T, clusterInstance *cluster.LocalProcessCluster // call this function only after check replication. // it inserts more data into the table vt_insert_test and checks that it is replicated too -func runAdditionalCommands(t *testing.T, primary *cluster.Vttablet, replicas []*cluster.Vttablet, timeToWait time.Duration) { +func verifyWritesSucceed(t *testing.T, primary *cluster.Vttablet, replicas []*cluster.Vttablet, timeToWait time.Duration) { confirmReplication(t, primary, replicas, timeToWait, lastUsedValue) lastUsedValue++ } diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index b6a94617eb9..1b4c8e28c2a 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -65,7 +65,7 @@ func TestDownPrimary(t *testing.T) { // check that the replica gets promoted checkPrimaryTablet(t, clusterInstance, replica, true) // also check that the replication is working correctly after failover - runAdditionalCommands(t, replica, []*cluster.Vttablet{rdonly}, 10*time.Second) + verifyWritesSucceed(t, replica, []*cluster.Vttablet{rdonly}, 10*time.Second) } // Failover should not be cross data centers, according to the configuration file @@ -108,7 +108,7 @@ func TestCrossDataCenterFailure(t *testing.T) { // we have a replica in the same cell, so that is the one which should be promoted and not the one from another cell checkPrimaryTablet(t, clusterInstance, replicaInSameCell, true) // also check that the replication is working correctly after failover - runAdditionalCommands(t, replicaInSameCell, []*cluster.Vttablet{crossCellReplica, rdonly}, 10*time.Second) + verifyWritesSucceed(t, replicaInSameCell, []*cluster.Vttablet{crossCellReplica, rdonly}, 10*time.Second) } // Failover should not be cross data centers, according to the configuration file @@ -200,7 +200,7 @@ func TestLostRdonlyOnPrimaryFailure(t *testing.T) { require.NoError(t, err) // check that aheadRdonly is able to replicate. We also want to add some queries to aheadRdonly which will not be there in replica and rdonly - runAdditionalCommands(t, curPrimary, []*cluster.Vttablet{aheadRdonly}, 15*time.Second) + verifyWritesSucceed(t, curPrimary, []*cluster.Vttablet{aheadRdonly}, 15*time.Second) // assert that the replica and rdonly are indeed lagging and do not have the new insertion by checking the count of rows in the tables out, err := runSQL(t, "SELECT * FROM vt_insert_test", replica, "vt_ks") @@ -226,7 +226,7 @@ func TestLostRdonlyOnPrimaryFailure(t *testing.T) { checkPrimaryTablet(t, clusterInstance, replica, true) // also check that the replication is setup correctly - runAdditionalCommands(t, replica, []*cluster.Vttablet{rdonly}, 15*time.Second) + verifyWritesSucceed(t, replica, []*cluster.Vttablet{rdonly}, 15*time.Second) // check that the rdonly is lost. The lost replica has is detached and its host is prepended with `//` out, err = runSQL(t, "SELECT HOST FROM performance_schema.replication_connection_configuration", aheadRdonly, "") @@ -273,7 +273,7 @@ func TestPromotionLagSuccess(t *testing.T) { // check that the replica gets promoted checkPrimaryTablet(t, clusterInstance, replica, true) // also check that the replication is working correctly after failover - runAdditionalCommands(t, replica, []*cluster.Vttablet{rdonly}, 10*time.Second) + verifyWritesSucceed(t, replica, []*cluster.Vttablet{rdonly}, 10*time.Second) } // This test checks that the promotion of a tablet succeeds if it passes the promotion lag test @@ -369,7 +369,7 @@ func TestDownPrimaryPromotionRule(t *testing.T) { // we have a replica in the same cell, so that is the one which should be promoted and not the one from another cell checkPrimaryTablet(t, clusterInstance, crossCellReplica, true) // also check that the replication is working correctly after failover - runAdditionalCommands(t, crossCellReplica, []*cluster.Vttablet{rdonly, replica}, 10*time.Second) + verifyWritesSucceed(t, crossCellReplica, []*cluster.Vttablet{rdonly, replica}, 10*time.Second) } // covers the test case master-failover-candidate-lag from orchestrator @@ -412,7 +412,7 @@ func TestDownPrimaryPromotionRuleWithLag(t *testing.T) { require.NoError(t, err) // check that rdonly and replica are able to replicate. We also want to add some queries to replica which will not be there in crossCellReplica - runAdditionalCommands(t, curPrimary, []*cluster.Vttablet{replica, rdonly}, 15*time.Second) + verifyWritesSucceed(t, curPrimary, []*cluster.Vttablet{replica, rdonly}, 15*time.Second) // reset the primary logs so that crossCellReplica can never catch up resetPrimaryLogs(t, curPrimary) @@ -446,7 +446,7 @@ func TestDownPrimaryPromotionRuleWithLag(t *testing.T) { require.Equal(t, 2, len(out.Rows)) // check that rdonly and replica are able to replicate from the crossCellReplica - runAdditionalCommands(t, crossCellReplica, []*cluster.Vttablet{replica, rdonly}, 15*time.Second) + verifyWritesSucceed(t, crossCellReplica, []*cluster.Vttablet{replica, rdonly}, 15*time.Second) } // covers the test case master-failover-candidate-lag-cross-datacenter from orchestrator @@ -489,7 +489,7 @@ func TestDownPrimaryPromotionRuleWithLagCrossCenter(t *testing.T) { require.NoError(t, err) // check that rdonly and crossCellReplica are able to replicate. We also want to add some queries to crossCenterReplica which will not be there in replica - runAdditionalCommands(t, curPrimary, []*cluster.Vttablet{rdonly, crossCellReplica}, 15*time.Second) + verifyWritesSucceed(t, curPrimary, []*cluster.Vttablet{rdonly, crossCellReplica}, 15*time.Second) // reset the primary logs so that crossCellReplica can never catch up resetPrimaryLogs(t, curPrimary) @@ -523,5 +523,5 @@ func TestDownPrimaryPromotionRuleWithLagCrossCenter(t *testing.T) { require.Equal(t, 2, len(out.Rows)) // check that rdonly and crossCellReplica are able to replicate from the replica - runAdditionalCommands(t, replica, []*cluster.Vttablet{crossCellReplica, rdonly}, 15*time.Second) + verifyWritesSucceed(t, replica, []*cluster.Vttablet{crossCellReplica, rdonly}, 15*time.Second) } From a8f4dcb6f3dfb1b3a16df0892ee7606d3aee8c81 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 16:34:06 +0530 Subject: [PATCH 167/176] added a end to end test verifying that the new primary can pull transactions from an advanced rdonly Signed-off-by: Manan Gupta --- .../reparent/emergencyreparent/ers_test.go | 62 +++++++++++++++++++ .../reparent/emergencyreparent/utils_test.go | 9 +++ 2 files changed, 71 insertions(+) diff --git a/go/test/endtoend/reparent/emergencyreparent/ers_test.go b/go/test/endtoend/reparent/emergencyreparent/ers_test.go index e28ca892c40..f8a353b0de7 100644 --- a/go/test/endtoend/reparent/emergencyreparent/ers_test.go +++ b/go/test/endtoend/reparent/emergencyreparent/ers_test.go @@ -139,3 +139,65 @@ func TestERSPrefersSameCell(t *testing.T) { newPrimary := getNewPrimary(t) require.Equal(t, newPrimary.Alias, tab3.Alias, "tab3 should be the promoted primary") } + +// TestPullFromRdonly tests that if a rdonly tablet is the most advanced, then our promoted primary should have +// caught up to it by pulling transactions from it +func TestPullFromRdonly(t *testing.T) { + defer cluster.PanicHandler(t) + setupReparentCluster(t) + defer teardownCluster() + var err error + + ctx := context.Background() + // make tab2 a rdonly tablet. + // rename tablet so that the test is not confusing + rdonly := tab2 + err = clusterInstance.VtctlclientProcess.ExecuteCommand("ChangeTabletType", rdonly.Alias, "rdonly") + require.NoError(t, err) + + // confirm that all the tablets can replicate successfully right now + confirmReplication(t, tab1, []*cluster.Vttablet{rdonly, tab3, tab4}) + + // stop replication on the other two tablets + err = clusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", tab3.Alias) + require.NoError(t, err) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("StopReplication", tab4.Alias) + require.NoError(t, err) + + // stop semi-sync on the primary so that any transaction now added does not require an ack + runSQL(ctx, t, "SET GLOBAL rpl_semi_sync_master_enabled = false", tab1) + + // confirm that rdonly is able to replicate from our primary + // This will also introduce a new transaction into the rdonly tablet which the other 2 replicas don't have + confirmReplication(t, tab1, []*cluster.Vttablet{rdonly}) + + // Make the current primary agent and database unavailable. + stopTablet(t, tab1, true) + + // start the replication back on the two tablets + err = clusterInstance.VtctlclientProcess.ExecuteCommand("StartReplication", tab3.Alias) + require.NoError(t, err) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("StartReplication", tab4.Alias) + require.NoError(t, err) + + // check that tab3 and tab4 still only has 1 value + err = checkCountOfInsertedValues(ctx, t, tab3, 1) + require.NoError(t, err) + err = checkCountOfInsertedValues(ctx, t, tab4, 1) + require.NoError(t, err) + + // At this point we have successfully made our rdonly tablet more advanced than tab3 and tab4 without introducing errant GTIDs + // We have simulated a network partition in which the primary and rdonly got isolated and then the primary went down leaving the rdonly most advanced + + // We expect that tab3 will be promoted since it is in the same cell as the previous primary + // Also it must be fully caught up + out, err := ers(nil, "60s", "30s") + require.NoError(t, err, out) + + newPrimary := getNewPrimary(t) + require.Equal(t, newPrimary.Alias, tab3.Alias, "tab3 should be the promoted primary") + + // check that the new primary has the last transaction that only the rdonly had + err = checkInsertedValues(ctx, t, newPrimary, insertVal) + require.NoError(t, err) +} diff --git a/go/test/endtoend/reparent/emergencyreparent/utils_test.go b/go/test/endtoend/reparent/emergencyreparent/utils_test.go index 15761ef6a03..39293c1282e 100644 --- a/go/test/endtoend/reparent/emergencyreparent/utils_test.go +++ b/go/test/endtoend/reparent/emergencyreparent/utils_test.go @@ -349,6 +349,15 @@ func checkInsertedValues(ctx context.Context, t *testing.T, tablet *cluster.Vtta return fmt.Errorf("data is not yet replicated on tablet %s", tablet.Alias) } +func checkCountOfInsertedValues(ctx context.Context, t *testing.T, tablet *cluster.Vttablet, count int) error { + selectSQL := "select * from vt_insert_test" + qr := runSQL(ctx, t, selectSQL, tablet) + if len(qr.Rows) == count { + return nil + } + return fmt.Errorf("count does not match on the tablet %s", tablet.Alias) +} + // endregion // region tablet operations From ed78b1bd915d00dbef7dd48ad88c10b0c0ecbb4a Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 18:25:25 +0530 Subject: [PATCH 168/176] refactor and add comments Signed-off-by: Manan Gupta --- .../reparent/emergencyreparent/utils_test.go | 10 +- .../reparent/plannedreparent/utils_test.go | 10 +- go/test/endtoend/vtorc/main_test.go | 1 - .../fakemysqldaemon/fakemysqldaemon.go | 4 +- go/vt/orchestrator/logic/topology_recovery.go | 10 +- .../testutil/test_tmclient.go | 26 --- .../reparentutil/emergency_reparenter.go | 161 ++++++++++++------ .../reparentutil/emergency_reparenter_test.go | 68 ++++---- go/vt/vtctl/reparentutil/util.go | 25 ++- go/vt/vtctl/reparentutil/util_test.go | 25 +-- 10 files changed, 190 insertions(+), 150 deletions(-) diff --git a/go/test/endtoend/reparent/emergencyreparent/utils_test.go b/go/test/endtoend/reparent/emergencyreparent/utils_test.go index 39293c1282e..8e870a93af1 100644 --- a/go/test/endtoend/reparent/emergencyreparent/utils_test.go +++ b/go/test/endtoend/reparent/emergencyreparent/utils_test.go @@ -27,17 +27,15 @@ import ( "time" "github.com/stretchr/testify/assert" - - "vitess.io/vitess/go/json2" - "vitess.io/vitess/go/vt/log" - querypb "vitess.io/vitess/go/vt/proto/query" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "github.com/stretchr/testify/require" + "vitess.io/vitess/go/json2" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/vt/log" + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" tmc "vitess.io/vitess/go/vt/vttablet/grpctmclient" ) diff --git a/go/test/endtoend/reparent/plannedreparent/utils_test.go b/go/test/endtoend/reparent/plannedreparent/utils_test.go index 2c8819fb2c5..b392ad6149c 100644 --- a/go/test/endtoend/reparent/plannedreparent/utils_test.go +++ b/go/test/endtoend/reparent/plannedreparent/utils_test.go @@ -29,17 +29,15 @@ import ( "time" "github.com/stretchr/testify/assert" - - "vitess.io/vitess/go/json2" - "vitess.io/vitess/go/vt/log" - querypb "vitess.io/vitess/go/vt/proto/query" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "github.com/stretchr/testify/require" + "vitess.io/vitess/go/json2" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/vt/log" + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" tmc "vitess.io/vitess/go/vt/vttablet/grpctmclient" ) diff --git a/go/test/endtoend/vtorc/main_test.go b/go/test/endtoend/vtorc/main_test.go index 4aa202db9c6..3612dec89b6 100644 --- a/go/test/endtoend/vtorc/main_test.go +++ b/go/test/endtoend/vtorc/main_test.go @@ -696,7 +696,6 @@ func resetPrimaryLogs(t *testing.T, curPrimary *cluster.Vttablet) { lastLogFile := binLogsOutput.Rows[len(binLogsOutput.Rows)-1][0].ToString() - // purge binary logs of the primary so that crossCellReplica cannot catch up _, err = runSQL(t, "PURGE BINARY LOGS TO '"+lastLogFile+"'", curPrimary, "") require.NoError(t, err) } diff --git a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go index 6f7749a7ae2..95333b97494 100644 --- a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go +++ b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go @@ -116,8 +116,8 @@ type FakeMysqlDaemon struct { // SetReplicationSourceError is used by SetReplicationSource SetReplicationSourceError error - // WaitPrimaryPositions is checked by WaitSourcePos, if the - // same it returns nil, if different it returns an error + // WaitPrimaryPositions is checked by WaitSourcePos, if the value is found + // in it, then the function returns nil, else the function returns an error WaitPrimaryPositions []mysql.Position // PromoteResult is returned by Promote diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index 21e2110cc33..e9c1e9e870c 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -28,22 +28,20 @@ import ( "sync/atomic" "time" - "vitess.io/vitess/go/vt/orchestrator/attributes" - "vitess.io/vitess/go/vt/orchestrator/kv" - - "vitess.io/vitess/go/vt/logutil" - logutilpb "vitess.io/vitess/go/vt/proto/logutil" - "github.com/patrickmn/go-cache" "github.com/rcrowley/go-metrics" + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/orchestrator/attributes" "vitess.io/vitess/go/vt/orchestrator/config" "vitess.io/vitess/go/vt/orchestrator/external/golib/log" "vitess.io/vitess/go/vt/orchestrator/inst" + "vitess.io/vitess/go/vt/orchestrator/kv" ometrics "vitess.io/vitess/go/vt/orchestrator/metrics" "vitess.io/vitess/go/vt/orchestrator/os" "vitess.io/vitess/go/vt/orchestrator/process" "vitess.io/vitess/go/vt/orchestrator/util" + logutilpb "vitess.io/vitess/go/vt/proto/logutil" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" diff --git a/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go b/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go index 45e61443425..48531972d24 100644 --- a/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go +++ b/go/vt/vtctl/grpcvtctldserver/testutil/test_tmclient.go @@ -510,32 +510,6 @@ func (fake *TabletManagerClient) SetMaster(ctx context.Context, tablet *topodata return assert.AnError } -// SetReplicationSource is part of the tmclient.TabletManagerClient interface. -func (fake *TabletManagerClient) SetReplicationSource(ctx context.Context, tablet *topodatapb.Tablet, parent *topodatapb.TabletAlias, timeCreatedNS int64, waitPosition string, forceStartReplication bool) error { - if fake.SetMasterResults == nil { - return assert.AnError - } - - key := topoproto.TabletAliasString(tablet.Alias) - - if fake.SetMasterDelays != nil { - if delay, ok := fake.SetMasterDelays[key]; ok { - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(delay): - // proceed to results - } - } - } - - if result, ok := fake.SetMasterResults[key]; ok { - return result - } - - return assert.AnError -} - // SetReadOnly is part of the tmclient.TabletManagerClient interface. func (fake *TabletManagerClient) SetReadOnly(ctx context.Context, tablet *topodatapb.Tablet) error { if fake.SetReadOnlyResults == nil { diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index dae1f5d6930..bc7846c2c9d 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -31,16 +31,17 @@ import ( "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/concurrency" "vitess.io/vitess/go/vt/logutil" - logutilpb "vitess.io/vitess/go/vt/proto/logutil" - replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/topotools/events" "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vttablet/tmclient" + + logutilpb "vitess.io/vitess/go/vt/proto/logutil" + replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/proto/vtrpc" ) // EmergencyReparenter performs EmergencyReparentShard operations. @@ -196,7 +197,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve if err != nil { return err } - // Now, we restrict the valid candidates list, which removes some tablets from consideration + // Restrict the valid candidates list. We remove any tablet which is of the type DRAINED, RESTORE or BACKUP. validCandidates, err = restrictValidCandidates(validCandidates, tabletMap) if err != nil { return err @@ -209,46 +210,51 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } - // find the intermediate replication source that we want to replicate from. This will always be the most advanced tablet that we have - // We let all the other tablets replicate from this tablet. We will then try to choose a better candidate and let it catch up + // Find the intermediate source for replication that we want other tablets to replicate from. + // This step chooses the most advanced tablet. Further ties are broken by using the cell of the previous primary and the promotion rule. + // In case the user has specified a tablet specifically, then it is selected, as long as it is the most advanced. + // Here we also check for split brain scenarios and check that the selected replica must be more advanced than all the other valid candidates. + // We fail in case there is a split brain detected. intermediateSource, validCandidateTablets, err = erp.findMostAdvanced(prevPrimary, validCandidates, tabletMap, opts) if err != nil { return err } - erp.logger.Infof("intermediate primary selected - %v", intermediateSource.Alias) + erp.logger.Infof("intermediate source selected - %v", intermediateSource.Alias) - // check weather the primary candidate selected is ideal or if it can be improved later - isIdeal, err = erp.intermediateCandidateIsIdeal(intermediateSource, prevPrimary, validCandidateTablets, tabletMap, opts) + // check weather the intermediate source candidate selected is ideal or if it can be improved later + isIdeal, err = erp.intermediateSourceIsIdeal(intermediateSource, prevPrimary, validCandidateTablets, tabletMap, opts) if err != nil { return err } - erp.logger.Infof("intermediate primary is ideal - %v", isIdeal) + erp.logger.Infof("intermediate source is ideal candidate- %v", isIdeal) // Check (again) we still have the topology lock. if err = topo.CheckShardLocked(ctx, keyspace, shard); err != nil { return vterrors.Wrapf(err, "lost topology lock, aborting: %v", err) } - // initialize the newPrimary with the intermediate primary, override this value if it is not the ideal candidate + // initialize the newPrimary with the intermediate source, override this value if it is not the ideal candidate newPrimary := intermediateSource if !isIdeal { - // we now promote our intermediate primary candidate and also reparent all the other tablets to start replicating from this candidate + // we now reparent all the tablets to start replicating from the intermediate source // we do not promote the tablet or change the shard record. We only change the replication for all the other tablets // it also returns the list of the tablets that started replication successfully including itself. These are the candidates that we can use to find a replacement - validReplacementCandidates, err = erp.promoteIntermediatePrimary(ctx, ev, intermediateSource, tabletMap, statusMap, opts) + validReplacementCandidates, err = erp.promoteIntermediateSource(ctx, ev, intermediateSource, tabletMap, statusMap, opts) if err != nil { return err } // try to find a better candidate using the list we got back + // We prefer to choose a candidate which is in the same cell as our previous primary and of the best possible durability rule. + // However, if there is an explicit request from the user to promote a specific tablet, then we choose that tablet. betterCandidate, err = erp.identifyPrimaryCandidate(intermediateSource, prevPrimary, validReplacementCandidates, tabletMap, opts) if err != nil { return err } - // if our better candidate is different from our previous candidate, then we wait for it to catch up to the intermediate primary + // if our better candidate is different from our intermediate source, then we wait for it to catch up to the intermediate source if !topoproto.TabletAliasEqual(betterCandidate.Alias, intermediateSource.Alias) { - err = waitForCatchUp(ctx, erp.tmc, erp.logger, intermediateSource, betterCandidate, opts.WaitReplicasTimeout) + err = waitForCatchUp(ctx, erp.tmc, erp.logger, betterCandidate, intermediateSource, opts.WaitReplicasTimeout) if err != nil { return err } @@ -256,7 +262,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } } - // now we check if all the constraints are satisfied. If they are not, then we should abort + // now we check if all the constraints are satisfied. If they are not, then we should exit constraintFailure := erp.checkIfConstraintsSatisfied(newPrimary, prevPrimary, opts) if constraintFailure != nil { erp.logger.Errorf("have to override promotion because of constraint failure - %v", constraintFailure) @@ -285,7 +291,13 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } -func (erp *EmergencyReparenter) waitForAllRelayLogsToApply(ctx context.Context, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, waitReplicasTimeout time.Duration) error { +func (erp *EmergencyReparenter) waitForAllRelayLogsToApply( + ctx context.Context, + validCandidates map[string]mysql.Position, + tabletMap map[string]*topo.TabletInfo, + statusMap map[string]*replicationdatapb.StopReplicationStatus, + waitReplicasTimeout time.Duration, +) error { errCh := make(chan error) defer close(errCh) @@ -341,9 +353,14 @@ func (erp *EmergencyReparenter) waitForAllRelayLogsToApply(ctx context.Context, return nil } -// findMostAdvanced finds the intermediate primary candidate for ERS. We always choose the most advanced one from our valid candidates list -func (erp *EmergencyReparenter) findMostAdvanced(prevPrimary *topodatapb.Tablet, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (*topodatapb.Tablet, []*topodatapb.Tablet, error) { - erp.logger.Infof("started finding the intermediate primary candidate") +// findMostAdvanced finds the intermediate source for ERS. We always choose the most advanced one from our valid candidates list. Further ties are broken by looking at the cell and promotion rules. +func (erp *EmergencyReparenter) findMostAdvanced( + prevPrimary *topodatapb.Tablet, + validCandidates map[string]mysql.Position, + tabletMap map[string]*topo.TabletInfo, + opts EmergencyReparentOptions, +) (*topodatapb.Tablet, []*topodatapb.Tablet, error) { + erp.logger.Infof("started finding the intermediate source") // convert the valid candidates into a list so that we can use it for sorting validTablets, tabletPositions, err := getValidCandidatesAndPositionsAsList(validCandidates, tabletMap) if err != nil { @@ -355,13 +372,13 @@ func (erp *EmergencyReparenter) findMostAdvanced(prevPrimary *topodatapb.Tablet, idealCell = prevPrimary.Alias.Cell } - // sort the tablets for finding the best intermediate primary in ERS + // sort the tablets for finding the best intermediate source in ERS err = sortTabletsForERS(validTablets, tabletPositions, idealCell) if err != nil { return nil, nil, err } for _, tablet := range validTablets { - erp.logger.Infof("finding intermediate primary - sorted replica: %v", tablet.Alias) + erp.logger.Infof("finding intermediate source - sorted replica: %v", tablet.Alias) } // The first tablet in the sorted list will be the most eligible candidate unless explicitly asked for some other tablet @@ -386,7 +403,7 @@ func (erp *EmergencyReparenter) findMostAdvanced(prevPrimary *topodatapb.Tablet, return nil, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "requested primary elect %v has errant GTIDs", requestedPrimaryAlias) } // if the requested tablet is as advanced as the most advanced tablet, then we can just use it for promotion. - // otherwise, we should let it catchup to the most advanced tablet and let it be the intermediate primary + // otherwise, we should let it catchup to the most advanced tablet and not change the intermediate source if pos.AtLeast(winningPosition) { requestedPrimaryInfo, isFound := tabletMap[requestedPrimaryAlias] if !isFound { @@ -399,23 +416,40 @@ func (erp *EmergencyReparenter) findMostAdvanced(prevPrimary *topodatapb.Tablet, return winningPrimaryTablet, validTablets, nil } -// promoteIntermediatePrimary promotes the primary candidate that we have, but it does not yet set to start accepting writes -func (erp *EmergencyReparenter) promoteIntermediatePrimary(ctx context.Context, ev *events.Reparent, newPrimary *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions) ([]*topodatapb.Tablet, error) { - // we reparent all the other tablets to start replication from our new primary +// promoteIntermediateSource reparents all the other tablets to start replicating from the intermediate source. +// It does not promote this tablet to a primary instance, we only let other replicas start replicating from this tablet +func (erp *EmergencyReparenter) promoteIntermediateSource( + ctx context.Context, + ev *events.Reparent, + source *topodatapb.Tablet, + tabletMap map[string]*topo.TabletInfo, + statusMap map[string]*replicationdatapb.StopReplicationStatus, + opts EmergencyReparentOptions, +) ([]*topodatapb.Tablet, error) { + // we reparent all the other tablets to start replication from our new source // we wait for all the replicas so that we can choose a better candidate from the ones that started replication later - validCandidatesForImprovement, err := erp.reparentReplicas(ctx, ev, newPrimary, tabletMap, statusMap, opts, true, false) + validCandidatesForImprovement, err := erp.reparentReplicas(ctx, ev, source, tabletMap, statusMap, opts, true, false) if err != nil { return nil, err } // also include the current tablet for being considered as part of valid candidates for ERS promotion - validCandidatesForImprovement = append(validCandidatesForImprovement, newPrimary) + validCandidatesForImprovement = append(validCandidatesForImprovement, source) return validCandidatesForImprovement, nil } -// reparentReplicas reparents all the replicas provided and populates the reparent journal on the primary. +// reparentReplicas reparents all the replicas provided and populates the reparent journal on the primary if asked. // Also, it returns the replicas which started replicating only in the case where we wait for all the replicas -func (erp *EmergencyReparenter) reparentReplicas(ctx context.Context, ev *events.Reparent, newPrimaryTablet *topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus, opts EmergencyReparentOptions, waitForAllReplicas bool, populateReparentJournal bool) ([]*topodatapb.Tablet, error) { +func (erp *EmergencyReparenter) reparentReplicas( + ctx context.Context, + ev *events.Reparent, + newPrimaryTablet *topodatapb.Tablet, + tabletMap map[string]*topo.TabletInfo, + statusMap map[string]*replicationdatapb.StopReplicationStatus, + opts EmergencyReparentOptions, + waitForAllReplicas bool, + populateReparentJournal bool, +) ([]*topodatapb.Tablet, error) { var ( replicasStartedReplication []*topodatapb.Tablet @@ -526,6 +560,10 @@ func (erp *EmergencyReparenter) reparentReplicas(ctx context.Context, ev *events return nil, vterrors.Wrapf(primaryErr, "failed to PopulateReparentJournal on primary: %v", primaryErr) } + // We should only cancel the context that all the replicas are using when they are done. + // Since this function can return early when only 1 replica succeeds, if we cancel this context as a deferred call from this function, + // then we would end up having cancelled the context for the replicas who have not yet finished running all the commands. + // This leads to some replicas not starting replication properly. So we must wait for all the replicas to finish before cancelling this context. go func() { replWg.Wait() defer replCancel() @@ -558,18 +596,30 @@ func (erp *EmergencyReparenter) reparentReplicas(ctx context.Context, ev *events } -// intermediateCandidateIsIdeal is used to find whether the intermediate candidate that ERS chose is also the ideal one or not -func (erp *EmergencyReparenter) intermediateCandidateIsIdeal(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (bool, error) { +// intermediateSourceIsIdeal is used to find whether the intermediate source that ERS chose is also the ideal one or not +func (erp *EmergencyReparenter) intermediateSourceIsIdeal( + intermediateSource *topodatapb.Tablet, + prevPrimary *topodatapb.Tablet, + validCandidates []*topodatapb.Tablet, + tabletMap map[string]*topo.TabletInfo, + opts EmergencyReparentOptions, +) (bool, error) { // we try to find a better candidate with the current list of valid candidates, and if it matches our current primary candidate, then we return true - candidate, err := erp.identifyPrimaryCandidate(newPrimary, prevPrimary, validCandidates, tabletMap, opts) + candidate, err := erp.identifyPrimaryCandidate(intermediateSource, prevPrimary, validCandidates, tabletMap, opts) if err != nil { return false, err } - return candidate == newPrimary, nil + return candidate == intermediateSource, nil } -// identifyPrimaryCandidate is used to find a better candidate for ERS promotion -func (erp *EmergencyReparenter) identifyPrimaryCandidate(newPrimary, prevPrimary *topodatapb.Tablet, validCandidates []*topodatapb.Tablet, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions) (candidate *topodatapb.Tablet, err error) { +// identifyPrimaryCandidate is used to find the final candidate for ERS promotion +func (erp *EmergencyReparenter) identifyPrimaryCandidate( + intermediateSource *topodatapb.Tablet, + prevPrimary *topodatapb.Tablet, + validCandidates []*topodatapb.Tablet, + tabletMap map[string]*topo.TabletInfo, + opts EmergencyReparentOptions, +) (candidate *topodatapb.Tablet, err error) { defer func() { if candidate != nil { erp.logger.Infof("found better candidate - %v", candidate.Alias) @@ -604,8 +654,9 @@ func (erp *EmergencyReparenter) identifyPrimaryCandidate(newPrimary, prevPrimary } } - // So we've already promoted a replica. - // However, can we improve on our choice? Are there any replicas with better promotion rules? + // So we already have an intermediate source. What if our intermediate source was a rdonly? + // So we will try to improve our candidate selection. + // Are there any replicas with better promotion rules? // Maybe we actually promoted such a replica. Does that mean we should keep it? // Maybe we promoted a "neutral", and some "prefer" server is available. // Maybe we promoted a "prefer_not" @@ -613,12 +664,12 @@ func (erp *EmergencyReparenter) identifyPrimaryCandidate(newPrimary, prevPrimary // There's many options. We may wish to replace the server we promoted with a better one. // check whether the one we promoted is in the same cell and belongs to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, true, true) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, true, true) if candidate != nil { return candidate, nil } // check whether there is some other tablet in the same cell belonging to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, false, true) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, false, true) if candidate != nil { return candidate, nil } @@ -626,40 +677,40 @@ func (erp *EmergencyReparenter) identifyPrimaryCandidate(newPrimary, prevPrimary if !opts.PreventCrossCellPromotion { // check whether the one we promoted belongs to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, true, false) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, true, false) if candidate != nil { return candidate, nil } // check whether there is some other tablet belonging to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, preferredCandidates, false, false) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, false, false) if candidate != nil { return candidate, nil } } // repeat the same process for the neutral candidates list - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, true, true) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, true, true) if candidate != nil { return candidate, nil } - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, false, true) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, false, true) if candidate != nil { return candidate, nil } if !opts.PreventCrossCellPromotion { - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, true, false) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, true, false) if candidate != nil { return candidate, nil } - candidate = findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary, neutralReplicas, false, false) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, false, false) if candidate != nil { return candidate, nil } } - // return the one that we have if nothing found - return newPrimary, nil + // return the one that we have if nothing is found + return intermediateSource, nil } // checkIfConstraintsSatisfied is used to check whether the constraints for ERS are satisfied or not. @@ -673,14 +724,22 @@ func (erp *EmergencyReparenter) checkIfConstraintsSatisfied(newPrimary, prevPrim return nil } -func (erp *EmergencyReparenter) promoteNewPrimary(ctx context.Context, ev *events.Reparent, newPrimary *topodatapb.Tablet, opts EmergencyReparentOptions, tabletMap map[string]*topo.TabletInfo, statusMap map[string]*replicationdatapb.StopReplicationStatus) error { +func (erp *EmergencyReparenter) promoteNewPrimary( + ctx context.Context, + ev *events.Reparent, + newPrimary *topodatapb.Tablet, + opts EmergencyReparentOptions, + tabletMap map[string]*topo.TabletInfo, + statusMap map[string]*replicationdatapb.StopReplicationStatus, +) error { erp.logger.Infof("starting promotion for the new primary - %v", newPrimary.Alias) // we call PromoteReplica which changes the tablet type, fixes the semi-sync, set the primary to read-write and flushes the binlogs _, err := erp.tmc.PromoteReplica(ctx, newPrimary) if err != nil { return vterrors.Wrapf(err, "primary-elect tablet %v failed to be upgraded to primary: %v", newPrimary.Alias, err) } - // we now reparent all the replicas to the new primary we have promoted + // we now reparent all the replicas to the new primary we have promoted. + // Here we do not need to wait for all the replicas, We can finish early when even 1 succeeds. _, err = erp.reparentReplicas(ctx, ev, newPrimary, tabletMap, statusMap, opts, false, true) if err != nil { return err diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 344f8601beb..4eef5703e03 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -3043,24 +3043,24 @@ func TestEmergencyReparenter_reparentReplicas(t *testing.T) { } } -func TestEmergencyReparenter_promoteIntermediatePrimary(t *testing.T) { +func TestEmergencyReparenter_promoteIntermediateSource(t *testing.T) { t.Parallel() tests := []struct { - name string - emergencyReparentOps EmergencyReparentOptions - tmc *testutil.TabletManagerClient - unlockTopo bool - newPrimaryTabletAlias string - ts *topo.Server - keyspace string - shard string - tablets []*topodatapb.Tablet - tabletMap map[string]*topo.TabletInfo - statusMap map[string]*replicationdatapb.StopReplicationStatus - shouldErr bool - errShouldContain string - result []*topodatapb.Tablet + name string + emergencyReparentOps EmergencyReparentOptions + tmc *testutil.TabletManagerClient + unlockTopo bool + newSourceTabletAlias string + ts *topo.Server + keyspace string + shard string + tablets []*topodatapb.Tablet + tabletMap map[string]*topo.TabletInfo + statusMap map[string]*replicationdatapb.StopReplicationStatus + shouldErr bool + errShouldContain string + result []*topodatapb.Tablet }{ { name: "success", @@ -3083,7 +3083,7 @@ func TestEmergencyReparenter_promoteIntermediatePrimary(t *testing.T) { "zone1-0000000404": assert.AnError, // okay, because we're ignoring it. }, }, - newPrimaryTabletAlias: "zone1-0000000100", + newSourceTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { Tablet: &topodatapb.Tablet{ @@ -3183,7 +3183,7 @@ func TestEmergencyReparenter_promoteIntermediatePrimary(t *testing.T) { "zone1-0000000102": assert.AnError, }, }, - newPrimaryTabletAlias: "zone1-0000000100", + newSourceTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { Tablet: &topodatapb.Tablet{ @@ -3237,7 +3237,7 @@ func TestEmergencyReparenter_promoteIntermediatePrimary(t *testing.T) { "zone1-0000000102": assert.AnError, }, }, - newPrimaryTabletAlias: "zone1-0000000100", + newSourceTabletAlias: "zone1-0000000100", tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { Tablet: &topodatapb.Tablet{ @@ -3314,10 +3314,10 @@ func TestEmergencyReparenter_promoteIntermediatePrimary(t *testing.T) { require.NoError(t, lerr, "could not unlock %s/%s after test", tt.keyspace, tt.shard) }() } - tabletInfo := tt.tabletMap[tt.newPrimaryTabletAlias] + tabletInfo := tt.tabletMap[tt.newSourceTabletAlias] erp := NewEmergencyReparenter(tt.ts, tt.tmc, logger) - res, err := erp.promoteIntermediatePrimary(ctx, ev, tabletInfo.Tablet, tt.tabletMap, tt.statusMap, tt.emergencyReparentOps) + res, err := erp.promoteIntermediateSource(ctx, ev, tabletInfo.Tablet, tt.tabletMap, tt.statusMap, tt.emergencyReparentOps) if tt.shouldErr { assert.Error(t, err) assert.Contains(t, err.Error(), tt.errShouldContain) @@ -3334,7 +3334,7 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { tests := []struct { name string emergencyReparentOps EmergencyReparentOptions - newPrimary *topodatapb.Tablet + intermediateSource *topodatapb.Tablet prevPrimary *topodatapb.Tablet validCandidates []*topodatapb.Tablet tabletMap map[string]*topo.TabletInfo @@ -3347,8 +3347,8 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { Cell: "zone1", Uid: 100, }}, - newPrimary: nil, - prevPrimary: nil, + intermediateSource: nil, + prevPrimary: nil, validCandidates: []*topodatapb.Tablet{ { Alias: &topodatapb.TabletAlias{ @@ -3379,9 +3379,9 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { Cell: "zone1", Uid: 100, }}, - newPrimary: nil, - prevPrimary: nil, - validCandidates: nil, + intermediateSource: nil, + prevPrimary: nil, + validCandidates: nil, tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { Tablet: &topodatapb.Tablet{ @@ -3399,8 +3399,8 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { Cell: "zone1", Uid: 100, }}, - newPrimary: nil, - prevPrimary: nil, + intermediateSource: nil, + prevPrimary: nil, validCandidates: []*topodatapb.Tablet{ { Alias: &topodatapb.TabletAlias{ @@ -3414,7 +3414,7 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { }, { name: "preferred candidate in the same cell same as our replica", emergencyReparentOps: EmergencyReparentOptions{}, - newPrimary: &topodatapb.Tablet{ + intermediateSource: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", Uid: 100, @@ -3468,7 +3468,7 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { }, { name: "preferred candidate in the same cell different from original replica", emergencyReparentOps: EmergencyReparentOptions{}, - newPrimary: &topodatapb.Tablet{ + intermediateSource: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", Uid: 100, @@ -3522,7 +3522,7 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { }, { name: "preferred candidate in the different cell same as original replica", emergencyReparentOps: EmergencyReparentOptions{}, - newPrimary: &topodatapb.Tablet{ + intermediateSource: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone2", Uid: 101, @@ -3576,7 +3576,7 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { }, { name: "preferred candidate in the different cell different from original replica", emergencyReparentOps: EmergencyReparentOptions{}, - newPrimary: &topodatapb.Tablet{ + intermediateSource: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone2", Uid: 101, @@ -3630,7 +3630,7 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { }, { name: "prevent cross cell promotion", emergencyReparentOps: EmergencyReparentOptions{PreventCrossCellPromotion: true}, - newPrimary: &topodatapb.Tablet{ + intermediateSource: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", Uid: 100, @@ -3690,7 +3690,7 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { logger := logutil.NewMemoryLogger() erp := NewEmergencyReparenter(nil, nil, logger) - res, err := erp.identifyPrimaryCandidate(test.newPrimary, test.newPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) + res, err := erp.identifyPrimaryCandidate(test.intermediateSource, test.prevPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) if test.err != "" { assert.EqualError(t, err, test.err) return diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 50eaecd0a32..c15ae1080f9 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -161,7 +161,7 @@ func getValidCandidatesAndPositionsAsList(validCandidates map[string]mysql.Posit return validTablets, tabletPositions, nil } -// restrictValidCandidates is used to restrict some candidates from being considered eligible for becoming the intermediate primary +// restrictValidCandidates is used to restrict some candidates from being considered eligible for becoming the intermediate source or the final promotion candidate func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo) (map[string]mysql.Position, error) { restrictedValidCandidates := make(map[string]mysql.Position) for candidate, position := range validCandidates { @@ -178,7 +178,13 @@ func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMa return restrictedValidCandidates, nil } -func findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary *topodatapb.Tablet, possibleCandidates []*topodatapb.Tablet, checkEqualPrimary bool, checkSameCell bool) *topodatapb.Tablet { +func findPossibleCandidateFromListWithRestrictions( + newPrimary *topodatapb.Tablet, + prevPrimary *topodatapb.Tablet, + possibleCandidates []*topodatapb.Tablet, + checkEqualPrimary bool, + checkSameCell bool, +) *topodatapb.Tablet { for _, candidate := range possibleCandidates { if checkEqualPrimary && !(topoproto.TabletAliasEqual(newPrimary.Alias, candidate.Alias)) { continue @@ -191,11 +197,18 @@ func findPossibleCandidateFromListWithRestrictions(newPrimary, prevPrimary *topo return nil } -// waitForCatchUp promotes the newer candidate over the primary candidate that we have, but it does not set to start accepting writes -func waitForCatchUp(ctx context.Context, tmc tmclient.TabletManagerClient, logger logutil.Logger, prevPrimary, newPrimary *topodatapb.Tablet, waitTime time.Duration) error { - logger.Infof("waiting for %v to catch up to %v", newPrimary.Alias, prevPrimary.Alias) +// waitForCatchUp is used to wait for the given tablet until it has caught up to the source +func waitForCatchUp( + ctx context.Context, + tmc tmclient.TabletManagerClient, + logger logutil.Logger, + newPrimary *topodatapb.Tablet, + source *topodatapb.Tablet, + waitTime time.Duration, +) error { + logger.Infof("waiting for %v to catch up to %v", newPrimary.Alias, source.Alias) // Find the primary position of the previous primary - pos, err := tmc.MasterPosition(ctx, prevPrimary) + pos, err := tmc.MasterPosition(ctx, source) if err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index cf3fa17ae5f..770d2f7030f 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -27,13 +27,14 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/utils" "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" + "vitess.io/vitess/go/vt/vttablet/tmclient" + replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/proto/vttime" - "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" - "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" - "vitess.io/vitess/go/vt/vttablet/tmclient" ) type chooseNewPrimaryTestTMClient struct { @@ -630,11 +631,11 @@ func TestGetValidCandidatesAndPositionsAsList(t *testing.T) { func TestWaitForCatchUp(t *testing.T) { tests := []struct { - name string - tmc tmclient.TabletManagerClient - prevPrimary *topodatapb.Tablet - newPrimary *topodatapb.Tablet - err string + name string + tmc tmclient.TabletManagerClient + source *topodatapb.Tablet + newPrimary *topodatapb.Tablet + err string }{ { name: "success", @@ -654,7 +655,7 @@ func TestWaitForCatchUp(t *testing.T) { }, }, }, - prevPrimary: &topodatapb.Tablet{ + source: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", Uid: 100, @@ -684,7 +685,7 @@ func TestWaitForCatchUp(t *testing.T) { }, }, }, - prevPrimary: &topodatapb.Tablet{ + source: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", Uid: 100, @@ -715,7 +716,7 @@ func TestWaitForCatchUp(t *testing.T) { }, }, }, - prevPrimary: &topodatapb.Tablet{ + source: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", Uid: 100, @@ -735,7 +736,7 @@ func TestWaitForCatchUp(t *testing.T) { t.Run(test.name, func(t *testing.T) { ctx := context.Background() logger := logutil.NewMemoryLogger() - err := waitForCatchUp(ctx, test.tmc, logger, test.prevPrimary, test.newPrimary, 2*time.Second) + err := waitForCatchUp(ctx, test.tmc, logger, test.newPrimary, test.source, 2*time.Second) if test.err != "" { assert.EqualError(t, err, test.err) } else { From cb76b33bc3904efc5c9d840653329594b5f53550 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 28 Sep 2021 18:34:37 +0530 Subject: [PATCH 169/176] keep pb imports separate Signed-off-by: Manan Gupta --- go/test/endtoend/reparent/emergencyreparent/utils_test.go | 7 ++++--- go/test/endtoend/reparent/plannedreparent/utils_test.go | 7 ++++--- go/vt/orchestrator/inst/instance_dao.go | 5 +++-- go/vt/orchestrator/logic/topology_recovery.go | 5 +++-- go/vt/vtctl/reparentutil/durability.go | 3 ++- go/vt/vtctl/reparentutil/durability_test.go | 1 + go/vt/vtctl/reparentutil/emergency_reparenter_test.go | 1 + go/vt/vtctl/reparentutil/ers_sorter.go | 3 ++- 8 files changed, 20 insertions(+), 12 deletions(-) diff --git a/go/test/endtoend/reparent/emergencyreparent/utils_test.go b/go/test/endtoend/reparent/emergencyreparent/utils_test.go index 8e870a93af1..71e231e7180 100644 --- a/go/test/endtoend/reparent/emergencyreparent/utils_test.go +++ b/go/test/endtoend/reparent/emergencyreparent/utils_test.go @@ -30,12 +30,13 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/json2" - "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/vt/log" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/endtoend/cluster" tmc "vitess.io/vitess/go/vt/vttablet/grpctmclient" ) diff --git a/go/test/endtoend/reparent/plannedreparent/utils_test.go b/go/test/endtoend/reparent/plannedreparent/utils_test.go index b392ad6149c..e9314b9cfd9 100644 --- a/go/test/endtoend/reparent/plannedreparent/utils_test.go +++ b/go/test/endtoend/reparent/plannedreparent/utils_test.go @@ -32,12 +32,13 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/json2" - "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/vt/log" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/endtoend/cluster" tmc "vitess.io/vitess/go/vt/vttablet/grpctmclient" ) diff --git a/go/vt/orchestrator/inst/instance_dao.go b/go/vt/orchestrator/inst/instance_dao.go index a909c438a16..25f77b6c868 100644 --- a/go/vt/orchestrator/inst/instance_dao.go +++ b/go/vt/orchestrator/inst/instance_dao.go @@ -40,6 +40,9 @@ import ( "vitess.io/vitess/go/vt/orchestrator/external/golib/math" "vitess.io/vitess/go/vt/orchestrator/external/golib/sqlutils" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/orchestrator/attributes" "vitess.io/vitess/go/vt/orchestrator/collection" "vitess.io/vitess/go/vt/orchestrator/config" @@ -47,8 +50,6 @@ import ( "vitess.io/vitess/go/vt/orchestrator/kv" "vitess.io/vitess/go/vt/orchestrator/metrics/query" "vitess.io/vitess/go/vt/orchestrator/util" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" ) diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index e9c1e9e870c..b2c27a93f1d 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -31,6 +31,9 @@ import ( "github.com/patrickmn/go-cache" "github.com/rcrowley/go-metrics" + logutilpb "vitess.io/vitess/go/vt/proto/logutil" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/orchestrator/attributes" "vitess.io/vitess/go/vt/orchestrator/config" @@ -41,8 +44,6 @@ import ( "vitess.io/vitess/go/vt/orchestrator/os" "vitess.io/vitess/go/vt/orchestrator/process" "vitess.io/vitess/go/vt/orchestrator/util" - logutilpb "vitess.io/vitess/go/vt/proto/logutil" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/vtctl/reparentutil" "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" "vitess.io/vitess/go/vt/vttablet/tmclient" diff --git a/go/vt/vtctl/reparentutil/durability.go b/go/vt/vtctl/reparentutil/durability.go index 715ba335a93..a1b470117ce 100644 --- a/go/vt/vtctl/reparentutil/durability.go +++ b/go/vt/vtctl/reparentutil/durability.go @@ -19,9 +19,10 @@ package reparentutil import ( "fmt" - "vitess.io/vitess/go/vt/log" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo/topoproto" + + "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" ) diff --git a/go/vt/vtctl/reparentutil/durability_test.go b/go/vt/vtctl/reparentutil/durability_test.go index d962b7f1f80..58740d1e600 100644 --- a/go/vt/vtctl/reparentutil/durability_test.go +++ b/go/vt/vtctl/reparentutil/durability_test.go @@ -24,6 +24,7 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/vtctl/reparentutil/promotionrule" ) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 4eef5703e03..c4254be934e 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -31,6 +31,7 @@ import ( "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/topotools/events" "vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil" diff --git a/go/vt/vtctl/reparentutil/ers_sorter.go b/go/vt/vtctl/reparentutil/ers_sorter.go index 832b4328dea..d4b3f70a03b 100644 --- a/go/vt/vtctl/reparentutil/ers_sorter.go +++ b/go/vt/vtctl/reparentutil/ers_sorter.go @@ -20,9 +20,10 @@ import ( "sort" "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/vt/vterrors" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" - "vitess.io/vitess/go/vt/vterrors" ) // ERSSorter sorts tablets by GTID positions and Promotion rules aimed at finding the best From c4f2b6be8f00a00d54d2683acbed0483ee8b37e2 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 29 Sep 2021 11:46:16 +0530 Subject: [PATCH 170/176] refactor according to review comments Signed-off-by: Manan Gupta --- go/vt/orchestrator/logic/orchestrator.go | 2 +- go/vt/orchestrator/logic/topology_recovery.go | 4 +-- go/vt/vtctl/reparentutil/durability.go | 2 +- .../reparentutil/emergency_reparenter.go | 26 ++++++++----------- .../reparentutil/emergency_reparenter_test.go | 2 +- go/vt/vtctl/reparentutil/ers_sorter.go | 6 ++--- go/vt/vtctl/reparentutil/util.go | 2 +- 7 files changed, 20 insertions(+), 24 deletions(-) diff --git a/go/vt/orchestrator/logic/orchestrator.go b/go/vt/orchestrator/logic/orchestrator.go index 814b7b5f058..74def388579 100644 --- a/go/vt/orchestrator/logic/orchestrator.go +++ b/go/vt/orchestrator/logic/orchestrator.go @@ -506,7 +506,7 @@ func ContinuousDiscovery() { } }() case <-tabletTopoTick: - go RefreshTablets(false) + go RefreshTablets(false /* forceRefresh */) } } } diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index b2c27a93f1d..cdfa494152d 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -637,7 +637,7 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat // So we need to check that we only run an ERS if the instance that we analyzed was actually a primary! Otherwise, we would end up running an ERS // even when the cluster is fine or the problem can be fixed via some other recovery if tablet.Type != topodatapb.TabletType_PRIMARY { - RefreshTablets(true) + RefreshTablets(true /* forceRefresh */) AuditTopologyRecovery(topologyRecovery, "another agent seems to have fixed the problem") return false, topologyRecovery, nil } @@ -684,7 +684,7 @@ func checkAndRecoverDeadPrimary(analysisEntry inst.ReplicationAnalysis, candidat // here we need to forcefully refresh all the tablets otherwise old information is used and failover scenarios are spawned off which are not required // For example, if we do not refresh the tablets forcefully and the new primary is found in the cache then its source key is not updated and this spawns off // PrimaryHasPrimary analysis which runs another ERS - RefreshTablets(true) + RefreshTablets(true /* forceRefresh */) var promotedReplica *inst.Instance if ev.NewPrimary != nil { promotedReplica, _, _ = inst.ReadInstance(&inst.InstanceKey{ diff --git a/go/vt/vtctl/reparentutil/durability.go b/go/vt/vtctl/reparentutil/durability.go index a1b470117ce..7f4449d7573 100644 --- a/go/vt/vtctl/reparentutil/durability.go +++ b/go/vt/vtctl/reparentutil/durability.go @@ -76,7 +76,7 @@ func SetDurabilityPolicy(name string, durabilityParams map[string]string) error if !found { return fmt.Errorf("durability policy %v not found", name) } - log.Infof("Durability setting: %v", name) + log.Infof("Setting durability policy to %v", name) curDurabilityPolicy = newDurabilityCreationFunc(durabilityParams) return nil } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index bc7846c2c9d..812a40306a7 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -102,7 +102,6 @@ func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace stri if err != nil { return nil, err } - // defer the unlock-shard function defer unlock(&err) // dispatch success or failure of ERS @@ -118,7 +117,6 @@ func (erp *EmergencyReparenter) ReparentShard(ctx context.Context, keyspace stri } }() - // run ERS with shard already locked err = erp.reparentShardLocked(ctx, ev, keyspace, shard, opts) return ev, err @@ -140,7 +138,6 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve erp.logger.Infof("will initiate emergency reparent shard in keyspace - %s, shard - %s", keyspace, shard) ersCounter.Add(1) - // variables used by the ERS functions are declared here var ( shardInfo *topo.ShardInfo prevPrimary *topodatapb.Tablet @@ -155,7 +152,6 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve isIdeal bool ) - // get the shard information from the topology server shardInfo, err = erp.ts.GetShard(ctx, keyspace, shard) if err != nil { return err @@ -173,7 +169,7 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve prevPrimary = prevPrimaryInfo.Tablet } - // read all the tablets and there information + // read all the tablets and their information event.DispatchUpdate(ev, "reading all tablets") tabletMap, err = erp.ts.GetTabletMapForShard(ctx, keyspace, shard) if err != nil { @@ -428,7 +424,7 @@ func (erp *EmergencyReparenter) promoteIntermediateSource( ) ([]*topodatapb.Tablet, error) { // we reparent all the other tablets to start replication from our new source // we wait for all the replicas so that we can choose a better candidate from the ones that started replication later - validCandidatesForImprovement, err := erp.reparentReplicas(ctx, ev, source, tabletMap, statusMap, opts, true, false) + validCandidatesForImprovement, err := erp.reparentReplicas(ctx, ev, source, tabletMap, statusMap, opts, true /* waitForAllReplicas */, false /* populateReparentJournal */) if err != nil { return nil, err } @@ -664,12 +660,12 @@ func (erp *EmergencyReparenter) identifyPrimaryCandidate( // There's many options. We may wish to replace the server we promoted with a better one. // check whether the one we promoted is in the same cell and belongs to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, true, true) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, true /* checkEqualPrimary */, true /* checkSameCell */) if candidate != nil { return candidate, nil } // check whether there is some other tablet in the same cell belonging to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, false, true) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, false /* checkEqualPrimary */, true /* checkSameCell */) if candidate != nil { return candidate, nil } @@ -677,33 +673,33 @@ func (erp *EmergencyReparenter) identifyPrimaryCandidate( if !opts.PreventCrossCellPromotion { // check whether the one we promoted belongs to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, true, false) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, true /* checkEqualPrimary */, false /* checkSameCell */) if candidate != nil { return candidate, nil } // check whether there is some other tablet belonging to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, false, false) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, false /* checkEqualPrimary */, false /* checkSameCell */) if candidate != nil { return candidate, nil } } // repeat the same process for the neutral candidates list - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, true, true) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, true /* checkEqualPrimary */, true /* checkSameCell */) if candidate != nil { return candidate, nil } - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, false, true) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, false /* checkEqualPrimary */, true /* checkSameCell */) if candidate != nil { return candidate, nil } if !opts.PreventCrossCellPromotion { - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, true, false) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, true /* checkEqualPrimary */, false /* checkSameCell */) if candidate != nil { return candidate, nil } - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, false, false) + candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, false /* checkEqualPrimary */, false /* checkSameCell */) if candidate != nil { return candidate, nil } @@ -740,7 +736,7 @@ func (erp *EmergencyReparenter) promoteNewPrimary( } // we now reparent all the replicas to the new primary we have promoted. // Here we do not need to wait for all the replicas, We can finish early when even 1 succeeds. - _, err = erp.reparentReplicas(ctx, ev, newPrimary, tabletMap, statusMap, opts, false, true) + _, err = erp.reparentReplicas(ctx, ev, newPrimary, tabletMap, statusMap, opts, false /* waitForAllReplicas */, true /* populateReparentJournal */) if err != nil { return err } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index c4254be934e..8fa4806a64a 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -3032,7 +3032,7 @@ func TestEmergencyReparenter_reparentReplicas(t *testing.T) { tabletInfo := tt.tabletMap[tt.newPrimaryTabletAlias] erp := NewEmergencyReparenter(tt.ts, tt.tmc, logger) - _, err := erp.reparentReplicas(ctx, ev, tabletInfo.Tablet, tt.tabletMap, tt.statusMap, tt.emergencyReparentOps, false, true) + _, err := erp.reparentReplicas(ctx, ev, tabletInfo.Tablet, tt.tabletMap, tt.statusMap, tt.emergencyReparentOps, false /* waitForAllReplicas */, true /* populateReparentJournal */) if tt.shouldErr { assert.Error(t, err) assert.Contains(t, err.Error(), tt.errShouldContain) diff --git a/go/vt/vtctl/reparentutil/ers_sorter.go b/go/vt/vtctl/reparentutil/ers_sorter.go index d4b3f70a03b..02dabe7ba9c 100644 --- a/go/vt/vtctl/reparentutil/ers_sorter.go +++ b/go/vt/vtctl/reparentutil/ers_sorter.go @@ -34,8 +34,8 @@ type ERSSorter struct { idealCell string } -// NewErsSorter creates a new ERSSorter -func NewErsSorter(tablets []*topodatapb.Tablet, positions []mysql.Position, idealCell string) *ERSSorter { +// NewERSSorter creates a new ERSSorter +func NewERSSorter(tablets []*topodatapb.Tablet, positions []mysql.Position, idealCell string) *ERSSorter { return &ERSSorter{ tablets: tablets, positions: positions, @@ -101,6 +101,6 @@ func sortTabletsForERS(tablets []*topodatapb.Tablet, positions []mysql.Position, return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unequal number of tablets and positions") } - sort.Sort(NewErsSorter(tablets, positions, idealCell)) + sort.Sort(NewERSSorter(tablets, positions, idealCell)) return nil } diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index c15ae1080f9..9d2a17006cb 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -170,7 +170,7 @@ func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMa return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) } // We do not allow BACKUP, DRAINED or RESTORE type of tablets to be considered for being the replication source or the candidate for primary - if candidateInfo.Type == topodatapb.TabletType_BACKUP || candidateInfo.Type == topodatapb.TabletType_RESTORE || candidateInfo.Type == topodatapb.TabletType_DRAINED { + if topoproto.IsTypeInList(candidateInfo.Type, []topodatapb.TabletType{topodatapb.TabletType_BACKUP, topodatapb.TabletType_RESTORE, topodatapb.TabletType_DRAINED}) { continue } restrictedValidCandidates[candidate] = position From 39ca79fc0c75fce91cd2d4491016ada75d54bbfc Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 29 Sep 2021 22:57:36 +0530 Subject: [PATCH 171/176] random selection of tablets with respect to cells if preventCrossCell set to false Signed-off-by: Manan Gupta --- .../reparent/emergencyreparent/ers_test.go | 13 +-- .../reparent/emergencyreparent/utils_test.go | 7 +- .../endtoend/vtorc/primary_failure_test.go | 2 +- .../reparentutil/emergency_reparenter.go | 60 +++++--------- .../reparentutil/emergency_reparenter_test.go | 80 ++----------------- go/vt/vtctl/reparentutil/ers_sorter.go | 21 +---- go/vt/vtctl/reparentutil/ers_sorter_test.go | 18 ++--- go/vt/vtctl/reparentutil/util.go | 34 ++++++-- 8 files changed, 77 insertions(+), 158 deletions(-) diff --git a/go/test/endtoend/reparent/emergencyreparent/ers_test.go b/go/test/endtoend/reparent/emergencyreparent/ers_test.go index f8a353b0de7..a5e0a4ec5d1 100644 --- a/go/test/endtoend/reparent/emergencyreparent/ers_test.go +++ b/go/test/endtoend/reparent/emergencyreparent/ers_test.go @@ -72,7 +72,7 @@ func TestReparentIgnoreReplicas(t *testing.T) { require.NotNil(t, err, out) // Now let's run it again, but set the command to ignore the unreachable replica. - out, err = ersIgnoreTablet(nil, "60s", "30s", []*cluster.Vttablet{tab3}) + out, err = ersIgnoreTablet(nil, "60s", "30s", []*cluster.Vttablet{tab3}, false) require.Nil(t, err, out) // We'll bring back the replica we took down. @@ -111,7 +111,7 @@ func TestERSPromoteRdonly(t *testing.T) { stopTablet(t, tab1, true) // We expect this one to fail because we have ignored all the replicas and have only the rdonly's which should not be promoted - out, err := ersIgnoreTablet(nil, "30s", "30s", []*cluster.Vttablet{tab4}) + out, err := ersIgnoreTablet(nil, "30s", "30s", []*cluster.Vttablet{tab4}, false) require.NotNil(t, err, out) out, err = clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("GetShard", keyspaceShard) @@ -119,8 +119,8 @@ func TestERSPromoteRdonly(t *testing.T) { require.Contains(t, out, `"uid": 101`, "the primary should still be 101 in the shard info") } -// TestERSPrefersSameCell tests that we prefer to promote a replica in the same cell as the previous primary -func TestERSPrefersSameCell(t *testing.T) { +// TestERSPreventCrossCellPromotion tests that we promote a replica in the same cell as the previous primary if prevent cross cell promotion flag is set +func TestERSPreventCrossCellPromotion(t *testing.T) { defer cluster.PanicHandler(t) setupReparentCluster(t) defer teardownCluster() @@ -133,7 +133,7 @@ func TestERSPrefersSameCell(t *testing.T) { stopTablet(t, tab1, true) // We expect that tab3 will be promoted since it is in the same cell as the previous primary - out, err := ersIgnoreTablet(nil, "60s", "30s", []*cluster.Vttablet{tab2}) + out, err := ersIgnoreTablet(nil, "60s", "30s", []*cluster.Vttablet{tab2}, true) require.NoError(t, err, out) newPrimary := getNewPrimary(t) @@ -190,8 +190,9 @@ func TestPullFromRdonly(t *testing.T) { // We have simulated a network partition in which the primary and rdonly got isolated and then the primary went down leaving the rdonly most advanced // We expect that tab3 will be promoted since it is in the same cell as the previous primary + // since we are preventing cross cell promotions // Also it must be fully caught up - out, err := ers(nil, "60s", "30s") + out, err := ersIgnoreTablet(nil, "60s", "30s", nil, true) require.NoError(t, err, out) newPrimary := getNewPrimary(t) diff --git a/go/test/endtoend/reparent/emergencyreparent/utils_test.go b/go/test/endtoend/reparent/emergencyreparent/utils_test.go index 71e231e7180..eb3199fa2db 100644 --- a/go/test/endtoend/reparent/emergencyreparent/utils_test.go +++ b/go/test/endtoend/reparent/emergencyreparent/utils_test.go @@ -216,10 +216,10 @@ func execute(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { // region ers func ers(tab *cluster.Vttablet, totalTimeout, waitReplicasTimeout string) (string, error) { - return ersIgnoreTablet(tab, totalTimeout, waitReplicasTimeout, nil) + return ersIgnoreTablet(tab, totalTimeout, waitReplicasTimeout, nil, false) } -func ersIgnoreTablet(tab *cluster.Vttablet, timeout, waitReplicasTimeout string, tabletsToIgnore []*cluster.Vttablet) (string, error) { +func ersIgnoreTablet(tab *cluster.Vttablet, timeout, waitReplicasTimeout string, tabletsToIgnore []*cluster.Vttablet, preventCrossCellPromotion bool) (string, error) { var args []string if timeout != "" { args = append(args, "-action_timeout", timeout) @@ -231,6 +231,9 @@ func ersIgnoreTablet(tab *cluster.Vttablet, timeout, waitReplicasTimeout string, if waitReplicasTimeout != "" { args = append(args, "-wait_replicas_timeout", waitReplicasTimeout) } + if preventCrossCellPromotion { + args = append(args, "-prevent_cross_cell_promotion=true") + } if len(tabletsToIgnore) != 0 { tabsString := "" for _, vttablet := range tabletsToIgnore { diff --git a/go/test/endtoend/vtorc/primary_failure_test.go b/go/test/endtoend/vtorc/primary_failure_test.go index 1b4c8e28c2a..08605d612ac 100644 --- a/go/test/endtoend/vtorc/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primary_failure_test.go @@ -366,7 +366,7 @@ func TestDownPrimaryPromotionRule(t *testing.T) { permanentlyRemoveVttablet(curPrimary) }() - // we have a replica in the same cell, so that is the one which should be promoted and not the one from another cell + // we have a replica with a preferred promotion rule, so that is the one which should be promoted checkPrimaryTablet(t, clusterInstance, crossCellReplica, true) // also check that the replication is working correctly after failover verifyWritesSucceed(t, crossCellReplica, []*cluster.Vttablet{rdonly, replica}, 10*time.Second) diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 812a40306a7..bcbaefe7e11 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -207,11 +207,11 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // Find the intermediate source for replication that we want other tablets to replicate from. - // This step chooses the most advanced tablet. Further ties are broken by using the cell of the previous primary and the promotion rule. + // This step chooses the most advanced tablet. Further ties are broken by using the promotion rule. // In case the user has specified a tablet specifically, then it is selected, as long as it is the most advanced. // Here we also check for split brain scenarios and check that the selected replica must be more advanced than all the other valid candidates. // We fail in case there is a split brain detected. - intermediateSource, validCandidateTablets, err = erp.findMostAdvanced(prevPrimary, validCandidates, tabletMap, opts) + intermediateSource, validCandidateTablets, err = erp.findMostAdvanced(validCandidates, tabletMap, opts) if err != nil { return err } @@ -349,9 +349,8 @@ func (erp *EmergencyReparenter) waitForAllRelayLogsToApply( return nil } -// findMostAdvanced finds the intermediate source for ERS. We always choose the most advanced one from our valid candidates list. Further ties are broken by looking at the cell and promotion rules. +// findMostAdvanced finds the intermediate source for ERS. We always choose the most advanced one from our valid candidates list. Further ties are broken by looking at the promotion rules. func (erp *EmergencyReparenter) findMostAdvanced( - prevPrimary *topodatapb.Tablet, validCandidates map[string]mysql.Position, tabletMap map[string]*topo.TabletInfo, opts EmergencyReparentOptions, @@ -363,13 +362,8 @@ func (erp *EmergencyReparenter) findMostAdvanced( return nil, nil, err } - idealCell := "" - if prevPrimary != nil { - idealCell = prevPrimary.Alias.Cell - } - // sort the tablets for finding the best intermediate source in ERS - err = sortTabletsForERS(validTablets, tabletPositions, idealCell) + err = sortTabletsForERS(validTablets, tabletPositions) if err != nil { return nil, nil, err } @@ -391,7 +385,6 @@ func (erp *EmergencyReparenter) findMostAdvanced( // If we were requested to elect a particular primary, verify it's a valid // candidate (non-zero position, no errant GTIDs) - // Also, if the candidate is if opts.NewPrimaryAlias != nil { requestedPrimaryAlias := topoproto.TabletAliasString(opts.NewPrimaryAlias) pos, ok := validCandidates[requestedPrimaryAlias] @@ -659,47 +652,34 @@ func (erp *EmergencyReparenter) identifyPrimaryCandidate( // Maybe we promoted a server in a different cell than the primary // There's many options. We may wish to replace the server we promoted with a better one. - // check whether the one we promoted is in the same cell and belongs to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, true /* checkEqualPrimary */, true /* checkSameCell */) - if candidate != nil { - return candidate, nil - } - // check whether there is some other tablet in the same cell belonging to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, false /* checkEqualPrimary */, true /* checkSameCell */) - if candidate != nil { - return candidate, nil - } - // we do not have a preferred candidate in the same cell - - if !opts.PreventCrossCellPromotion { - // check whether the one we promoted belongs to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, true /* checkEqualPrimary */, false /* checkSameCell */) + // If the user requested for prevention of cross cell promotion then we should only search for valid candidates in the same cell + // otherwise we can search in any cell + if opts.PreventCrossCellPromotion { + // find candidates in the same cell from the preferred candidates list + candidate = findCandidateSameCell(intermediateSource, prevPrimary, preferredCandidates) if candidate != nil { return candidate, nil } - // check whether there is some other tablet belonging to the preferred candidates list - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, preferredCandidates, false /* checkEqualPrimary */, false /* checkSameCell */) + // we do not have a preferred candidate in the same cell + } else { + // find candidates in any cell from the preferred candidates list + candidate = findCandidateAnyCell(intermediateSource, preferredCandidates) if candidate != nil { return candidate, nil } } // repeat the same process for the neutral candidates list - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, true /* checkEqualPrimary */, true /* checkSameCell */) - if candidate != nil { - return candidate, nil - } - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, false /* checkEqualPrimary */, true /* checkSameCell */) - if candidate != nil { - return candidate, nil - } - - if !opts.PreventCrossCellPromotion { - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, true /* checkEqualPrimary */, false /* checkSameCell */) + if opts.PreventCrossCellPromotion { + // find candidates in the same cell from the neutral candidates list + candidate = findCandidateSameCell(intermediateSource, prevPrimary, neutralReplicas) if candidate != nil { return candidate, nil } - candidate = findPossibleCandidateFromListWithRestrictions(intermediateSource, prevPrimary, neutralReplicas, false /* checkEqualPrimary */, false /* checkSameCell */) + // we do not have a neutral candidate in the same cell + } else { + // find candidates in any cell from the neutral candidates list + candidate = findCandidateAnyCell(intermediateSource, neutralReplicas) if candidate != nil { return candidate, nil } diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index 8fa4806a64a..af254126094 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -2269,13 +2269,12 @@ func TestEmergencyReparenter_findMostAdvanced(t *testing.T) { name string validCandidates map[string]mysql.Position tabletMap map[string]*topo.TabletInfo - prevPrimary *topodatapb.Tablet emergencyReparentOps EmergencyReparentOptions result *topodatapb.Tablet err string }{ { - name: "choose most advanced - with nil previous primary", + name: "choose most advanced", validCandidates: map[string]mysql.Position{ "zone1-0000000100": positionMostAdvanced, "zone1-0000000101": positionIntermediate1, @@ -2316,60 +2315,6 @@ func TestEmergencyReparenter_findMostAdvanced(t *testing.T) { }, }, }, - prevPrimary: nil, - result: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, { - name: "choose most advanced in the same cell of previous primary", - validCandidates: map[string]mysql.Position{ - "zone1-0000000100": positionMostAdvanced, - "zone1-0000000101": positionIntermediate1, - "zone2-0000000100": positionMostAdvanced, - }, - tabletMap: map[string]*topo.TabletInfo{ - "zone1-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 100, - }, - }, - }, - "zone1-0000000101": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 101, - }, - }, - }, - "zone2-0000000100": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone2", - Uid: 100, - }, - }, - }, - "zone1-0000000404": { - Tablet: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - Uid: 404, - }, - Hostname: "ignored tablet", - }, - }, - }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - }, - }, result: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", @@ -2420,11 +2365,6 @@ func TestEmergencyReparenter_findMostAdvanced(t *testing.T) { }, }, }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - }, - }, result: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", @@ -2479,11 +2419,6 @@ func TestEmergencyReparenter_findMostAdvanced(t *testing.T) { }, }, }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - }, - }, result: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", @@ -2538,11 +2473,6 @@ func TestEmergencyReparenter_findMostAdvanced(t *testing.T) { }, }, }, - prevPrimary: &topodatapb.Tablet{ - Alias: &topodatapb.TabletAlias{ - Cell: "zone1", - }, - }, err: "split brain detected between servers", }, } @@ -2553,7 +2483,7 @@ func TestEmergencyReparenter_findMostAdvanced(t *testing.T) { t.Run(test.name, func(t *testing.T) { erp := NewEmergencyReparenter(nil, nil, logutil.NewMemoryLogger()) - winningTablet, _, err := erp.findMostAdvanced(test.prevPrimary, test.validCandidates, test.tabletMap, test.emergencyReparentOps) + winningTablet, _, err := erp.findMostAdvanced(test.validCandidates, test.tabletMap, test.emergencyReparentOps) if test.err != "" { assert.Error(t, err) assert.Contains(t, err.Error(), test.err) @@ -3414,7 +3344,7 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { err: "candidate zone1-0000000100 not found in the tablet map; this an impossible situation", }, { name: "preferred candidate in the same cell same as our replica", - emergencyReparentOps: EmergencyReparentOptions{}, + emergencyReparentOps: EmergencyReparentOptions{PreventCrossCellPromotion: true}, intermediateSource: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", @@ -3468,7 +3398,7 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { }, }, { name: "preferred candidate in the same cell different from original replica", - emergencyReparentOps: EmergencyReparentOptions{}, + emergencyReparentOps: EmergencyReparentOptions{PreventCrossCellPromotion: true}, intermediateSource: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", @@ -3540,7 +3470,7 @@ func TestEmergencyReparenter_identifyPrimaryCandidate(t *testing.T) { Cell: "zone1", Uid: 100, }, - Type: topodatapb.TabletType_RDONLY, + Type: topodatapb.TabletType_REPLICA, }, { Alias: &topodatapb.TabletAlias{ Cell: "zone1", diff --git a/go/vt/vtctl/reparentutil/ers_sorter.go b/go/vt/vtctl/reparentutil/ers_sorter.go index 02dabe7ba9c..8a93178cdcd 100644 --- a/go/vt/vtctl/reparentutil/ers_sorter.go +++ b/go/vt/vtctl/reparentutil/ers_sorter.go @@ -31,15 +31,13 @@ import ( type ERSSorter struct { tablets []*topodatapb.Tablet positions []mysql.Position - idealCell string } // NewERSSorter creates a new ERSSorter -func NewERSSorter(tablets []*topodatapb.Tablet, positions []mysql.Position, idealCell string) *ERSSorter { +func NewERSSorter(tablets []*topodatapb.Tablet, positions []mysql.Position) *ERSSorter { return &ERSSorter{ tablets: tablets, positions: positions, - idealCell: idealCell, } } @@ -76,31 +74,20 @@ func (ersSorter *ERSSorter) Less(i, j int) bool { } // at this point, both have the same GTIDs - // So, we will now check which one is in the same cell - iInSameCell := ersSorter.tablets[i].Alias.Cell == ersSorter.idealCell - jInSameCell := ersSorter.tablets[j].Alias.Cell == ersSorter.idealCell - if iInSameCell && !jInSameCell { - return true - } - if jInSameCell && !iInSameCell { - return false - } - - // at this point, either both are in the ideal cell - // or neither is, so we check their promotion rules + // so we check their promotion rules jPromotionRule := PromotionRule(ersSorter.tablets[j]) iPromotionRule := PromotionRule(ersSorter.tablets[i]) return !jPromotionRule.BetterThan(iPromotionRule) } // sortTabletsForERS sorts the tablets, given their positions for emergency reparent shard -func sortTabletsForERS(tablets []*topodatapb.Tablet, positions []mysql.Position, idealCell string) error { +func sortTabletsForERS(tablets []*topodatapb.Tablet, positions []mysql.Position) error { // throw an error internal error in case of unequal number of tablets and positions // fail-safe code prevents panic in sorting in case the lengths are unequal if len(tablets) != len(positions) { return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unequal number of tablets and positions") } - sort.Sort(NewERSSorter(tablets, positions, idealCell)) + sort.Sort(NewERSSorter(tablets, positions)) return nil } diff --git a/go/vt/vtctl/reparentutil/ers_sorter_test.go b/go/vt/vtctl/reparentutil/ers_sorter_test.go index d22aacd216a..4d527675048 100644 --- a/go/vt/vtctl/reparentutil/ers_sorter_test.go +++ b/go/vt/vtctl/reparentutil/ers_sorter_test.go @@ -91,40 +91,34 @@ func TestErsSorter(t *testing.T) { name string tablets []*topodatapb.Tablet positions []mysql.Position - idealCell string containsErr string sortedTablets []*topodatapb.Tablet }{ { - name: "all advanced, ideal cell 1", - tablets: []*topodatapb.Tablet{nil, tabletReplica2_100, tabletReplica1_100, tabletRdonly1_102}, - positions: []mysql.Position{positionMostAdvanced, positionMostAdvanced, positionMostAdvanced, positionMostAdvanced}, - idealCell: cell1, - sortedTablets: []*topodatapb.Tablet{tabletReplica1_100, tabletRdonly1_102, tabletReplica2_100, nil}, + name: "all advanced, sort via promotion rules", + tablets: []*topodatapb.Tablet{nil, tabletReplica1_100, tabletRdonly1_102}, + positions: []mysql.Position{positionMostAdvanced, positionMostAdvanced, positionMostAdvanced}, + sortedTablets: []*topodatapb.Tablet{tabletReplica1_100, tabletRdonly1_102, nil}, }, { name: "ordering by position", tablets: []*topodatapb.Tablet{tabletReplica1_101, tabletReplica2_100, tabletReplica1_100, tabletRdonly1_102}, positions: []mysql.Position{positionEmpty, positionIntermediate1, positionIntermediate2, positionMostAdvanced}, - idealCell: cell1, sortedTablets: []*topodatapb.Tablet{tabletRdonly1_102, tabletReplica1_100, tabletReplica2_100, tabletReplica1_101}, }, { name: "tablets and positions count error", tablets: []*topodatapb.Tablet{tabletReplica1_101, tabletReplica2_100}, positions: []mysql.Position{positionEmpty, positionIntermediate1, positionMostAdvanced}, - idealCell: cell1, containsErr: "unequal number of tablets and positions", }, { name: "promotion rule check", tablets: []*topodatapb.Tablet{tabletReplica1_101, tabletRdonly1_102}, positions: []mysql.Position{positionMostAdvanced, positionMostAdvanced}, - idealCell: cell1, sortedTablets: []*topodatapb.Tablet{tabletReplica1_101, tabletRdonly1_102}, }, { name: "mixed", tablets: []*topodatapb.Tablet{tabletReplica1_101, tabletReplica2_100, tabletReplica1_100, tabletRdonly1_102}, positions: []mysql.Position{positionEmpty, positionIntermediate1, positionMostAdvanced, positionIntermediate1}, - idealCell: cell1, - sortedTablets: []*topodatapb.Tablet{tabletReplica1_100, tabletRdonly1_102, tabletReplica2_100, tabletReplica1_101}, + sortedTablets: []*topodatapb.Tablet{tabletReplica1_100, tabletReplica2_100, tabletRdonly1_102, tabletReplica1_101}, }, } @@ -132,7 +126,7 @@ func TestErsSorter(t *testing.T) { require.NoError(t, err) for _, testcase := range testcases { t.Run(testcase.name, func(t *testing.T) { - err := sortTabletsForERS(testcase.tablets, testcase.positions, testcase.idealCell) + err := sortTabletsForERS(testcase.tablets, testcase.positions) if testcase.containsErr != "" { require.EqualError(t, err, testcase.containsErr) } else { diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 9d2a17006cb..c2d80ca5f5d 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -178,22 +178,46 @@ func restrictValidCandidates(validCandidates map[string]mysql.Position, tabletMa return restrictedValidCandidates, nil } -func findPossibleCandidateFromListWithRestrictions( +func findCandidateSameCell( newPrimary *topodatapb.Tablet, prevPrimary *topodatapb.Tablet, possibleCandidates []*topodatapb.Tablet, - checkEqualPrimary bool, - checkSameCell bool, ) *topodatapb.Tablet { + // check whether the one we have selected as the source is in the same cell and belongs to the candidate list provided for _, candidate := range possibleCandidates { - if checkEqualPrimary && !(topoproto.TabletAliasEqual(newPrimary.Alias, candidate.Alias)) { + if !(topoproto.TabletAliasEqual(newPrimary.Alias, candidate.Alias)) { continue } - if checkSameCell && prevPrimary != nil && !(prevPrimary.Alias.Cell == candidate.Alias.Cell) { + if prevPrimary != nil && !(prevPrimary.Alias.Cell == candidate.Alias.Cell) { continue } return candidate } + // check whether there is some other tablet in the same cell belonging to the candidate list provided + for _, candidate := range possibleCandidates { + if prevPrimary != nil && !(prevPrimary.Alias.Cell == candidate.Alias.Cell) { + continue + } + return candidate + } + return nil +} + +func findCandidateAnyCell( + newPrimary *topodatapb.Tablet, + possibleCandidates []*topodatapb.Tablet, +) *topodatapb.Tablet { + // check whether the one we have selected as the source belongs to the candidate list provided + for _, candidate := range possibleCandidates { + if !(topoproto.TabletAliasEqual(newPrimary.Alias, candidate.Alias)) { + continue + } + return candidate + } + // return the first candidate from this list, if it isn't empty + if len(possibleCandidates) > 0 { + return possibleCandidates[0] + } return nil } From 9d8456cc62dfd1ef7c5e416fba5056cf49bfb775 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 29 Sep 2021 22:58:10 +0530 Subject: [PATCH 172/176] add new flag to the usage as well Signed-off-by: Manan Gupta --- go/vt/vtctl/reparent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtctl/reparent.go b/go/vt/vtctl/reparent.go index ded144f7ba4..90c1de52ef5 100644 --- a/go/vt/vtctl/reparent.go +++ b/go/vt/vtctl/reparent.go @@ -54,7 +54,7 @@ func init() { addCommand("Shards", command{ "EmergencyReparentShard", commandEmergencyReparentShard, - "-keyspace_shard= [-new_primary=] [-wait_replicas_timeout=] [-ignore_replicas=]", + "-keyspace_shard= [-new_primary=] [-wait_replicas_timeout=] [-ignore_replicas=] [-prevent_cross_cell_promotion=]", "Reparents the shard to the new primary. Assumes the old primary is dead and not responding."}) addCommand("Shards", command{ "TabletExternallyReparented", From 653efb3392f18761d8bb9f640b5fae494e2e4aaa Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 29 Sep 2021 23:11:31 +0530 Subject: [PATCH 173/176] add comments to other boolean literals Signed-off-by: Manan Gupta --- go/vt/orchestrator/http/api.go | 2 +- go/vt/orchestrator/logic/command_applier.go | 2 +- go/vt/orchestrator/logic/orchestrator.go | 2 +- go/vt/orchestrator/logic/tablet_discovery.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/vt/orchestrator/http/api.go b/go/vt/orchestrator/http/api.go index b8c884c18fb..15278508bfc 100644 --- a/go/vt/orchestrator/http/api.go +++ b/go/vt/orchestrator/http/api.go @@ -217,7 +217,7 @@ func (this *HttpAPI) Discover(params martini.Params, r render.Render, req *http. if orcraft.IsRaftEnabled() { orcraft.PublishCommand("discover", instanceKey) } else { - logic.DiscoverInstance(instanceKey, false) + logic.DiscoverInstance(instanceKey, false /* forceDiscovery */) } Respond(r, &APIResponse{Code: OK, Message: fmt.Sprintf("Instance discovered: %+v", instance.Key), Details: instance}) diff --git a/go/vt/orchestrator/logic/command_applier.go b/go/vt/orchestrator/logic/command_applier.go index f56a101135f..25e1968984e 100644 --- a/go/vt/orchestrator/logic/command_applier.go +++ b/go/vt/orchestrator/logic/command_applier.go @@ -93,7 +93,7 @@ func (applier *CommandApplier) discover(value []byte) interface{} { if err := json.Unmarshal(value, &instanceKey); err != nil { return log.Errore(err) } - DiscoverInstance(instanceKey, false) + DiscoverInstance(instanceKey, false /* forceDiscovery */) return nil } diff --git a/go/vt/orchestrator/logic/orchestrator.go b/go/vt/orchestrator/logic/orchestrator.go index 74def388579..2d2138c4ce9 100644 --- a/go/vt/orchestrator/logic/orchestrator.go +++ b/go/vt/orchestrator/logic/orchestrator.go @@ -171,7 +171,7 @@ func handleDiscoveryRequests() { continue } - DiscoverInstance(instanceKey, false) + DiscoverInstance(instanceKey, false /* forceDiscovery */) discoveryQueue.Release(instanceKey) } }() diff --git a/go/vt/orchestrator/logic/tablet_discovery.go b/go/vt/orchestrator/logic/tablet_discovery.go index 25a41729aad..8b0ee13221e 100644 --- a/go/vt/orchestrator/logic/tablet_discovery.go +++ b/go/vt/orchestrator/logic/tablet_discovery.go @@ -61,7 +61,7 @@ func OpenTabletDiscovery() <-chan time.Time { } refreshTabletsUsing(func(instanceKey *inst.InstanceKey) { _ = inst.InjectSeed(instanceKey) - }, false) + }, false /* forceRefresh */) // TODO(sougou): parameterize poll interval. return time.Tick(15 * time.Second) //nolint SA1015: using time.Tick leaks the underlying ticker } From 1f7411144cc05b7521f7cd46584a0a6344f48d39 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 8 Oct 2021 21:17:07 +0530 Subject: [PATCH 174/176] rename some structs and functions for better readability Signed-off-by: Manan Gupta --- go/vt/orchestrator/inst/analysis_dao.go | 4 +-- go/vt/orchestrator/inst/durability.go | 6 ++-- .../inst/instance_topology_dao.go | 3 ++ go/vt/orchestrator/logic/topology_recovery.go | 4 +-- go/vt/vtctl/reparentutil/durability.go | 16 ++++----- go/vt/vtctl/reparentutil/durability_test.go | 6 ++-- ...r.go => ers_intermediate_source_sorter.go} | 34 +++++++++---------- ...=> ers_intermediate_source_sorter_test.go} | 4 +-- 8 files changed, 40 insertions(+), 37 deletions(-) rename go/vt/vtctl/reparentutil/{ers_sorter.go => ers_intermediate_source_sorter.go} (63%) rename go/vt/vtctl/reparentutil/{ers_sorter_test.go => ers_intermediate_source_sorter_test.go} (97%) diff --git a/go/vt/orchestrator/inst/analysis_dao.go b/go/vt/orchestrator/inst/analysis_dao.go index 57d3cfd86f3..679b6dcb858 100644 --- a/go/vt/orchestrator/inst/analysis_dao.go +++ b/go/vt/orchestrator/inst/analysis_dao.go @@ -504,11 +504,11 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints) a.Analysis = PrimaryIsReadOnly a.Description = "Primary is read-only" // - } else if a.IsClusterPrimary && reparentutil.PrimarySemiSyncFromTablet(tablet) != 0 && !a.SemiSyncPrimaryEnabled { + } else if a.IsClusterPrimary && reparentutil.SemiSyncAckersFromTablet(tablet) != 0 && !a.SemiSyncPrimaryEnabled { a.Analysis = PrimarySemiSyncMustBeSet a.Description = "Primary semi-sync must be set" // - } else if a.IsClusterPrimary && reparentutil.PrimarySemiSyncFromTablet(tablet) == 0 && a.SemiSyncPrimaryEnabled { + } else if a.IsClusterPrimary && reparentutil.SemiSyncAckersFromTablet(tablet) == 0 && a.SemiSyncPrimaryEnabled { a.Analysis = PrimarySemiSyncMustNotBeSet a.Description = "Primary semi-sync must not be set" // diff --git a/go/vt/orchestrator/inst/durability.go b/go/vt/orchestrator/inst/durability.go index 4307d228904..c104a72f8a3 100644 --- a/go/vt/orchestrator/inst/durability.go +++ b/go/vt/orchestrator/inst/durability.go @@ -33,12 +33,12 @@ func ReplicaSemiSync(primaryKey, replicaKey InstanceKey) bool { return reparentutil.ReplicaSemiSyncFromTablet(primary, replica) } -// PrimarySemiSync returns the primary semi-sync setting for the instance. +// SemiSyncAckers returns the primary semi-sync setting for the instance. // 0 means none. Non-zero specifies the number of required ackers. -func PrimarySemiSync(instanceKey InstanceKey) int { +func SemiSyncAckers(instanceKey InstanceKey) int { primary, err := ReadTablet(instanceKey) if err != nil { return 0 } - return reparentutil.PrimarySemiSyncFromTablet(primary) + return reparentutil.SemiSyncAckersFromTablet(primary) } diff --git a/go/vt/orchestrator/inst/instance_topology_dao.go b/go/vt/orchestrator/inst/instance_topology_dao.go index 9e9538e8b22..d2405f688f4 100644 --- a/go/vt/orchestrator/inst/instance_topology_dao.go +++ b/go/vt/orchestrator/inst/instance_topology_dao.go @@ -361,6 +361,9 @@ func StopReplicas(replicas [](*Instance), stopReplicationMethod StopReplicationM // StopReplicasNicely will attemt to stop all given replicas nicely, up to timeout func StopReplicasNicely(replicas [](*Instance), timeout time.Duration) [](*Instance) { stoppedReplicas := StopReplicas(replicas, StopReplicationNice, timeout) + // We remove nil instances because StopReplicas might introduce nils in the array that it returns in case of + // failures while reading the tablet from the backend. This could happen when the tablet is forgotten while we are + // trying to stop the replication on the tablets. stoppedReplicas = RemoveNilInstances(stoppedReplicas) return stoppedReplicas } diff --git a/go/vt/orchestrator/logic/topology_recovery.go b/go/vt/orchestrator/logic/topology_recovery.go index cdfa494152d..765695bd74d 100644 --- a/go/vt/orchestrator/logic/topology_recovery.go +++ b/go/vt/orchestrator/logic/topology_recovery.go @@ -1878,7 +1878,7 @@ func electNewPrimary(analysisEntry inst.ReplicationAnalysis, candidateInstanceKe return false, topologyRecovery, err } } - count := inst.PrimarySemiSync(candidate.Key) + count := inst.SemiSyncAckers(candidate.Key) err = inst.SetSemiSyncPrimary(&candidate.Key, count > 0) AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("- electNewPrimary: applying semi-sync %v: success=%t", count > 0, (err == nil))) if err != nil { @@ -1941,7 +1941,7 @@ func fixPrimary(analysisEntry inst.ReplicationAnalysis, candidateInstanceKey *in defer unlock(&err) // TODO(sougou): this code pattern has reached DRY limits. Reuse. - count := inst.PrimarySemiSync(analysisEntry.AnalyzedInstanceKey) + count := inst.SemiSyncAckers(analysisEntry.AnalyzedInstanceKey) err = inst.SetSemiSyncPrimary(&analysisEntry.AnalyzedInstanceKey, count > 0) //AuditTopologyRecovery(topologyRecovery, fmt.Sprintf("- fixPrimary: applying semi-sync %v: success=%t", count > 0, (err == nil))) if err != nil { diff --git a/go/vt/vtctl/reparentutil/durability.go b/go/vt/vtctl/reparentutil/durability.go index 7f4449d7573..5a12e35e682 100644 --- a/go/vt/vtctl/reparentutil/durability.go +++ b/go/vt/vtctl/reparentutil/durability.go @@ -57,7 +57,7 @@ func init() { // durabler is the interface which is used to get the promotion rules for candidates and the semi sync setup type durabler interface { promotionRule(*topodatapb.Tablet) promotionrule.CandidatePromotionRule - primarySemiSync(*topodatapb.Tablet) int + semiSyncAckers(*topodatapb.Tablet) int replicaSemiSync(primary, replica *topodatapb.Tablet) bool } @@ -86,10 +86,10 @@ func PromotionRule(tablet *topodatapb.Tablet) promotionrule.CandidatePromotionRu return curDurabilityPolicy.promotionRule(tablet) } -// PrimarySemiSyncFromTablet returns the primary semi-sync setting for the instance. +// SemiSyncAckersFromTablet returns the primary semi-sync setting for the instance. // 0 means none. Non-zero specifies the number of required ackers. -func PrimarySemiSyncFromTablet(tablet *topodatapb.Tablet) int { - return curDurabilityPolicy.primarySemiSync(tablet) +func SemiSyncAckersFromTablet(tablet *topodatapb.Tablet) int { + return curDurabilityPolicy.semiSyncAckers(tablet) } // ReplicaSemiSyncFromTablet returns the replica semi-sync setting from the tablet record. @@ -111,7 +111,7 @@ func (d *durabilityNone) promotionRule(tablet *topodatapb.Tablet) promotionrule. return promotionrule.MustNot } -func (d *durabilityNone) primarySemiSync(tablet *topodatapb.Tablet) int { +func (d *durabilityNone) semiSyncAckers(tablet *topodatapb.Tablet) int { return 0 } @@ -133,7 +133,7 @@ func (d *durabilitySemiSync) promotionRule(tablet *topodatapb.Tablet) promotionr return promotionrule.MustNot } -func (d *durabilitySemiSync) primarySemiSync(tablet *topodatapb.Tablet) int { +func (d *durabilitySemiSync) semiSyncAckers(tablet *topodatapb.Tablet) int { return 1 } @@ -160,7 +160,7 @@ func (d *durabilityCrossCell) promotionRule(tablet *topodatapb.Tablet) promotion return promotionrule.MustNot } -func (d *durabilityCrossCell) primarySemiSync(tablet *topodatapb.Tablet) int { +func (d *durabilityCrossCell) semiSyncAckers(tablet *topodatapb.Tablet) int { return 1 } @@ -197,7 +197,7 @@ func (d *durabilitySpecified) promotionRule(tablet *topodatapb.Tablet) promotion return promotionrule.MustNot } -func (d *durabilitySpecified) primarySemiSync(tablet *topodatapb.Tablet) int { +func (d *durabilitySpecified) semiSyncAckers(tablet *topodatapb.Tablet) int { return 0 } diff --git a/go/vt/vtctl/reparentutil/durability_test.go b/go/vt/vtctl/reparentutil/durability_test.go index 58740d1e600..4ae5b238c49 100644 --- a/go/vt/vtctl/reparentutil/durability_test.go +++ b/go/vt/vtctl/reparentutil/durability_test.go @@ -51,7 +51,7 @@ func TestDurabilityNone(t *testing.T) { Type: topodatapb.TabletType_SPARE, }) assert.Equal(t, promotionrule.MustNot, promoteRule) - assert.Equal(t, 0, PrimarySemiSyncFromTablet(nil)) + assert.Equal(t, 0, SemiSyncAckersFromTablet(nil)) assert.Equal(t, false, ReplicaSemiSyncFromTablet(nil, nil)) } @@ -78,7 +78,7 @@ func TestDurabilitySemiSync(t *testing.T) { Type: topodatapb.TabletType_SPARE, }) assert.Equal(t, promotionrule.MustNot, promoteRule) - assert.Equal(t, 1, PrimarySemiSyncFromTablet(nil)) + assert.Equal(t, 1, SemiSyncAckersFromTablet(nil)) assert.Equal(t, true, ReplicaSemiSyncFromTablet(nil, &topodatapb.Tablet{ Type: topodatapb.TabletType_REPLICA, })) @@ -110,7 +110,7 @@ func TestDurabilityCrossCell(t *testing.T) { Type: topodatapb.TabletType_SPARE, }) assert.Equal(t, promotionrule.MustNot, promoteRule) - assert.Equal(t, 1, PrimarySemiSyncFromTablet(nil)) + assert.Equal(t, 1, SemiSyncAckersFromTablet(nil)) assert.Equal(t, false, ReplicaSemiSyncFromTablet(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, Alias: &topodatapb.TabletAlias{ diff --git a/go/vt/vtctl/reparentutil/ers_sorter.go b/go/vt/vtctl/reparentutil/ers_intermediate_source_sorter.go similarity index 63% rename from go/vt/vtctl/reparentutil/ers_sorter.go rename to go/vt/vtctl/reparentutil/ers_intermediate_source_sorter.go index 8a93178cdcd..f625ec7f306 100644 --- a/go/vt/vtctl/reparentutil/ers_sorter.go +++ b/go/vt/vtctl/reparentutil/ers_intermediate_source_sorter.go @@ -26,57 +26,57 @@ import ( vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) -// ERSSorter sorts tablets by GTID positions and Promotion rules aimed at finding the best +// ERSIntermediateSourceSorter sorts tablets by GTID positions and Promotion rules aimed at finding the best // candidate for intermediate promotion in emergency reparent shard -type ERSSorter struct { +type ERSIntermediateSourceSorter struct { tablets []*topodatapb.Tablet positions []mysql.Position } -// NewERSSorter creates a new ERSSorter -func NewERSSorter(tablets []*topodatapb.Tablet, positions []mysql.Position) *ERSSorter { - return &ERSSorter{ +// NewERSIntermediateSourceSorter creates a new ERSIntermediateSourceSorter +func NewERSIntermediateSourceSorter(tablets []*topodatapb.Tablet, positions []mysql.Position) *ERSIntermediateSourceSorter { + return &ERSIntermediateSourceSorter{ tablets: tablets, positions: positions, } } // Len implements the Interface for sorting -func (ersSorter *ERSSorter) Len() int { return len(ersSorter.tablets) } +func (ersISSorter *ERSIntermediateSourceSorter) Len() int { return len(ersISSorter.tablets) } // Swap implements the Interface for sorting -func (ersSorter *ERSSorter) Swap(i, j int) { - ersSorter.tablets[i], ersSorter.tablets[j] = ersSorter.tablets[j], ersSorter.tablets[i] - ersSorter.positions[i], ersSorter.positions[j] = ersSorter.positions[j], ersSorter.positions[i] +func (ersISSorter *ERSIntermediateSourceSorter) Swap(i, j int) { + ersISSorter.tablets[i], ersISSorter.tablets[j] = ersISSorter.tablets[j], ersISSorter.tablets[i] + ersISSorter.positions[i], ersISSorter.positions[j] = ersISSorter.positions[j], ersISSorter.positions[i] } // Less implements the Interface for sorting -func (ersSorter *ERSSorter) Less(i, j int) bool { +func (ersISSorter *ERSIntermediateSourceSorter) Less(i, j int) bool { // Returning "true" in this function means [i] is before [j] in the sorting order, // which will lead to [i] be a better candidate for promotion // Should not happen // fail-safe code - if ersSorter.tablets[i] == nil { + if ersISSorter.tablets[i] == nil { return false } - if ersSorter.tablets[j] == nil { + if ersISSorter.tablets[j] == nil { return true } - if !ersSorter.positions[i].AtLeast(ersSorter.positions[j]) { + if !ersISSorter.positions[i].AtLeast(ersISSorter.positions[j]) { // [i] does not have all GTIDs that [j] does return false } - if !ersSorter.positions[j].AtLeast(ersSorter.positions[i]) { + if !ersISSorter.positions[j].AtLeast(ersISSorter.positions[i]) { // [j] does not have all GTIDs that [i] does return true } // at this point, both have the same GTIDs // so we check their promotion rules - jPromotionRule := PromotionRule(ersSorter.tablets[j]) - iPromotionRule := PromotionRule(ersSorter.tablets[i]) + jPromotionRule := PromotionRule(ersISSorter.tablets[j]) + iPromotionRule := PromotionRule(ersISSorter.tablets[i]) return !jPromotionRule.BetterThan(iPromotionRule) } @@ -88,6 +88,6 @@ func sortTabletsForERS(tablets []*topodatapb.Tablet, positions []mysql.Position) return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unequal number of tablets and positions") } - sort.Sort(NewERSSorter(tablets, positions)) + sort.Sort(NewERSIntermediateSourceSorter(tablets, positions)) return nil } diff --git a/go/vt/vtctl/reparentutil/ers_sorter_test.go b/go/vt/vtctl/reparentutil/ers_intermediate_source_sorter_test.go similarity index 97% rename from go/vt/vtctl/reparentutil/ers_sorter_test.go rename to go/vt/vtctl/reparentutil/ers_intermediate_source_sorter_test.go index 4d527675048..9c731371026 100644 --- a/go/vt/vtctl/reparentutil/ers_sorter_test.go +++ b/go/vt/vtctl/reparentutil/ers_intermediate_source_sorter_test.go @@ -25,8 +25,8 @@ import ( topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) -// TestErsSorter tests that the sorting for ERS works correctly -func TestErsSorter(t *testing.T) { +// TestErsInterMediateSourceSorter tests that the sorting for ERS works correctly +func TestErsInterMediateSourceSorter(t *testing.T) { sid1 := mysql.SID{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} sid2 := mysql.SID{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16} cell1 := "cell1" From da23be50f9eda07b0ac36fc384dc59e1da4f48b1 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 11 Oct 2021 11:51:27 +0530 Subject: [PATCH 175/176] added mutex for protecting concurrent access to the durability policies Signed-off-by: Manan Gupta --- go/vt/vtctl/reparentutil/durability.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/go/vt/vtctl/reparentutil/durability.go b/go/vt/vtctl/reparentutil/durability.go index 5a12e35e682..e6e58cb2afb 100644 --- a/go/vt/vtctl/reparentutil/durability.go +++ b/go/vt/vtctl/reparentutil/durability.go @@ -18,6 +18,7 @@ package reparentutil import ( "fmt" + "sync" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo/topoproto" @@ -38,6 +39,8 @@ var ( durabilityPolicies = make(map[string]newDurabler) // curDurabilityPolicy is the current durability policy in use curDurabilityPolicy durabler + // curDurabilityPolicyMutex is the mutex protecting the curDurabilityPolicy variable + curDurabilityPolicyMutex sync.Mutex ) func init() { @@ -77,24 +80,32 @@ func SetDurabilityPolicy(name string, durabilityParams map[string]string) error return fmt.Errorf("durability policy %v not found", name) } log.Infof("Setting durability policy to %v", name) + curDurabilityPolicyMutex.Lock() + defer curDurabilityPolicyMutex.Unlock() curDurabilityPolicy = newDurabilityCreationFunc(durabilityParams) return nil } // PromotionRule returns the promotion rule for the instance. func PromotionRule(tablet *topodatapb.Tablet) promotionrule.CandidatePromotionRule { + curDurabilityPolicyMutex.Lock() + defer curDurabilityPolicyMutex.Unlock() return curDurabilityPolicy.promotionRule(tablet) } // SemiSyncAckersFromTablet returns the primary semi-sync setting for the instance. // 0 means none. Non-zero specifies the number of required ackers. func SemiSyncAckersFromTablet(tablet *topodatapb.Tablet) int { + curDurabilityPolicyMutex.Lock() + defer curDurabilityPolicyMutex.Unlock() return curDurabilityPolicy.semiSyncAckers(tablet) } // ReplicaSemiSyncFromTablet returns the replica semi-sync setting from the tablet record. // Prefer using this function if tablet record is available. func ReplicaSemiSyncFromTablet(primary, replica *topodatapb.Tablet) bool { + curDurabilityPolicyMutex.Lock() + defer curDurabilityPolicyMutex.Unlock() return curDurabilityPolicy.replicaSemiSync(primary, replica) } From 1332dc6fb612ac5901e68bc76b31294bf0485152 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 11 Oct 2021 22:21:06 +0530 Subject: [PATCH 176/176] rename 2 functions Signed-off-by: Manan Gupta --- go/vt/orchestrator/inst/analysis_dao.go | 8 ++++---- go/vt/orchestrator/inst/durability.go | 4 ++-- go/vt/vtctl/reparentutil/durability.go | 8 ++++---- go/vt/vtctl/reparentutil/durability_test.go | 18 +++++++++--------- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/go/vt/orchestrator/inst/analysis_dao.go b/go/vt/orchestrator/inst/analysis_dao.go index 679b6dcb858..487af0d0dde 100644 --- a/go/vt/orchestrator/inst/analysis_dao.go +++ b/go/vt/orchestrator/inst/analysis_dao.go @@ -504,11 +504,11 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints) a.Analysis = PrimaryIsReadOnly a.Description = "Primary is read-only" // - } else if a.IsClusterPrimary && reparentutil.SemiSyncAckersFromTablet(tablet) != 0 && !a.SemiSyncPrimaryEnabled { + } else if a.IsClusterPrimary && reparentutil.SemiSyncAckers(tablet) != 0 && !a.SemiSyncPrimaryEnabled { a.Analysis = PrimarySemiSyncMustBeSet a.Description = "Primary semi-sync must be set" // - } else if a.IsClusterPrimary && reparentutil.SemiSyncAckersFromTablet(tablet) == 0 && a.SemiSyncPrimaryEnabled { + } else if a.IsClusterPrimary && reparentutil.SemiSyncAckers(tablet) == 0 && a.SemiSyncPrimaryEnabled { a.Analysis = PrimarySemiSyncMustNotBeSet a.Description = "Primary semi-sync must not be set" // @@ -532,11 +532,11 @@ func GetReplicationAnalysis(clusterName string, hints *ReplicationAnalysisHints) a.Analysis = ReplicationStopped a.Description = "Replication is stopped" // - } else if topo.IsReplicaType(a.TabletType) && !a.IsPrimary && reparentutil.ReplicaSemiSyncFromTablet(primaryTablet, tablet) && !a.SemiSyncReplicaEnabled { + } else if topo.IsReplicaType(a.TabletType) && !a.IsPrimary && reparentutil.ReplicaSemiSync(primaryTablet, tablet) && !a.SemiSyncReplicaEnabled { a.Analysis = ReplicaSemiSyncMustBeSet a.Description = "Replica semi-sync must be set" // - } else if topo.IsReplicaType(a.TabletType) && !a.IsPrimary && !reparentutil.ReplicaSemiSyncFromTablet(primaryTablet, tablet) && a.SemiSyncReplicaEnabled { + } else if topo.IsReplicaType(a.TabletType) && !a.IsPrimary && !reparentutil.ReplicaSemiSync(primaryTablet, tablet) && a.SemiSyncReplicaEnabled { a.Analysis = ReplicaSemiSyncMustNotBeSet a.Description = "Replica semi-sync must not be set" // diff --git a/go/vt/orchestrator/inst/durability.go b/go/vt/orchestrator/inst/durability.go index c104a72f8a3..547ae03916f 100644 --- a/go/vt/orchestrator/inst/durability.go +++ b/go/vt/orchestrator/inst/durability.go @@ -30,7 +30,7 @@ func ReplicaSemiSync(primaryKey, replicaKey InstanceKey) bool { if err != nil { return false } - return reparentutil.ReplicaSemiSyncFromTablet(primary, replica) + return reparentutil.ReplicaSemiSync(primary, replica) } // SemiSyncAckers returns the primary semi-sync setting for the instance. @@ -40,5 +40,5 @@ func SemiSyncAckers(instanceKey InstanceKey) int { if err != nil { return 0 } - return reparentutil.SemiSyncAckersFromTablet(primary) + return reparentutil.SemiSyncAckers(primary) } diff --git a/go/vt/vtctl/reparentutil/durability.go b/go/vt/vtctl/reparentutil/durability.go index e6e58cb2afb..8b80cc5cf9f 100644 --- a/go/vt/vtctl/reparentutil/durability.go +++ b/go/vt/vtctl/reparentutil/durability.go @@ -93,17 +93,17 @@ func PromotionRule(tablet *topodatapb.Tablet) promotionrule.CandidatePromotionRu return curDurabilityPolicy.promotionRule(tablet) } -// SemiSyncAckersFromTablet returns the primary semi-sync setting for the instance. +// SemiSyncAckers returns the primary semi-sync setting for the instance. // 0 means none. Non-zero specifies the number of required ackers. -func SemiSyncAckersFromTablet(tablet *topodatapb.Tablet) int { +func SemiSyncAckers(tablet *topodatapb.Tablet) int { curDurabilityPolicyMutex.Lock() defer curDurabilityPolicyMutex.Unlock() return curDurabilityPolicy.semiSyncAckers(tablet) } -// ReplicaSemiSyncFromTablet returns the replica semi-sync setting from the tablet record. +// ReplicaSemiSync returns the replica semi-sync setting from the tablet record. // Prefer using this function if tablet record is available. -func ReplicaSemiSyncFromTablet(primary, replica *topodatapb.Tablet) bool { +func ReplicaSemiSync(primary, replica *topodatapb.Tablet) bool { curDurabilityPolicyMutex.Lock() defer curDurabilityPolicyMutex.Unlock() return curDurabilityPolicy.replicaSemiSync(primary, replica) diff --git a/go/vt/vtctl/reparentutil/durability_test.go b/go/vt/vtctl/reparentutil/durability_test.go index 4ae5b238c49..f862b7b6cbe 100644 --- a/go/vt/vtctl/reparentutil/durability_test.go +++ b/go/vt/vtctl/reparentutil/durability_test.go @@ -51,8 +51,8 @@ func TestDurabilityNone(t *testing.T) { Type: topodatapb.TabletType_SPARE, }) assert.Equal(t, promotionrule.MustNot, promoteRule) - assert.Equal(t, 0, SemiSyncAckersFromTablet(nil)) - assert.Equal(t, false, ReplicaSemiSyncFromTablet(nil, nil)) + assert.Equal(t, 0, SemiSyncAckers(nil)) + assert.Equal(t, false, ReplicaSemiSync(nil, nil)) } func TestDurabilitySemiSync(t *testing.T) { @@ -78,11 +78,11 @@ func TestDurabilitySemiSync(t *testing.T) { Type: topodatapb.TabletType_SPARE, }) assert.Equal(t, promotionrule.MustNot, promoteRule) - assert.Equal(t, 1, SemiSyncAckersFromTablet(nil)) - assert.Equal(t, true, ReplicaSemiSyncFromTablet(nil, &topodatapb.Tablet{ + assert.Equal(t, 1, SemiSyncAckers(nil)) + assert.Equal(t, true, ReplicaSemiSync(nil, &topodatapb.Tablet{ Type: topodatapb.TabletType_REPLICA, })) - assert.Equal(t, false, ReplicaSemiSyncFromTablet(nil, &topodatapb.Tablet{ + assert.Equal(t, false, ReplicaSemiSync(nil, &topodatapb.Tablet{ Type: topodatapb.TabletType_EXPERIMENTAL, })) } @@ -110,8 +110,8 @@ func TestDurabilityCrossCell(t *testing.T) { Type: topodatapb.TabletType_SPARE, }) assert.Equal(t, promotionrule.MustNot, promoteRule) - assert.Equal(t, 1, SemiSyncAckersFromTablet(nil)) - assert.Equal(t, false, ReplicaSemiSyncFromTablet(&topodatapb.Tablet{ + assert.Equal(t, 1, SemiSyncAckers(nil)) + assert.Equal(t, false, ReplicaSemiSync(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, Alias: &topodatapb.TabletAlias{ Cell: "cell1", @@ -122,7 +122,7 @@ func TestDurabilityCrossCell(t *testing.T) { Cell: "cell1", }, })) - assert.Equal(t, true, ReplicaSemiSyncFromTablet(&topodatapb.Tablet{ + assert.Equal(t, true, ReplicaSemiSync(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, Alias: &topodatapb.TabletAlias{ Cell: "cell1", @@ -133,7 +133,7 @@ func TestDurabilityCrossCell(t *testing.T) { Cell: "cell2", }, })) - assert.Equal(t, false, ReplicaSemiSyncFromTablet(&topodatapb.Tablet{ + assert.Equal(t, false, ReplicaSemiSync(&topodatapb.Tablet{ Type: topodatapb.TabletType_PRIMARY, Alias: &topodatapb.TabletAlias{ Cell: "cell1",