Skip to content

Commit

Permalink
functional/tester: add TODO for member remove fail
Browse files Browse the repository at this point in the history
Signed-off-by: Gyuho Lee <[email protected]>
  • Loading branch information
gyuho committed Apr 11, 2018
1 parent 15791d7 commit d7bf471
Showing 1 changed file with 22 additions and 3 deletions.
25 changes: 22 additions & 3 deletions functional/tester/failure_case_sigquit_remove.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,15 @@ func inject_SIGQUIT_ETCD_AND_REMOVE_DATA(clus *Cluster, idx1 int) error {
id1 := sresp.Header.MemberId
is1 := fmt.Sprintf("%016x", id1)

clus.lg.Info(
"disastrous machine failure START",
zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint),
zap.String("target-member-id", is1),
zap.Error(err),
)
err = clus.sendOp(idx1, rpcpb.Operation_SIGQUIT_ETCD_AND_REMOVE_DATA)
clus.lg.Info(
"disastrous machine failure",
"disastrous machine failure END",
zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint),
zap.String("target-member-id", is1),
zap.Error(err),
Expand All @@ -78,9 +84,22 @@ func inject_SIGQUIT_ETCD_AND_REMOVE_DATA(clus *Cluster, idx1 int) error {
}
defer cli2.Close()

_, err = cli2.MemberRemove(context.Background(), id1)
// FIXME(bug): this may block forever during
// "SIGQUIT_AND_REMOVE_LEADER_UNTIL_TRIGGER_SNAPSHOT"
// is the new leader too busy with snapshotting?
// is raft proposal dropped?
// enable client keepalive for failover?
clus.lg.Info(
"member remove after disaster START",
zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint),
zap.String("target-member-id", is1),
zap.String("request-to", clus.Members[idx2].EtcdClientEndpoint),
)
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
_, err = cli2.MemberRemove(ctx, id1)
cancel()
clus.lg.Info(
"member remove after disaster",
"member remove after disaster END",
zap.String("target-endpoint", clus.Members[idx1].EtcdClientEndpoint),
zap.String("target-member-id", is1),
zap.String("request-to", clus.Members[idx2].EtcdClientEndpoint),
Expand Down

0 comments on commit d7bf471

Please sign in to comment.