diff --git a/pkg/kv/kvserver/replica_store_liveness.go b/pkg/kv/kvserver/replica_store_liveness.go index b900027732fa..8fb0891af80f 100644 --- a/pkg/kv/kvserver/replica_store_liveness.go +++ b/pkg/kv/kvserver/replica_store_liveness.go @@ -45,6 +45,11 @@ func (r *replicaRLockedStoreLiveness) SupportFor(replicaID raftpb.PeerID) (raftp if !ok { return 0, false } + // TODO(arul): we can remove this once we start to assign storeLiveness in the + // Store constructor. + if r.store.storeLiveness == nil { + return 0, false + } epoch, ok := r.store.storeLiveness.SupportFor(storeID) if !ok { return 0, false diff --git a/pkg/raft/node_test.go b/pkg/raft/node_test.go index dd574168283e..69299ac96ef0 100644 --- a/pkg/raft/node_test.go +++ b/pkg/raft/node_test.go @@ -453,13 +453,13 @@ func TestNodeStart(t *testing.T) { MustSync: true, }, { - HardState: raftpb.HardState{Term: 2, Commit: 2, Vote: 1, Lead: 1}, + HardState: raftpb.HardState{Term: 2, Commit: 2, Vote: 1, Lead: 1, LeadEpoch: 1}, Entries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}}, CommittedEntries: []raftpb.Entry{{Term: 2, Index: 2, Data: nil}}, MustSync: true, }, { - HardState: raftpb.HardState{Term: 2, Commit: 3, Vote: 1, Lead: 1}, + HardState: raftpb.HardState{Term: 2, Commit: 3, Vote: 1, Lead: 1, LeadEpoch: 1}, Entries: nil, CommittedEntries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}}, MustSync: false, diff --git a/pkg/raft/raft.go b/pkg/raft/raft.go index 3c1effecab98..7372460262a2 100644 --- a/pkg/raft/raft.go +++ b/pkg/raft/raft.go @@ -518,19 +518,22 @@ func (r *raft) send(m pb.Message) { m.Term = r.Term } } - if m.Type == pb.MsgAppResp || m.Type == pb.MsgVoteResp || m.Type == pb.MsgPreVoteResp { + if m.Type == pb.MsgAppResp || m.Type == pb.MsgVoteResp || + m.Type == pb.MsgPreVoteResp || m.Type == pb.MsgFortifyLeaderResp { // If async storage writes are enabled, messages added to the msgs slice // are allowed to be sent out before unstable state (e.g. log entry // writes and election votes) have been durably synced to the local // disk. // - // For most message types, this is not an issue. However, response - // messages that relate to "voting" on either leader election or log - // appends require durability before they can be sent. It would be - // incorrect to publish a vote in an election before that vote has been - // synced to stable storage locally. Similarly, it would be incorrect to - // acknowledge a log append to the leader before that entry has been - // synced to stable storage locally. + // For most message types, this is not an issue. However, response messages + // that relate to "voting" on either leader election or log appends or + // messages that relate to leader fortification require durability before + // they can be sent. It would be incorrect to publish a vote in an election + // before that vote has been synced to stable storage locally. Similarly, it + // would be incorrect to acknowledge a log append to the leader before that + // entry has been synced to stable storage locally. Similarly, it would also + // be incorrect to promise fortification support to a leader without durably + // persisting the leader's epoch being supported. // // Per the Raft thesis, section 3.8 Persisted state and server restarts: // @@ -683,7 +686,16 @@ func (r *raft) sendFortify(to pb.PeerID) { if to == r.id { // We handle the case where the leader is trying to fortify itself specially. // Doing so avoids a self-addressed message. - // TODO(arul): do this handling. + epoch, live := r.storeLiveness.SupportFor(r.lead) + if live { + r.leadEpoch = epoch + // TODO(arul): For now, we're not recording any support on the leader. Do + // this once we implement handleFortifyResp correctly. + } else { + r.logger.Infof( + "%x leader at term %d does not support itself in the liveness fabric", r.id, r.Term, + ) + } return } r.send(pb.Message{To: to, Type: pb.MsgFortifyLeader}) @@ -1701,6 +1713,8 @@ func stepFollower(r *raft, m pb.Message) error { r.lead = m.From r.handleSnapshot(m) case pb.MsgFortifyLeader: + r.electionElapsed = 0 + r.lead = m.From r.handleFortify(m) case pb.MsgTransferLeader: if r.lead == None { @@ -1868,7 +1882,22 @@ func (r *raft) handleSnapshot(m pb.Message) { } func (r *raft) handleFortify(m pb.Message) { - // TODO(arul): currently a no-op; implement. + assertTrue(r.state == StateFollower, "leaders should locally fortify without sending a message") + + epoch, live := r.storeLiveness.SupportFor(r.lead) + if !live { + // The leader isn't supported by this peer in its liveness fabric. Reject + // the fortification request. + r.send(pb.Message{ + To: m.From, + Type: pb.MsgFortifyLeaderResp, + Reject: true, + }) + return + } + r.leadEpoch = epoch + // TODO(arul): for now, we reject the fortification request because the leader + // hasn't been taught how to handle it. r.send(pb.Message{ To: m.From, Type: pb.MsgFortifyLeaderResp, diff --git a/pkg/raft/rafttest/BUILD.bazel b/pkg/raft/rafttest/BUILD.bazel index efc5d3b9752d..ec636a88a9ec 100644 --- a/pkg/raft/rafttest/BUILD.bazel +++ b/pkg/raft/rafttest/BUILD.bazel @@ -24,6 +24,7 @@ go_library( "interaction_env_handler_set_randomized_election_timeout.go", "interaction_env_handler_stabilize.go", "interaction_env_handler_status.go", + "interaction_env_handler_store_liveness.go", "interaction_env_handler_tick.go", "interaction_env_handler_transfer_leadership.go", "interaction_env_logger.go", @@ -37,7 +38,9 @@ go_library( "//pkg/raft/raftpb", "//pkg/raft/raftstoreliveness", "//pkg/raft/tracker", + "//pkg/util/hlc", "@com_github_cockroachdb_datadriven//:datadriven", + "@com_github_cockroachdb_errors//:errors", "@com_github_stretchr_testify//require", ], ) diff --git a/pkg/raft/rafttest/interaction_env.go b/pkg/raft/rafttest/interaction_env.go index 07c2acd8ee07..663803bd2545 100644 --- a/pkg/raft/rafttest/interaction_env.go +++ b/pkg/raft/rafttest/interaction_env.go @@ -25,7 +25,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/raft" pb "github.com/cockroachdb/cockroach/pkg/raft/raftpb" - "github.com/cockroachdb/cockroach/pkg/raft/raftstoreliveness" ) // InteractionOpts groups the options for an InteractionEnv. @@ -54,6 +53,7 @@ type InteractionEnv struct { Options *InteractionOpts Nodes []Node Messages []pb.Message // in-flight messages + Fabric *livenessFabric Output *RedirectLogger } @@ -65,6 +65,7 @@ func NewInteractionEnv(opts *InteractionOpts) *InteractionEnv { } return &InteractionEnv{ Options: opts, + Fabric: newLivenessFabric(), Output: &RedirectLogger{ Builder: &strings.Builder{}, }, @@ -103,7 +104,6 @@ func raftConfigStub() raft.Config { HeartbeatTick: 1, MaxSizePerMsg: math.MaxUint64, MaxInflightMsgs: math.MaxInt32, - StoreLiveness: raftstoreliveness.AlwaysLive{}, } } diff --git a/pkg/raft/rafttest/interaction_env_handler.go b/pkg/raft/rafttest/interaction_env_handler.go index 2ffec2b40026..b9c7b6dbe775 100644 --- a/pkg/raft/rafttest/interaction_env_handler.go +++ b/pkg/raft/rafttest/interaction_env_handler.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/cockroachdb/datadriven" + "github.com/cockroachdb/errors" ) // Handle is the entrypoint for data-driven interaction testing. Commands and @@ -192,6 +193,65 @@ func (env *InteractionEnv) Handle(t *testing.T, d datadriven.TestData) string { // Example: // report-unreachable 1 2 err = env.handleReportUnreachable(t, d) + case "store-liveness": + // Prints the global store liveness state. + // + // Example: + // store-liveness + if env.Fabric == nil { + err = errors.Newf("empty liveness fabric") + break + } + _, err = env.Output.WriteString(env.Fabric.String()) + case "bump-epoch": + // Bumps the epoch of a store. As a result, the store stops seeking support + // from all remote stores at the prior epoch. It instead (successfully) + // seeks support for the newer epoch. + // + // bump-epoch id + // Arguments are: + // id - id of the store whose epoch is being bumped. + // + // Example: + // bump-epoch 1 + err = env.handleBumpEpoch(t, d) + + case "withdraw-support": + // Withdraw support for another store (for_store) from a store (from_store). + // + // Note that after invoking "withdraw-support", a test may establish support + // from from_store for for_store at a higher epoch by calling + // "grant-support". + // + // withdraw-support from_id for_id + // Arguments are: + // from_id - id of the store who is withdrawing support. + // for_id - id of the store for which support is being withdrawn. + // + // Example: + // withdraw-support 1 2 + // Explanation: + // 1 (from_store) withdraws support for 2's (for_store) current epoch. + err = env.handleWithdrawSupport(t, d) + + case "grant-support": + // Grant support for another store (for_store) by forcing for_store to bump + // its epoch and using it to seek support from from_store at this new epoch. + // + // Note that From_store should not be supporting for_store already; if it + // is, an error will be returned. + // + // grant-support from_id for_id + // Arguments are: + // from_id - id of the store who is withdrawing support. + // for_id - id of the store for which support is being withdrawn. + // + // Example: + // grant-support 1 2 + // Explanation: + // 1 (from_store) grants support for 2 (for_store) at a higher epoch. + err = env.handleGrantSupport(t, d) + default: err = fmt.Errorf("unknown command") } @@ -211,12 +271,16 @@ func (env *InteractionEnv) Handle(t *testing.T, d datadriven.TestData) string { } func firstAsInt(t *testing.T, d datadriven.TestData) int { + return nthAsInt(t, d, 0) +} + +func nthAsInt(t *testing.T, d datadriven.TestData, n int) int { t.Helper() - n, err := strconv.Atoi(d.CmdArgs[0].Key) + ret, err := strconv.Atoi(d.CmdArgs[n].Key) if err != nil { t.Fatal(err) } - return n + return ret } func firstAsNodeIdx(t *testing.T, d datadriven.TestData) int { diff --git a/pkg/raft/rafttest/interaction_env_handler_add_nodes.go b/pkg/raft/rafttest/interaction_env_handler_add_nodes.go index 1d7f17599cfa..3f55a5eceb9c 100644 --- a/pkg/raft/rafttest/interaction_env_handler_add_nodes.go +++ b/pkg/raft/rafttest/interaction_env_handler_add_nodes.go @@ -126,6 +126,10 @@ func (env *InteractionEnv) AddNodes(n int, cfg raft.Config, snap pb.Snapshot) er } cfg := cfg // fork the config stub cfg.ID, cfg.Storage = id, s + + env.Fabric.addNode() + cfg.StoreLiveness = newStoreLiveness(env.Fabric, id) + if env.Options.OnConfig != nil { env.Options.OnConfig(&cfg) if cfg.ID != id { diff --git a/pkg/raft/rafttest/interaction_env_handler_store_liveness.go b/pkg/raft/rafttest/interaction_env_handler_store_liveness.go new file mode 100644 index 000000000000..c4a5047a2b91 --- /dev/null +++ b/pkg/raft/rafttest/interaction_env_handler_store_liveness.go @@ -0,0 +1,196 @@ +// Copyright 2024 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package rafttest + +import ( + "fmt" + "strings" + "testing" + + pb "github.com/cockroachdb/cockroach/pkg/raft/raftpb" + "github.com/cockroachdb/cockroach/pkg/raft/raftstoreliveness" + "github.com/cockroachdb/cockroach/pkg/util/hlc" + "github.com/cockroachdb/datadriven" + "github.com/cockroachdb/errors" +) + +// livenessEntry is an entry in the liveness fabric. +type livenessEntry struct { + epoch pb.Epoch + isSupported bool +} + +// initLIvenessEntry is the initial liveness entry placed in the liveness fabric +// for new stores. +var initLivenessEntry = livenessEntry{ + epoch: 1, + isSupported: true, +} + +// livenessFabric is a global view of the store liveness state. +type livenessFabric struct { + // state is a 2D array, where state[i][j] represents store i's support for + // store j. Stores are 1-indexed. + state [][]livenessEntry +} + +// newLivenessFabric initializes and returns a livenessFabric. +func newLivenessFabric() *livenessFabric { + // Stores are 1-indexed. Pad our state with an empty, unusued throughout, + // liveness entry. This is useful in addNode below. + state := make([][]livenessEntry, 1) + state[0] = make([]livenessEntry, 1) + state[0][0] = initLivenessEntry + return &livenessFabric{ + state: state, + } +} + +// addNode adds another node (store) to the liveness fabric. +func (l *livenessFabric) addNode() { + // For every store that already exists, add another column for the node we're + // adding. + for i := range l.state { + l.state[i] = append(l.state[i], initLivenessEntry) + } + // Add a row for the node we're adding. + newNodeState := make([]livenessEntry, len(l.state)+1) // 1-indexed stores + for i := range l.state { + newNodeState[i] = livenessEntry{ + // NB: The most recent epoch nodes are using to seek support is stored at + // l.state[i][i].epoch. Start providing support at this epoch. + epoch: l.state[i][i].epoch, + isSupported: true, + } + } + l.state = append(l.state, newNodeState) + + // Finally, initialize the liveness entry for the node we've just added. It'll + // start off with epoch 1 and support itself. + l.state[len(l.state)-1][len(l.state)-1] = initLivenessEntry +} + +func (l *livenessFabric) String() string { + var buf strings.Builder + for i := range l.state { + for j := range l.state[i] { + if i == 0 && j == 0 { + buf.WriteString(" ") + continue + } + if i == 0 { + buf.WriteString(fmt.Sprintf(" %d", j)) + continue + } + if j == 0 { + buf.WriteString(fmt.Sprintf("%d", i)) + continue + } + + buf.WriteString(" ") + if l.state[i][j].isSupported { + buf.WriteString(fmt.Sprintf("%d", l.state[i][j].epoch)) + } else { + buf.WriteString("x") + } + } + buf.WriteString("\n") + } + return buf.String() +} + +// storeLiveness is a per-peer view of the store liveness state. +type storeLiveness struct { + livenessFabric *livenessFabric + nodeID pb.PeerID +} + +var _ raftstoreliveness.StoreLiveness = &storeLiveness{} + +func newStoreLiveness(livenessFabric *livenessFabric, nodeID pb.PeerID) *storeLiveness { + return &storeLiveness{ + nodeID: nodeID, + livenessFabric: livenessFabric, + } +} + +// SupportFor implements the StoreLiveness interface. +func (s *storeLiveness) SupportFor(id pb.PeerID) (pb.Epoch, bool) { + entry := s.livenessFabric.state[s.nodeID][id] + return entry.epoch, entry.isSupported +} + +// SupportFrom implements the StoreLiveness interface. +func (s *storeLiveness) SupportFrom(id pb.PeerID) (pb.Epoch, hlc.Timestamp, bool) { + entry := s.livenessFabric.state[id][s.nodeID] + if !entry.isSupported { + return 0, hlc.MinTimestamp, false + } + // TODO(arul): we may need to inject timestamps in here as well. + return entry.epoch, hlc.MaxTimestamp, entry.isSupported +} + +// SupportFromEnabled implements the StoreLiveness interface. +func (s *storeLiveness) SupportFromEnabled() bool { + return true +} + +// SupportExpired implements the StoreLiveness interface. +func (s *storeLiveness) SupportExpired(hlc.Timestamp) bool { + // TODO(arul): we may need to implement this if we start injecting timestamps. + return false +} + +// handleBumpEpoch handles the case where the epoch of a store is bumped and the +// store then seeks support from all other stores at this new epoch. +func (env *InteractionEnv) handleBumpEpoch(t *testing.T, d datadriven.TestData) error { + n := firstAsInt(t, d) + for i := range env.Fabric.state { + env.Fabric.state[i][n].epoch++ + } + _, err := env.Output.WriteString(env.Fabric.String()) + return err +} + +// handleWithdrawSupport handles the case where a store withdraws support for +// another store. The store for which support has been withdrawn may seek +// support from the store that withdrew support in the future by bumping its +// epoch. +func (env *InteractionEnv) handleWithdrawSupport(t *testing.T, d datadriven.TestData) error { + fromStore := nthAsInt(t, d, 0) + forStore := nthAsInt(t, d, 1) + // Bump forStore's epoch and mark it as unsupported. + env.Fabric.state[fromStore][forStore].epoch++ + env.Fabric.state[fromStore][forStore].isSupported = false + _, err := env.Output.WriteString(env.Fabric.String()) + return err +} + +// handleGrantSupport handles the case where where a store grants support for +// another store. To enable this, the store for whom support is being granted +// must seek support at a higher epoch. +func (env *InteractionEnv) handleGrantSupport(t *testing.T, d datadriven.TestData) error { + fromStore := nthAsInt(t, d, 0) + forStore := nthAsInt(t, d, 1) + if env.Fabric.state[fromStore][forStore].isSupported { + return errors.Newf("store %d is already supporting %d", fromStore, forStore) + } + // First, copy over the bumped epoch (while ensuring it doesn't regress) onto + // env.Fabric.state[forStore][forStore].epoch. + env.Fabric.state[forStore][forStore].epoch = max( + env.Fabric.state[forStore][forStore].epoch, env.Fabric.state[fromStore][forStore].epoch, + ) + // Then, provide support from fromStore for forStore at this new epoch. + env.Fabric.state[fromStore][forStore].epoch = env.Fabric.state[forStore][forStore].epoch + env.Fabric.state[fromStore][forStore].isSupported = true + _, err := env.Output.WriteString(env.Fabric.String()) + return err +} diff --git a/pkg/raft/rawnode_test.go b/pkg/raft/rawnode_test.go index 48fcf0ce95d0..2a035ec8b4d4 100644 --- a/pkg/raft/rawnode_test.go +++ b/pkg/raft/rawnode_test.go @@ -521,7 +521,7 @@ func TestRawNodeStart(t *testing.T) { } want := Ready{ SoftState: &SoftState{RaftState: StateLeader}, - HardState: pb.HardState{Term: 1, Commit: 3, Vote: 1, Lead: 1}, + HardState: pb.HardState{Term: 1, Commit: 3, Vote: 1, Lead: 1, LeadEpoch: 1}, Entries: nil, // emitted & checked in intermediate Ready cycle CommittedEntries: entries, MustSync: false, // since we're only applying, not appending diff --git a/pkg/raft/testdata/async_storage_writes.txt b/pkg/raft/testdata/async_storage_writes.txt index 3b56bb1d0306..79dd5398e707 100644 --- a/pkg/raft/testdata/async_storage_writes.txt +++ b/pkg/raft/testdata/async_storage_writes.txt @@ -84,7 +84,7 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:1 Vote:1 Commit:10 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:10 Lead:1 LeadEpoch:1 Entries: 1/11 EntryNormal "" Messages: @@ -110,45 +110,47 @@ stabilize AppendThread->1 MsgStorageAppendResp Term:0 Log:1/11 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:10 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:10 Lead:1 LeadEpoch:1 Entries: 1/11 EntryNormal "" Messages: - 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 2->AppendThread MsgStorageAppend Term:1 Log:1/11 Commit:10 Vote:1 Lead:1 Entries:[1/11 EntryNormal ""] Responses:[ + 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 2->1 MsgAppResp Term:1 Log:0/11 AppendThread->2 MsgStorageAppendResp Term:0 Log:1/11 ] > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:10 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:10 Lead:1 LeadEpoch:1 Entries: 1/11 EntryNormal "" Messages: - 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 3->AppendThread MsgStorageAppend Term:1 Log:1/11 Commit:10 Vote:1 Lead:1 Entries:[1/11 EntryNormal ""] Responses:[ + 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 3->1 MsgAppResp Term:1 Log:0/11 AppendThread->3 MsgStorageAppendResp Term:0 Log:1/11 ] > 1 receiving messages 1->1 MsgAppResp Term:1 Log:0/11 AppendThread->1 MsgStorageAppendResp Term:0 Log:1/11 - 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) - 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) > 2 processing append thread Processing: 2->AppendThread MsgStorageAppend Term:1 Log:1/11 Commit:10 Vote:1 Lead:1 Entries:[1/11 EntryNormal ""] Responses: + 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 2->1 MsgAppResp Term:1 Log:0/11 AppendThread->2 MsgStorageAppendResp Term:0 Log:1/11 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:1 Log:1/11 Commit:10 Vote:1 Lead:1 Entries:[1/11 EntryNormal ""] Responses: + 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 3->1 MsgAppResp Term:1 Log:0/11 AppendThread->3 MsgStorageAppendResp Term:0 Log:1/11 > 1 receiving messages + 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 2->1 MsgAppResp Term:1 Log:0/11 + 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 3->1 MsgAppResp Term:1 Log:0/11 > 2 receiving messages AppendThread->2 MsgStorageAppendResp Term:0 Log:1/11 @@ -156,7 +158,7 @@ stabilize AppendThread->3 MsgStorageAppendResp Term:0 Log:1/11 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:1 CommittedEntries: 1/11 EntryNormal "" Messages: @@ -181,7 +183,7 @@ stabilize ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:1 CommittedEntries: 1/11 EntryNormal "" Messages: @@ -193,7 +195,7 @@ stabilize ] > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:1 CommittedEntries: 1/11 EntryNormal "" Messages: @@ -383,7 +385,7 @@ process-ready 1 2 3 ---- > 1 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:1 Entries: 1/14 EntryNormal "prop_3" CommittedEntries: @@ -418,7 +420,7 @@ process-ready 1 2 3 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:1 Entries: 1/14 EntryNormal "prop_3" CommittedEntries: @@ -434,7 +436,7 @@ process-ready 1 2 3 ] > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:1 Entries: 1/14 EntryNormal "prop_3" CommittedEntries: @@ -487,7 +489,7 @@ process-ready 1 2 3 ---- > 1 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1 Entries: 1/15 EntryNormal "prop_4" CommittedEntries: @@ -522,7 +524,7 @@ process-ready 1 2 3 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1 Entries: 1/15 EntryNormal "prop_4" CommittedEntries: @@ -538,7 +540,7 @@ process-ready 1 2 3 ] > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1 Entries: 1/15 EntryNormal "prop_4" CommittedEntries: @@ -612,7 +614,7 @@ process-ready 1 2 3 ---- > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:14 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:14 Lead:1 LeadEpoch:1 CommittedEntries: 1/14 EntryNormal "prop_3" Messages: @@ -638,7 +640,7 @@ process-ready 1 2 3 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:14 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:14 Lead:1 LeadEpoch:1 CommittedEntries: 1/14 EntryNormal "prop_3" Messages: @@ -650,7 +652,7 @@ process-ready 1 2 3 ] > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:14 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:14 Lead:1 LeadEpoch:1 CommittedEntries: 1/14 EntryNormal "prop_3" Messages: @@ -720,7 +722,7 @@ process-ready 1 2 3 ---- > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:15 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:15 Lead:1 LeadEpoch:1 CommittedEntries: 1/15 EntryNormal "prop_4" Messages: @@ -746,7 +748,7 @@ process-ready 1 2 3 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:15 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:15 Lead:1 LeadEpoch:1 CommittedEntries: 1/15 EntryNormal "prop_4" Messages: @@ -758,7 +760,7 @@ process-ready 1 2 3 ] > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:15 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:15 Lead:1 LeadEpoch:1 CommittedEntries: 1/15 EntryNormal "prop_4" Messages: diff --git a/pkg/raft/testdata/async_storage_writes_append_aba_race.txt b/pkg/raft/testdata/async_storage_writes_append_aba_race.txt index c498902c1488..c34774b8cd32 100644 --- a/pkg/raft/testdata/async_storage_writes_append_aba_race.txt +++ b/pkg/raft/testdata/async_storage_writes_append_aba_race.txt @@ -86,7 +86,7 @@ process-ready 3 ---- Ready MustSync=true: State:StateCandidate -HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:0 +HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:1 Messages: 3->1 MsgVote Term:2 Log:1/11 3->2 MsgVote Term:2 Log:1/11 @@ -117,21 +117,21 @@ process-ready 4 5 6 ---- > 4 handling Ready Ready MustSync=true: - HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:1 Messages: 4->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[ 4->3 MsgVoteResp Term:2 Log:0/0 ] > 5 handling Ready Ready MustSync=true: - HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:1 Messages: 5->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[ 5->3 MsgVoteResp Term:2 Log:0/0 ] > 6 handling Ready Ready MustSync=true: - HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:1 Messages: 6->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[ 6->3 MsgVoteResp Term:2 Log:0/0 @@ -185,7 +185,7 @@ process-ready 3 ---- Ready MustSync=true: State:StateLeader -HardState Term:2 Vote:3 Commit:11 Lead:3 LeadEpoch:0 +HardState Term:2 Vote:3 Commit:11 Lead:3 LeadEpoch:1 Entries: 2/12 EntryNormal "" Messages: @@ -232,13 +232,13 @@ dropped: 3->7 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] process-ready 1 ---- Ready MustSync=true: -HardState Term:2 Commit:11 Lead:3 LeadEpoch:0 +HardState Term:2 Commit:11 Lead:3 LeadEpoch:1 Entries: 2/12 EntryNormal "" Messages: -1->3 MsgFortifyLeaderResp Term:2 Log:0/0 Rejected (Hint: 0) 1->AppendThread MsgStorageAppend Term:2 Log:2/12 Commit:11 Lead:3 Entries:[2/12 EntryNormal ""] Responses:[ 1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0) + 1->3 MsgFortifyLeaderResp Term:2 Log:0/0 Rejected (Hint: 0) 1->3 MsgAppResp Term:2 Log:0/12 AppendThread->1 MsgStorageAppendResp Term:0 Log:2/12 ] @@ -260,7 +260,7 @@ process-ready 4 ---- Ready MustSync=true: State:StateCandidate -HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:0 +HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:1 Messages: 4->1 MsgVote Term:3 Log:1/11 4->2 MsgVote Term:3 Log:1/11 @@ -291,21 +291,21 @@ process-ready 5 6 7 ---- > 5 handling Ready Ready MustSync=true: - HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:1 Messages: 5->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[ 5->4 MsgVoteResp Term:3 Log:0/0 ] > 6 handling Ready Ready MustSync=true: - HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:1 Messages: 6->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[ 6->4 MsgVoteResp Term:3 Log:0/0 ] > 7 handling Ready Ready MustSync=true: - HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:1 Messages: 7->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[ 7->4 MsgVoteResp Term:3 Log:0/0 @@ -354,7 +354,7 @@ process-ready 4 ---- Ready MustSync=true: State:StateLeader -HardState Term:3 Vote:4 Commit:11 Lead:4 LeadEpoch:0 +HardState Term:3 Vote:4 Commit:11 Lead:4 LeadEpoch:1 Entries: 3/12 EntryNormal "" Messages: @@ -411,7 +411,7 @@ INFO 1 became follower at term 3 process-ready 1 ---- Ready MustSync=true: -HardState Term:3 Commit:11 Lead:4 LeadEpoch:0 +HardState Term:3 Commit:11 Lead:4 LeadEpoch:1 Messages: 1->4 MsgHeartbeatResp Term:3 Log:0/0 1->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Lead:4 @@ -483,6 +483,7 @@ Processing: 1->AppendThread MsgStorageAppend Term:2 Log:2/12 Commit:11 Lead:3 Entries:[2/12 EntryNormal ""] Responses: 1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0) +1->3 MsgFortifyLeaderResp Term:2 Log:0/0 Rejected (Hint: 0) 1->3 MsgAppResp Term:2 Log:0/12 AppendThread->1 MsgStorageAppendResp Term:0 Log:2/12 diff --git a/pkg/raft/testdata/campaign.txt b/pkg/raft/testdata/campaign.txt index fd1b9bfe8389..ab6defa9e554 100644 --- a/pkg/raft/testdata/campaign.txt +++ b/pkg/raft/testdata/campaign.txt @@ -61,7 +61,7 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" Messages: @@ -77,7 +77,7 @@ stabilize 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" Messages: @@ -85,7 +85,7 @@ stabilize 2->1 MsgAppResp Term:1 Log:0/3 > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" Messages: @@ -98,7 +98,7 @@ stabilize 3->1 MsgAppResp Term:1 Log:0/3 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" Messages: @@ -110,14 +110,14 @@ stabilize 1->3 MsgApp Term:1 Log:1/3 Commit:3 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" Messages: 2->1 MsgAppResp Term:1 Log:0/3 > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/campaign_learner_must_vote.txt b/pkg/raft/testdata/campaign_learner_must_vote.txt index 911ea75f5aa5..a263fc5b9504 100644 --- a/pkg/raft/testdata/campaign_learner_must_vote.txt +++ b/pkg/raft/testdata/campaign_learner_must_vote.txt @@ -62,7 +62,7 @@ process-ready 2 ---- Ready MustSync=true: State:StateCandidate -HardState Term:2 Vote:2 Commit:4 Lead:0 LeadEpoch:0 +HardState Term:2 Vote:2 Commit:4 Lead:0 LeadEpoch:1 Messages: 2->1 MsgVote Term:2 Log:1/4 2->3 MsgVote Term:2 Log:1/4 @@ -82,7 +82,7 @@ stabilize 3 INFO 3 [logterm: 1, index: 3, vote: 0] cast MsgVote for 2 [logterm: 1, index: 4] at term 2 > 3 handling Ready Ready MustSync=true: - HardState Term:2 Vote:2 Commit:3 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:3 Lead:0 LeadEpoch:1 Messages: 3->2 MsgVoteResp Term:2 Log:0/0 @@ -96,7 +96,7 @@ stabilize 2 3 > 2 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:2 Vote:2 Commit:4 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:4 Lead:2 LeadEpoch:1 Entries: 2/5 EntryNormal "" Messages: @@ -110,7 +110,7 @@ stabilize 2 3 DEBUG 3 [logterm: 0, index: 4] rejected MsgApp [logterm: 1, index: 4] from 2 > 3 handling Ready Ready MustSync=true: - HardState Term:2 Vote:2 Commit:3 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:3 Lead:2 LeadEpoch:1 Messages: 3->2 MsgFortifyLeaderResp Term:2 Log:0/0 Rejected (Hint: 0) 3->2 MsgAppResp Term:2 Log:1/4 Rejected (Hint: 3) @@ -133,7 +133,7 @@ stabilize 2 3 ] > 3 handling Ready Ready MustSync=true: - HardState Term:2 Vote:2 Commit:4 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:4 Lead:2 LeadEpoch:1 Entries: 1/4 EntryConfChangeV2 v3 2/5 EntryNormal "" @@ -146,7 +146,7 @@ stabilize 2 3 3->2 MsgAppResp Term:2 Log:0/5 > 2 handling Ready Ready MustSync=false: - HardState Term:2 Vote:2 Commit:5 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:5 Lead:2 LeadEpoch:1 CommittedEntries: 2/5 EntryNormal "" Messages: @@ -155,7 +155,7 @@ stabilize 2 3 2->3 MsgApp Term:2 Log:2/5 Commit:5 > 3 handling Ready Ready MustSync=false: - HardState Term:2 Vote:2 Commit:5 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:5 Lead:2 LeadEpoch:1 CommittedEntries: 2/5 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/checkquorum.txt b/pkg/raft/testdata/checkquorum.txt index 204ccfa8d676..7d694f612a45 100644 --- a/pkg/raft/testdata/checkquorum.txt +++ b/pkg/raft/testdata/checkquorum.txt @@ -37,7 +37,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StateCandidate - HardState Term:2 Vote:2 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:11 Lead:0 LeadEpoch:1 Messages: 2->1 MsgVote Term:2 Log:1/11 2->3 MsgVote Term:2 Log:1/11 @@ -128,7 +128,7 @@ stabilize INFO 1 [term: 2] ignored a MsgHeartbeatResp message with lower term from 3 [term: 1] > 1 handling Ready Ready MustSync=true: - HardState Term:2 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Commit:11 Lead:0 LeadEpoch:1 # Other nodes can now successfully campaign. Note that we haven't ticked 3, so # it won't grant votes. @@ -142,7 +142,7 @@ INFO 2 [logterm: 1, index: 11] sent MsgVote request to 3 at term 3 process-ready 2 ---- Ready MustSync=true: -HardState Term:3 Vote:2 Commit:11 Lead:0 LeadEpoch:0 +HardState Term:3 Vote:2 Commit:11 Lead:0 LeadEpoch:1 Messages: 2->1 MsgVote Term:3 Log:1/11 2->3 MsgVote Term:3 Log:1/11 @@ -165,7 +165,7 @@ stabilize ---- > 1 handling Ready Ready MustSync=true: - HardState Term:3 Vote:2 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:11 Lead:0 LeadEpoch:1 Messages: 1->2 MsgVoteResp Term:3 Log:0/0 > 2 receiving messages @@ -176,7 +176,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:3 Vote:2 Commit:11 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:11 Lead:2 LeadEpoch:1 Entries: 3/12 EntryNormal "" Messages: @@ -194,7 +194,7 @@ stabilize 2->3 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] > 1 handling Ready Ready MustSync=true: - HardState Term:3 Vote:2 Commit:11 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:11 Lead:2 LeadEpoch:1 Entries: 3/12 EntryNormal "" Messages: @@ -202,7 +202,7 @@ stabilize 1->2 MsgAppResp Term:3 Log:0/12 > 3 handling Ready Ready MustSync=true: - HardState Term:3 Commit:11 Lead:2 LeadEpoch:0 + HardState Term:3 Commit:11 Lead:2 LeadEpoch:1 Entries: 3/12 EntryNormal "" Messages: @@ -215,7 +215,7 @@ stabilize 3->2 MsgAppResp Term:3 Log:0/12 > 2 handling Ready Ready MustSync=false: - HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:1 CommittedEntries: 3/12 EntryNormal "" Messages: @@ -227,14 +227,14 @@ stabilize 2->3 MsgApp Term:3 Log:3/12 Commit:12 > 1 handling Ready Ready MustSync=false: - HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:1 CommittedEntries: 3/12 EntryNormal "" Messages: 1->2 MsgAppResp Term:3 Log:0/12 > 3 handling Ready Ready MustSync=false: - HardState Term:3 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:3 Commit:12 Lead:2 LeadEpoch:1 CommittedEntries: 3/12 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/confchange_disable_validation.txt b/pkg/raft/testdata/confchange_disable_validation.txt index 373de2045f26..98ef9892a61b 100644 --- a/pkg/raft/testdata/confchange_disable_validation.txt +++ b/pkg/raft/testdata/confchange_disable_validation.txt @@ -42,7 +42,7 @@ stabilize 1 1/5 EntryConfChangeV2 l2 l3 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/4 EntryNormal "foo" > 1 handling Ready diff --git a/pkg/raft/testdata/confchange_drop_if_unapplied.txt b/pkg/raft/testdata/confchange_drop_if_unapplied.txt index 832e57fe0877..10fb2395d900 100644 --- a/pkg/raft/testdata/confchange_drop_if_unapplied.txt +++ b/pkg/raft/testdata/confchange_drop_if_unapplied.txt @@ -43,7 +43,7 @@ stabilize 1 ---- > 1 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 Entries: 1/5 EntryNormal "" CommittedEntries: @@ -51,7 +51,7 @@ stabilize 1 INFO 1 switched to configuration voters=(1)&&(1) learners=(2 3) > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/5 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/confchange_v1_add_single.txt b/pkg/raft/testdata/confchange_v1_add_single.txt index f24a13a64ae1..84b926de2c39 100644 --- a/pkg/raft/testdata/confchange_v1_add_single.txt +++ b/pkg/raft/testdata/confchange_v1_add_single.txt @@ -42,13 +42,13 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" 1/4 EntryConfChange v2 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" 1/4 EntryConfChange v2 diff --git a/pkg/raft/testdata/confchange_v1_remove_leader.txt b/pkg/raft/testdata/confchange_v1_remove_leader.txt index 83fc3b523283..59e8b9d51725 100644 --- a/pkg/raft/testdata/confchange_v1_remove_leader.txt +++ b/pkg/raft/testdata/confchange_v1_remove_leader.txt @@ -96,7 +96,7 @@ stabilize 1 2->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/4 EntryConfChange r1 1/5 EntryNormal "foo" @@ -122,7 +122,7 @@ stabilize 2 1->2 MsgApp Term:1 Log:1/6 Commit:5 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 Entries: 1/6 EntryNormal "bar" CommittedEntries: @@ -153,7 +153,7 @@ stabilize 1->3 MsgApp Term:1 Log:1/6 Commit:5 > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 Entries: 1/4 EntryConfChange r1 1/5 EntryNormal "foo" @@ -176,7 +176,7 @@ stabilize 3->1 MsgAppResp Term:1 Log:0/6 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:1 CommittedEntries: 1/6 EntryNormal "bar" Messages: @@ -188,14 +188,14 @@ stabilize 1->3 MsgApp Term:1 Log:1/6 Commit:6 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:1 CommittedEntries: 1/6 EntryNormal "bar" Messages: 2->1 MsgAppResp Term:1 Log:0/6 > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:1 CommittedEntries: 1/6 EntryNormal "bar" Messages: diff --git a/pkg/raft/testdata/confchange_v1_remove_leader_stepdown.txt b/pkg/raft/testdata/confchange_v1_remove_leader_stepdown.txt index 007444ada490..50b32c3901e3 100644 --- a/pkg/raft/testdata/confchange_v1_remove_leader_stepdown.txt +++ b/pkg/raft/testdata/confchange_v1_remove_leader_stepdown.txt @@ -95,7 +95,7 @@ stabilize 1 2->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/4 EntryConfChange r1 1/5 EntryNormal "foo" @@ -125,7 +125,7 @@ stabilize 2 1->2 MsgApp Term:1 Log:1/6 Commit:5 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 Entries: 1/6 EntryNormal "bar" CommittedEntries: @@ -156,7 +156,7 @@ stabilize 1->3 MsgApp Term:1 Log:1/6 Commit:5 > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 Entries: 1/4 EntryConfChange r1 1/5 EntryNormal "foo" diff --git a/pkg/raft/testdata/confchange_v2_add_double_auto.txt b/pkg/raft/testdata/confchange_v2_add_double_auto.txt index 76e33b474e1f..b1f083cd7b5d 100644 --- a/pkg/raft/testdata/confchange_v2_add_double_auto.txt +++ b/pkg/raft/testdata/confchange_v2_add_double_auto.txt @@ -43,7 +43,7 @@ process-ready 1 ---- Ready MustSync=true: State:StateLeader -HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 +HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 v3 @@ -55,7 +55,7 @@ Entries: process-ready 1 ---- Ready MustSync=false: -HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 +HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 v3 @@ -133,7 +133,7 @@ stabilize 1 2 2->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: @@ -267,7 +267,7 @@ stabilize 1 3->1 MsgAppResp Term:1 Log:0/6 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:1 CommittedEntries: 1/6 EntryConfChangeV2 r2 r3 Messages: @@ -343,7 +343,7 @@ stabilize 3->1 MsgAppResp Term:1 Log:0/9 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:9 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:9 Lead:1 LeadEpoch:1 CommittedEntries: 1/7 EntryNormal "foo" 1/8 EntryNormal "bar" diff --git a/pkg/raft/testdata/confchange_v2_add_double_implicit.txt b/pkg/raft/testdata/confchange_v2_add_double_implicit.txt index 3c747cd8ecb0..8d08ec33512c 100644 --- a/pkg/raft/testdata/confchange_v2_add_double_implicit.txt +++ b/pkg/raft/testdata/confchange_v2_add_double_implicit.txt @@ -45,13 +45,13 @@ stabilize 1 2 > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 @@ -116,7 +116,7 @@ stabilize 1 2 2->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: diff --git a/pkg/raft/testdata/confchange_v2_add_single_auto.txt b/pkg/raft/testdata/confchange_v2_add_single_auto.txt index e032be859863..e2c9026bd48b 100644 --- a/pkg/raft/testdata/confchange_v2_add_single_auto.txt +++ b/pkg/raft/testdata/confchange_v2_add_single_auto.txt @@ -43,13 +43,13 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 diff --git a/pkg/raft/testdata/confchange_v2_add_single_explicit.txt b/pkg/raft/testdata/confchange_v2_add_single_explicit.txt index 8413305402d8..9b6ea40396ea 100644 --- a/pkg/raft/testdata/confchange_v2_add_single_explicit.txt +++ b/pkg/raft/testdata/confchange_v2_add_single_explicit.txt @@ -43,13 +43,13 @@ stabilize 1 2 > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 @@ -136,7 +136,7 @@ stabilize 2->1 MsgAppResp Term:1 Log:0/6 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:1 CommittedEntries: 1/5 EntryNormal "" 1/6 EntryConfChangeV2 @@ -187,7 +187,7 @@ stabilize 2->1 MsgAppResp Term:1 Log:0/7 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:7 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:7 Lead:1 LeadEpoch:1 CommittedEntries: 1/7 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/confchange_v2_replace_leader.txt b/pkg/raft/testdata/confchange_v2_replace_leader.txt index b11afdc0a93e..0b07247a83fc 100644 --- a/pkg/raft/testdata/confchange_v2_replace_leader.txt +++ b/pkg/raft/testdata/confchange_v2_replace_leader.txt @@ -77,7 +77,7 @@ stabilize 3->1 MsgAppResp Term:1 Log:0/4 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/4 EntryConfChangeV2 r1 v4 Messages: @@ -94,7 +94,7 @@ stabilize 1->4 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 r1 v4] > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/4 EntryConfChangeV2 r1 v4 Messages: @@ -102,7 +102,7 @@ stabilize INFO 2 switched to configuration voters=(2 3 4)&&(1 2 3) > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/4 EntryConfChangeV2 r1 v4 Messages: @@ -201,17 +201,17 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:2 Vote:4 Commit:4 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:4 Lead:0 LeadEpoch:1 Messages: 1->4 MsgVoteResp Term:2 Log:0/0 > 2 handling Ready Ready MustSync=true: - HardState Term:2 Vote:4 Commit:4 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:4 Lead:0 LeadEpoch:1 Messages: 2->4 MsgVoteResp Term:2 Log:0/0 > 3 handling Ready Ready MustSync=true: - HardState Term:2 Vote:4 Commit:4 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:4 Lead:0 LeadEpoch:1 Messages: 3->4 MsgVoteResp Term:2 Log:0/0 > 4 receiving messages @@ -226,7 +226,7 @@ stabilize > 4 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:1 Entries: 2/5 EntryNormal "" Messages: @@ -247,7 +247,7 @@ stabilize 4->3 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] > 1 handling Ready Ready MustSync=true: - HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:1 Entries: 2/5 EntryNormal "" Messages: @@ -255,7 +255,7 @@ stabilize 1->4 MsgAppResp Term:2 Log:0/5 > 2 handling Ready Ready MustSync=true: - HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:1 Entries: 2/5 EntryNormal "" Messages: @@ -263,7 +263,7 @@ stabilize 2->4 MsgAppResp Term:2 Log:0/5 > 3 handling Ready Ready MustSync=true: - HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:1 Entries: 2/5 EntryNormal "" Messages: @@ -278,7 +278,7 @@ stabilize 3->4 MsgAppResp Term:2 Log:0/5 > 4 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:1 CommittedEntries: 2/5 EntryNormal "" Messages: @@ -293,21 +293,21 @@ stabilize 4->3 MsgApp Term:2 Log:2/5 Commit:5 > 1 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:1 CommittedEntries: 2/5 EntryNormal "" Messages: 1->4 MsgAppResp Term:2 Log:0/5 > 2 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:1 CommittedEntries: 2/5 EntryNormal "" Messages: 2->4 MsgAppResp Term:2 Log:0/5 > 3 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:1 CommittedEntries: 2/5 EntryNormal "" Messages: @@ -371,7 +371,7 @@ stabilize 3->4 MsgAppResp Term:2 Log:0/6 > 4 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:1 CommittedEntries: 2/6 EntryConfChangeV2 Messages: @@ -387,7 +387,7 @@ stabilize 4->3 MsgApp Term:2 Log:2/6 Commit:6 > 1 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:1 CommittedEntries: 2/6 EntryConfChangeV2 Messages: @@ -395,7 +395,7 @@ stabilize INFO 1 switched to configuration voters=(2 3 4) > 2 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:1 CommittedEntries: 2/6 EntryConfChangeV2 Messages: @@ -403,7 +403,7 @@ stabilize INFO 2 switched to configuration voters=(2 3 4) > 3 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:1 CommittedEntries: 2/6 EntryConfChangeV2 Messages: diff --git a/pkg/raft/testdata/confchange_v2_replace_leader_stepdown.txt b/pkg/raft/testdata/confchange_v2_replace_leader_stepdown.txt index 333031ef386c..267d553835c1 100644 --- a/pkg/raft/testdata/confchange_v2_replace_leader_stepdown.txt +++ b/pkg/raft/testdata/confchange_v2_replace_leader_stepdown.txt @@ -112,7 +112,7 @@ stabilize 4->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: @@ -132,7 +132,7 @@ stabilize State:StateFollower > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: @@ -140,7 +140,7 @@ stabilize INFO 2 switched to configuration voters=(2 3 4) > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: diff --git a/pkg/raft/testdata/forget_leader.txt b/pkg/raft/testdata/forget_leader.txt index 058ea66e308a..65875ca3a87a 100644 --- a/pkg/raft/testdata/forget_leader.txt +++ b/pkg/raft/testdata/forget_leader.txt @@ -81,10 +81,10 @@ stabilize 1->4 MsgHeartbeat Term:1 Log:0/0 Commit:11 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:1 > 4 handling Ready Ready MustSync=true: - HardState Term:1 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:1 Commit:11 Lead:0 LeadEpoch:1 > 2 receiving messages 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 > 3 receiving messages @@ -93,7 +93,7 @@ stabilize 1->4 MsgHeartbeat Term:1 Log:0/0 Commit:11 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:1 Messages: 2->1 MsgHeartbeatResp Term:1 Log:0/0 > 3 handling Ready @@ -102,7 +102,7 @@ stabilize 3->1 MsgHeartbeatResp Term:1 Log:0/0 > 4 handling Ready Ready MustSync=true: - HardState Term:1 Commit:11 Lead:1 LeadEpoch:0 + HardState Term:1 Commit:11 Lead:1 LeadEpoch:1 Messages: 4->1 MsgHeartbeatResp Term:1 Log:0/0 > 1 receiving messages diff --git a/pkg/raft/testdata/forget_leader_prevote_checkquorum.txt b/pkg/raft/testdata/forget_leader_prevote_checkquorum.txt index 1df1ab2fd182..a929b9f1ca21 100644 --- a/pkg/raft/testdata/forget_leader_prevote_checkquorum.txt +++ b/pkg/raft/testdata/forget_leader_prevote_checkquorum.txt @@ -36,7 +36,7 @@ stabilize 3 > 3 handling Ready Ready MustSync=true: State:StatePreCandidate - HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:1 Messages: 3->1 MsgPreVote Term:2 Log:1/11 3->2 MsgPreVote Term:2 Log:1/11 @@ -74,7 +74,7 @@ stabilize > 3 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:1 Messages: 3->1 MsgHeartbeatResp Term:1 Log:0/0 > 1 receiving messages @@ -111,7 +111,7 @@ stabilize 3 > 3 handling Ready Ready MustSync=true: State:StatePreCandidate - HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:1 Messages: 3->1 MsgPreVote Term:2 Log:1/11 3->2 MsgPreVote Term:2 Log:1/11 @@ -122,7 +122,7 @@ stabilize 2 ---- > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:1 > 2 receiving messages 3->2 MsgPreVote Term:2 Log:1/11 INFO 2 [logterm: 1, index: 11, vote: 1] cast MsgPreVote for 3 [logterm: 1, index: 11] at term 1 @@ -143,7 +143,7 @@ stabilize 3 > 3 handling Ready Ready MustSync=true: State:StateCandidate - HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:1 Messages: 3->1 MsgVote Term:2 Log:1/11 3->2 MsgVote Term:2 Log:1/11 @@ -208,7 +208,7 @@ process-ready 1 ---- Ready MustSync=true: State:StatePreCandidate -HardState Term:2 Commit:12 Lead:0 LeadEpoch:0 +HardState Term:2 Commit:12 Lead:0 LeadEpoch:1 Messages: 1->2 MsgPreVote Term:3 Log:2/12 1->3 MsgPreVote Term:3 Log:2/12 @@ -219,7 +219,7 @@ stabilize 2 ---- > 2 handling Ready Ready MustSync=true: - HardState Term:2 Vote:3 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:12 Lead:0 LeadEpoch:1 > 2 receiving messages 1->2 MsgPreVote Term:3 Log:2/12 INFO 2 [logterm: 2, index: 13, vote: 3] rejected MsgPreVote from 1 [logterm: 2, index: 12] at term 2 diff --git a/pkg/raft/testdata/fortification_basic.txt b/pkg/raft/testdata/fortification_basic.txt new file mode 100644 index 000000000000..20e3713aef50 --- /dev/null +++ b/pkg/raft/testdata/fortification_basic.txt @@ -0,0 +1,159 @@ +# Basic tests for leader fortification. + +log-level info +---- +ok + +add-nodes 3 voters=(1,2,3) index=2 +---- +INFO 1 switched to configuration voters=(1 2 3) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] +INFO 2 switched to configuration voters=(1 2 3) +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] +INFO 3 switched to configuration voters=(1 2 3) +INFO 3 became follower at term 0 +INFO newRaft 3 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +# Muck around with StoreLiveness to make it somewhat interesting. +bump-epoch 1 +---- + 1 2 3 +1 2 1 1 +2 2 1 1 +3 2 1 1 + +withdraw-support 1 1 +---- + 1 2 3 +1 x 1 1 +2 2 1 1 +3 2 1 1 + +grant-support 1 1 +---- + 1 2 3 +1 3 1 1 +2 2 1 1 +3 2 1 1 + +withdraw-support 3 1 +---- + 1 2 3 +1 3 1 1 +2 2 1 1 +3 x 1 1 + + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 +INFO 1 [logterm: 1, index: 2] sent MsgVote request to 2 at term 1 +INFO 1 [logterm: 1, index: 2] sent MsgVote request to 3 at term 1 + +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + State:StateCandidate + HardState Term:1 Vote:1 Commit:2 Lead:0 LeadEpoch:0 + Messages: + 1->2 MsgVote Term:1 Log:1/2 + 1->3 MsgVote Term:1 Log:1/2 + INFO 1 received MsgVoteResp from 1 at term 1 + INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections +> 2 receiving messages + 1->2 MsgVote Term:1 Log:1/2 + INFO 2 [term: 0] received a MsgVote message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + INFO 2 [logterm: 1, index: 2, vote: 0] cast MsgVote for 1 [logterm: 1, index: 2] at term 1 +> 3 receiving messages + 1->3 MsgVote Term:1 Log:1/2 + INFO 3 [term: 0] received a MsgVote message with higher term from 1 [term: 1] + INFO 3 became follower at term 1 + INFO 3 [logterm: 1, index: 2, vote: 0] cast MsgVote for 1 [logterm: 1, index: 2] at term 1 +> 2 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:0 LeadEpoch:0 + Messages: + 2->1 MsgVoteResp Term:1 Log:0/0 +> 3 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:0 LeadEpoch:0 + Messages: + 3->1 MsgVoteResp Term:1 Log:0/0 +> 1 receiving messages + 2->1 MsgVoteResp Term:1 Log:0/0 + INFO 1 received MsgVoteResp from 2 at term 1 + INFO 1 has received 2 MsgVoteResp votes and 0 vote rejections + INFO 1 became leader at term 1 + 3->1 MsgVoteResp Term:1 Log:0/0 +> 1 handling Ready + Ready MustSync=true: + State:StateLeader + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:3 + Entries: + 1/3 EntryNormal "" + Messages: + 1->2 MsgFortifyLeader Term:1 Log:0/0 + 1->3 MsgFortifyLeader Term:1 Log:0/0 + 1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] + 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] +> 2 receiving messages + 1->2 MsgFortifyLeader Term:1 Log:0/0 + 1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] +> 3 receiving messages + 1->3 MsgFortifyLeader Term:1 Log:0/0 + 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] +> 2 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:2 + Entries: + 1/3 EntryNormal "" + Messages: + 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 2->1 MsgAppResp Term:1 Log:0/3 +> 3 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + Entries: + 1/3 EntryNormal "" + Messages: + 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 3->1 MsgAppResp Term:1 Log:0/3 +> 1 receiving messages + 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 2->1 MsgAppResp Term:1 Log:0/3 + 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 3->1 MsgAppResp Term:1 Log:0/3 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:3 + CommittedEntries: + 1/3 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:3 + 1->3 MsgApp Term:1 Log:1/3 Commit:3 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:3 +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/3 Commit:3 +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:2 + CommittedEntries: + 1/3 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:0 + CommittedEntries: + 1/3 EntryNormal "" + Messages: + 3->1 MsgAppResp Term:1 Log:0/3 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/3 + 3->1 MsgAppResp Term:1 Log:0/3 diff --git a/pkg/raft/testdata/fortification_leader_does_not_support_itself.txt b/pkg/raft/testdata/fortification_leader_does_not_support_itself.txt new file mode 100644 index 000000000000..f11b0785c604 --- /dev/null +++ b/pkg/raft/testdata/fortification_leader_does_not_support_itself.txt @@ -0,0 +1,160 @@ +# This file tests the scenario where the leader doesn't support itself in the +# liveness fabric for whatever reason. + +log-level info +---- +ok + +add-nodes 3 voters=(1,2,3) index=2 +---- +INFO 1 switched to configuration voters=(1 2 3) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] +INFO 2 switched to configuration voters=(1 2 3) +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] +INFO 3 switched to configuration voters=(1 2 3) +INFO 3 became follower at term 0 +INFO newRaft 3 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +# Make StoreLiveness interesting. Then, withdraw support for 1 from 1. +bump-epoch 1 +---- + 1 2 3 +1 2 1 1 +2 2 1 1 +3 2 1 1 + +withdraw-support 2 1 +---- + 1 2 3 +1 2 1 1 +2 x 1 1 +3 2 1 1 + +grant-support 2 1 +---- + 1 2 3 +1 3 1 1 +2 3 1 1 +3 2 1 1 + +withdraw-support 1 1 +---- + 1 2 3 +1 x 1 1 +2 3 1 1 +3 2 1 1 + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 +INFO 1 [logterm: 1, index: 2] sent MsgVote request to 2 at term 1 +INFO 1 [logterm: 1, index: 2] sent MsgVote request to 3 at term 1 + +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + State:StateCandidate + HardState Term:1 Vote:1 Commit:2 Lead:0 LeadEpoch:0 + Messages: + 1->2 MsgVote Term:1 Log:1/2 + 1->3 MsgVote Term:1 Log:1/2 + INFO 1 received MsgVoteResp from 1 at term 1 + INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections +> 2 receiving messages + 1->2 MsgVote Term:1 Log:1/2 + INFO 2 [term: 0] received a MsgVote message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + INFO 2 [logterm: 1, index: 2, vote: 0] cast MsgVote for 1 [logterm: 1, index: 2] at term 1 +> 3 receiving messages + 1->3 MsgVote Term:1 Log:1/2 + INFO 3 [term: 0] received a MsgVote message with higher term from 1 [term: 1] + INFO 3 became follower at term 1 + INFO 3 [logterm: 1, index: 2, vote: 0] cast MsgVote for 1 [logterm: 1, index: 2] at term 1 +> 2 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:0 LeadEpoch:0 + Messages: + 2->1 MsgVoteResp Term:1 Log:0/0 +> 3 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:0 LeadEpoch:0 + Messages: + 3->1 MsgVoteResp Term:1 Log:0/0 +> 1 receiving messages + 2->1 MsgVoteResp Term:1 Log:0/0 + INFO 1 received MsgVoteResp from 2 at term 1 + INFO 1 has received 2 MsgVoteResp votes and 0 vote rejections + INFO 1 became leader at term 1 + INFO 1 leader at term 1 does not support itself in the liveness fabric + 3->1 MsgVoteResp Term:1 Log:0/0 +> 1 handling Ready + Ready MustSync=true: + State:StateLeader + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + Entries: + 1/3 EntryNormal "" + Messages: + 1->2 MsgFortifyLeader Term:1 Log:0/0 + 1->3 MsgFortifyLeader Term:1 Log:0/0 + 1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] + 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] +> 2 receiving messages + 1->2 MsgFortifyLeader Term:1 Log:0/0 + 1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] +> 3 receiving messages + 1->3 MsgFortifyLeader Term:1 Log:0/0 + 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] +> 2 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:3 + Entries: + 1/3 EntryNormal "" + Messages: + 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 2->1 MsgAppResp Term:1 Log:0/3 +> 3 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:2 + Entries: + 1/3 EntryNormal "" + Messages: + 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 3->1 MsgAppResp Term:1 Log:0/3 +> 1 receiving messages + 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 2->1 MsgAppResp Term:1 Log:0/3 + 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 3->1 MsgAppResp Term:1 Log:0/3 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:0 + CommittedEntries: + 1/3 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:3 + 1->3 MsgApp Term:1 Log:1/3 Commit:3 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:3 +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/3 Commit:3 +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:3 + CommittedEntries: + 1/3 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:2 + CommittedEntries: + 1/3 EntryNormal "" + Messages: + 3->1 MsgAppResp Term:1 Log:0/3 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/3 + 3->1 MsgAppResp Term:1 Log:0/3 diff --git a/pkg/raft/testdata/lagging_commit.txt b/pkg/raft/testdata/lagging_commit.txt index c23146b78740..16914f3346f7 100644 --- a/pkg/raft/testdata/lagging_commit.txt +++ b/pkg/raft/testdata/lagging_commit.txt @@ -78,7 +78,7 @@ stabilize 1 2 2->1 MsgAppResp Term:1 Log:0/13 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1 CommittedEntries: 1/12 EntryNormal "data1" 1/13 EntryNormal "data2" @@ -92,7 +92,7 @@ stabilize 1 2 1->2 MsgApp Term:1 Log:1/13 Commit:13 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1 CommittedEntries: 1/12 EntryNormal "data1" 1/13 EntryNormal "data2" @@ -164,7 +164,7 @@ stabilize 1 3 1->3 MsgApp Term:1 Log:1/13 Commit:13 > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1 CommittedEntries: 1/12 EntryNormal "data1" 1/13 EntryNormal "data2" diff --git a/pkg/raft/testdata/prevote.txt b/pkg/raft/testdata/prevote.txt index 8dcb3eac935d..973468911a02 100644 --- a/pkg/raft/testdata/prevote.txt +++ b/pkg/raft/testdata/prevote.txt @@ -67,7 +67,7 @@ process-ready 3 ---- Ready MustSync=true: State:StatePreCandidate -HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:0 +HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:1 Messages: 3->1 MsgPreVote Term:2 Log:1/11 3->2 MsgPreVote Term:2 Log:1/11 @@ -87,7 +87,7 @@ stabilize ---- > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:1 CommittedEntries: 1/12 EntryNormal "prop_1" Messages: @@ -108,7 +108,7 @@ stabilize 2->3 MsgPreVoteResp Term:1 Log:0/0 Rejected (Hint: 0) > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:1 CommittedEntries: 1/12 EntryNormal "prop_1" Messages: @@ -116,7 +116,7 @@ stabilize > 3 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:1 Entries: 1/12 EntryNormal "prop_1" CommittedEntries: @@ -142,7 +142,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StatePreCandidate - HardState Term:1 Vote:1 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:12 Lead:0 LeadEpoch:1 Messages: 2->1 MsgPreVote Term:2 Log:1/12 2->3 MsgPreVote Term:2 Log:1/12 @@ -173,7 +173,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StateCandidate - HardState Term:2 Vote:2 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:12 Lead:0 LeadEpoch:1 Messages: 2->1 MsgVote Term:2 Log:1/12 2->3 MsgVote Term:2 Log:1/12 @@ -192,12 +192,12 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:2 Vote:2 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:12 Lead:0 LeadEpoch:1 Messages: 1->2 MsgVoteResp Term:2 Log:0/0 > 3 handling Ready Ready MustSync=true: - HardState Term:2 Vote:2 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:12 Lead:0 LeadEpoch:1 Messages: 3->2 MsgVoteResp Term:2 Log:0/0 > 2 receiving messages @@ -209,7 +209,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:2 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:12 Lead:2 LeadEpoch:1 Entries: 2/13 EntryNormal "" Messages: @@ -225,7 +225,7 @@ stabilize 2->3 MsgApp Term:2 Log:1/12 Commit:12 Entries:[2/13 EntryNormal ""] > 1 handling Ready Ready MustSync=true: - HardState Term:2 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:12 Lead:2 LeadEpoch:1 Entries: 2/13 EntryNormal "" Messages: @@ -233,7 +233,7 @@ stabilize 1->2 MsgAppResp Term:2 Log:0/13 > 3 handling Ready Ready MustSync=true: - HardState Term:2 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:12 Lead:2 LeadEpoch:1 Entries: 2/13 EntryNormal "" Messages: @@ -246,7 +246,7 @@ stabilize 3->2 MsgAppResp Term:2 Log:0/13 > 2 handling Ready Ready MustSync=false: - HardState Term:2 Vote:2 Commit:13 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:13 Lead:2 LeadEpoch:1 CommittedEntries: 2/13 EntryNormal "" Messages: @@ -258,14 +258,14 @@ stabilize 2->3 MsgApp Term:2 Log:2/13 Commit:13 > 1 handling Ready Ready MustSync=false: - HardState Term:2 Vote:2 Commit:13 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:13 Lead:2 LeadEpoch:1 CommittedEntries: 2/13 EntryNormal "" Messages: 1->2 MsgAppResp Term:2 Log:0/13 > 3 handling Ready Ready MustSync=false: - HardState Term:2 Vote:2 Commit:13 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:13 Lead:2 LeadEpoch:1 CommittedEntries: 2/13 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/prevote_checkquorum.txt b/pkg/raft/testdata/prevote_checkquorum.txt index 9b0da76aa083..decde104d0fc 100644 --- a/pkg/raft/testdata/prevote_checkquorum.txt +++ b/pkg/raft/testdata/prevote_checkquorum.txt @@ -36,7 +36,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StatePreCandidate - HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:1 Messages: 2->1 MsgPreVote Term:2 Log:1/11 2->3 MsgPreVote Term:2 Log:1/11 @@ -70,7 +70,7 @@ process-ready 3 ---- Ready MustSync=true: State:StatePreCandidate -HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:0 +HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:1 Messages: 3->1 MsgPreVote Term:2 Log:1/11 3->2 MsgPreVote Term:2 Log:1/11 @@ -103,7 +103,7 @@ stabilize > 3 handling Ready Ready MustSync=true: State:StateCandidate - HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:1 Messages: 3->1 MsgVote Term:2 Log:1/11 3->2 MsgVote Term:2 Log:1/11 @@ -120,7 +120,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:1 Messages: 2->3 MsgVoteResp Term:2 Log:0/0 > 3 receiving messages @@ -131,7 +131,7 @@ stabilize > 3 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:2 Vote:3 Commit:11 Lead:3 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:3 LeadEpoch:1 Entries: 2/12 EntryNormal "" Messages: @@ -150,7 +150,7 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:2 Commit:11 Lead:3 LeadEpoch:0 + HardState Term:2 Commit:11 Lead:3 LeadEpoch:1 Entries: 2/12 EntryNormal "" Messages: @@ -158,7 +158,7 @@ stabilize 1->3 MsgAppResp Term:2 Log:0/12 > 2 handling Ready Ready MustSync=true: - HardState Term:2 Vote:3 Commit:11 Lead:3 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:3 LeadEpoch:1 Entries: 2/12 EntryNormal "" Messages: @@ -171,7 +171,7 @@ stabilize 2->3 MsgAppResp Term:2 Log:0/12 > 3 handling Ready Ready MustSync=false: - HardState Term:2 Vote:3 Commit:12 Lead:3 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:12 Lead:3 LeadEpoch:1 CommittedEntries: 2/12 EntryNormal "" Messages: @@ -183,14 +183,14 @@ stabilize 3->2 MsgApp Term:2 Log:2/12 Commit:12 > 1 handling Ready Ready MustSync=false: - HardState Term:2 Commit:12 Lead:3 LeadEpoch:0 + HardState Term:2 Commit:12 Lead:3 LeadEpoch:1 CommittedEntries: 2/12 EntryNormal "" Messages: 1->3 MsgAppResp Term:2 Log:0/12 > 2 handling Ready Ready MustSync=false: - HardState Term:2 Vote:3 Commit:12 Lead:3 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:12 Lead:3 LeadEpoch:1 CommittedEntries: 2/12 EntryNormal "" Messages: @@ -219,7 +219,7 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StatePreCandidate - HardState Term:2 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:2 Commit:12 Lead:0 LeadEpoch:1 Messages: 1->2 MsgPreVote Term:3 Log:2/12 1->3 MsgPreVote Term:3 Log:2/12 @@ -244,7 +244,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StatePreCandidate - HardState Term:2 Vote:3 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:12 Lead:0 LeadEpoch:1 Messages: 2->1 MsgPreVote Term:3 Log:2/12 2->3 MsgPreVote Term:3 Log:2/12 @@ -270,7 +270,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StateCandidate - HardState Term:3 Vote:2 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:12 Lead:0 LeadEpoch:1 Messages: 2->1 MsgVote Term:3 Log:2/12 2->3 MsgVote Term:3 Log:2/12 @@ -287,7 +287,7 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:3 Vote:2 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:12 Lead:0 LeadEpoch:1 Messages: 1->2 MsgVoteResp Term:3 Log:0/0 > 2 receiving messages @@ -298,7 +298,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:1 Entries: 3/13 EntryNormal "" Messages: @@ -316,7 +316,7 @@ stabilize 2->3 MsgApp Term:3 Log:2/12 Commit:12 Entries:[3/13 EntryNormal ""] > 1 handling Ready Ready MustSync=true: - HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:1 Entries: 3/13 EntryNormal "" Messages: @@ -325,7 +325,7 @@ stabilize > 3 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:3 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:3 Commit:12 Lead:2 LeadEpoch:1 Entries: 3/13 EntryNormal "" Messages: @@ -338,7 +338,7 @@ stabilize 3->2 MsgAppResp Term:3 Log:0/13 > 2 handling Ready Ready MustSync=false: - HardState Term:3 Vote:2 Commit:13 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:13 Lead:2 LeadEpoch:1 CommittedEntries: 3/13 EntryNormal "" Messages: @@ -350,14 +350,14 @@ stabilize 2->3 MsgApp Term:3 Log:3/13 Commit:13 > 1 handling Ready Ready MustSync=false: - HardState Term:3 Vote:2 Commit:13 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:13 Lead:2 LeadEpoch:1 CommittedEntries: 3/13 EntryNormal "" Messages: 1->2 MsgAppResp Term:3 Log:0/13 > 3 handling Ready Ready MustSync=false: - HardState Term:3 Commit:13 Lead:2 LeadEpoch:0 + HardState Term:3 Commit:13 Lead:2 LeadEpoch:1 CommittedEntries: 3/13 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/probe_and_replicate.txt b/pkg/raft/testdata/probe_and_replicate.txt index 4e7535c76684..9b8af246b5aa 100644 --- a/pkg/raft/testdata/probe_and_replicate.txt +++ b/pkg/raft/testdata/probe_and_replicate.txt @@ -370,7 +370,7 @@ stabilize 1 > 1 handling Ready Ready MustSync=true: State:StateCandidate - HardState Term:8 Vote:1 Commit:18 Lead:0 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:18 Lead:0 LeadEpoch:1 Messages: 1->2 MsgVote Term:8 Log:6/20 1->3 MsgVote Term:8 Log:6/20 @@ -415,34 +415,34 @@ stabilize 2 3 4 5 6 7 INFO 7 [logterm: 3, index: 21, vote: 0] cast MsgVote for 1 [logterm: 6, index: 20] at term 8 > 2 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:18 Lead:0 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:18 Lead:0 LeadEpoch:1 Messages: 2->1 MsgVoteResp Term:8 Log:0/0 > 3 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:14 Lead:0 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:14 Lead:0 LeadEpoch:1 Messages: 3->1 MsgVoteResp Term:8 Log:0/0 > 4 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:8 Commit:18 Lead:0 LeadEpoch:0 + HardState Term:8 Commit:18 Lead:0 LeadEpoch:1 Messages: 4->1 MsgVoteResp Term:8 Log:0/0 Rejected (Hint: 0) > 5 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:8 Commit:18 Lead:0 LeadEpoch:0 + HardState Term:8 Commit:18 Lead:0 LeadEpoch:1 Messages: 5->1 MsgVoteResp Term:8 Log:0/0 Rejected (Hint: 0) > 6 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:15 Lead:0 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:15 Lead:0 LeadEpoch:1 Messages: 6->1 MsgVoteResp Term:8 Log:0/0 > 7 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:13 Lead:0 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:13 Lead:0 LeadEpoch:1 Messages: 7->1 MsgVoteResp Term:8 Log:0/0 @@ -469,7 +469,7 @@ stabilize 1 > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:8 Vote:1 Commit:18 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:18 Lead:1 LeadEpoch:1 Entries: 8/21 EntryNormal "" Messages: @@ -494,7 +494,7 @@ stabilize 1 2 1->2 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] > 2 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:18 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:18 Lead:1 LeadEpoch:1 Messages: 2->1 MsgFortifyLeaderResp Term:8 Log:0/0 Rejected (Hint: 0) 2->1 MsgAppResp Term:8 Log:6/20 Rejected (Hint: 19) @@ -530,7 +530,7 @@ stabilize 1 3 1->3 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] > 3 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:14 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:14 Lead:1 LeadEpoch:1 Messages: 3->1 MsgFortifyLeaderResp Term:8 Log:0/0 Rejected (Hint: 0) 3->1 MsgAppResp Term:8 Log:4/20 Rejected (Hint: 14) @@ -561,7 +561,7 @@ stabilize 1 3 ] > 3 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:18 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:18 Lead:1 LeadEpoch:1 Entries: 4/15 EntryNormal "prop_4_15" 5/16 EntryNormal "" @@ -589,7 +589,7 @@ stabilize 1 4 INFO replace the unstable entries from index 21 > 4 handling Ready Ready MustSync=true: - HardState Term:8 Commit:18 Lead:1 LeadEpoch:0 + HardState Term:8 Commit:18 Lead:1 LeadEpoch:1 Entries: 8/21 EntryNormal "" Messages: @@ -600,7 +600,7 @@ stabilize 1 4 4->1 MsgAppResp Term:8 Log:0/21 > 1 handling Ready Ready MustSync=false: - HardState Term:8 Vote:1 Commit:21 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:21 Lead:1 LeadEpoch:1 CommittedEntries: 6/19 EntryNormal "prop_6_19" 6/20 EntryNormal "prop_6_20" @@ -613,7 +613,7 @@ stabilize 1 4 1->4 MsgApp Term:8 Log:8/21 Commit:21 > 4 handling Ready Ready MustSync=false: - HardState Term:8 Commit:21 Lead:1 LeadEpoch:0 + HardState Term:8 Commit:21 Lead:1 LeadEpoch:1 CommittedEntries: 6/19 EntryNormal "prop_6_19" 6/20 EntryNormal "prop_6_20" @@ -630,7 +630,7 @@ stabilize 1 5 1->5 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] > 5 handling Ready Ready MustSync=true: - HardState Term:8 Commit:18 Lead:1 LeadEpoch:0 + HardState Term:8 Commit:18 Lead:1 LeadEpoch:1 Messages: 5->1 MsgFortifyLeaderResp Term:8 Log:0/0 Rejected (Hint: 0) 5->1 MsgAppResp Term:8 Log:6/20 Rejected (Hint: 18) @@ -655,7 +655,7 @@ stabilize 1 5 INFO replace the unstable entries from index 19 > 5 handling Ready Ready MustSync=true: - HardState Term:8 Commit:21 Lead:1 LeadEpoch:0 + HardState Term:8 Commit:21 Lead:1 LeadEpoch:1 Entries: 6/19 EntryNormal "prop_6_19" 6/20 EntryNormal "prop_6_20" @@ -676,7 +676,7 @@ stabilize 1 6 1->6 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] > 6 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:15 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:15 Lead:1 LeadEpoch:1 Messages: 6->1 MsgFortifyLeaderResp Term:8 Log:0/0 Rejected (Hint: 0) 6->1 MsgAppResp Term:8 Log:4/20 Rejected (Hint: 17) @@ -707,7 +707,7 @@ stabilize 1 6 INFO replace the unstable entries from index 16 > 6 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:21 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:21 Lead:1 LeadEpoch:1 Entries: 5/16 EntryNormal "" 5/17 EntryNormal "prop_5_17" @@ -734,7 +734,7 @@ stabilize 1 7 1->7 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] > 7 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:13 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:13 Lead:1 LeadEpoch:1 Messages: 7->1 MsgFortifyLeaderResp Term:8 Log:0/0 Rejected (Hint: 0) 7->1 MsgAppResp Term:8 Log:3/20 Rejected (Hint: 20) @@ -769,7 +769,7 @@ stabilize 1 7 INFO replace the unstable entries from index 14 > 7 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:21 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:21 Lead:1 LeadEpoch:1 Entries: 4/14 EntryNormal "" 4/15 EntryNormal "prop_4_15" diff --git a/pkg/raft/testdata/single_node.txt b/pkg/raft/testdata/single_node.txt index 23a85c198f8e..f819261495b8 100644 --- a/pkg/raft/testdata/single_node.txt +++ b/pkg/raft/testdata/single_node.txt @@ -25,11 +25,11 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:1 Entries: 1/4 EntryNormal "" > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/4 EntryNormal "" diff --git a/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt b/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt index 8ce4226d3227..23d8e09639a3 100644 --- a/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt +++ b/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt @@ -143,8 +143,8 @@ dropped: 1->3 MsgSnap Term:1 Log:0/0 stabilize 3 ---- > 3 handling Ready - Ready MustSync=false: - HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:0 + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:1 Snapshot Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false Messages: 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) diff --git a/pkg/raft/testdata/store_liveness_basic.txt b/pkg/raft/testdata/store_liveness_basic.txt new file mode 100644 index 000000000000..88f696ea068d --- /dev/null +++ b/pkg/raft/testdata/store_liveness_basic.txt @@ -0,0 +1,167 @@ +# This file has some basic tests that manipulate the store liveness fabric. +# They're more to test the directives themselves than anything else. + +add-nodes 2 index=10 +---- +INFO 1 switched to configuration voters=() +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [], term: 0, commit: 10, applied: 10, lastindex: 10, lastterm: 1] +INFO 2 switched to configuration voters=() +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [], term: 0, commit: 10, applied: 10, lastindex: 10, lastterm: 1] + +store-liveness +---- + 1 2 +1 1 1 +2 1 1 + +add-nodes 1 +---- +INFO 3 switched to configuration voters=() +INFO 3 became follower at term 0 +INFO newRaft 3 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +store-liveness +---- + 1 2 3 +1 1 1 1 +2 1 1 1 +3 1 1 1 + +# Bump epoch of store 1. This will result in it seeking support from all other +# stores at epoch 2, which they'll provide successfully. +bump-epoch 1 +---- + 1 2 3 +1 2 1 1 +2 2 1 1 +3 2 1 1 + +# Withdraw support from 3 for 2. +withdraw-support 3 2 +---- + 1 2 3 +1 2 1 1 +2 2 1 1 +3 2 x 1 + +# Withdraw support from 2 for 3. +withdraw-support 2 3 +---- + 1 2 3 +1 2 1 1 +2 2 1 x +3 2 x 1 + +# Grant support from 2 for 3. This causes 3 to bump its epoch and successfully +# seek support from 2 at the newer epoch (2). +grant-support 2 3 +---- + 1 2 3 +1 2 1 1 +2 2 1 2 +3 2 x 2 + + +# Repeat this process of withdrawing support and granting support from 2 for 3 +# another time to distinguish 3s epoch from 1s epoch. + +withdraw-support 2 3 +---- + 1 2 3 +1 2 1 1 +2 2 1 x +3 2 x 2 + +grant-support 2 3 +---- + 1 2 3 +1 2 1 1 +2 2 1 3 +3 2 x 3 + +# Finally, add another node in the mix. The node should start seeking support +# from all other stores at epoch 1. Moreover, it should provide support to store +# 1 at epoch 2 and store 3 at epoch 3. +add-nodes 1 +---- +INFO 4 switched to configuration voters=() +INFO 4 became follower at term 0 +INFO newRaft 4 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +store-liveness +---- + 1 2 3 4 +1 2 1 1 1 +2 2 1 3 1 +3 2 x 3 1 +4 2 1 3 1 + +# Ensure a store can withdraw support from itself. + +withdraw-support 4 4 +---- + 1 2 3 4 +1 2 1 1 1 +2 2 1 3 1 +3 2 x 3 1 +4 2 1 3 x + +grant-support 4 4 +---- + 1 2 3 4 +1 2 1 1 1 +2 2 1 3 1 +3 2 x 3 1 +4 2 1 3 2 + +# Ensure epochs do not regress, and ensure that whenever support is granted, it +# happens at the latest epoch that for_store is seeking support at. +withdraw-support 1 2 +---- + 1 2 3 4 +1 2 x 1 1 +2 2 1 3 1 +3 2 x 3 1 +4 2 1 3 2 + +# Withdraw support from 1 for 2 again. Internally, it'll bump the epoch 1 is +# tracking for 2 twice (so it'll be 3 now). +withdraw-support 1 2 +---- + 1 2 3 4 +1 2 x 1 1 +2 2 1 3 1 +3 2 x 3 1 +4 2 1 3 2 + +# Withdraw support from 4 for 2 now. Internally, it'll bump 2's epoch to 2. +withdraw-support 4 2 +---- + 1 2 3 4 +1 2 x 1 1 +2 2 1 3 1 +3 2 x 3 1 +4 2 x 3 2 + +# Grant support from 1 for 2; this will happen at epoch 3, which should also be +# mirrored at state[2][2]. +grant-support 1 2 +---- + 1 2 3 4 +1 2 3 1 1 +2 2 3 3 1 +3 2 x 3 1 +4 2 x 3 2 + +# Grant support from 4 for 2. This could have happened at epoch 2, but it will +# happen at epoch 3 instead because of the previous step. Also, note that +# state[2][2] doesn't regress. +grant-support 4 2 +---- + 1 2 3 4 +1 2 3 1 1 +2 2 3 3 1 +3 2 x 3 1 +4 2 3 3 2