From 5892efba690eb6c38d44b63f447f61743ff283ae Mon Sep 17 00:00:00 2001 From: Arul Ajmani Date: Mon, 12 Aug 2024 15:00:12 -0400 Subject: [PATCH] raft: update LeadEpoch in response to MsgFortify This patch partially implements the handling of MsgFortify on peers. In particular, peers (including the leader) now check whether they support the leader when handling a fortification request. They persist the leadEpoch from the StoreLiveness fabric if they do. For now, the fortification request is trivially rejected regardless; this will change shortly. Most of this patch is building out some of the test infra we'll use to set up the underlying StoreLiveness fabric. Informs https://github.com/cockroachdb/cockroach/issues/125261 Release note: None --- pkg/kv/kvserver/replica_store_liveness.go | 5 + pkg/raft/node_test.go | 4 +- pkg/raft/raft.go | 47 ++++- pkg/raft/rafttest/BUILD.bazel | 3 + pkg/raft/rafttest/interaction_env.go | 4 +- pkg/raft/rafttest/interaction_env_handler.go | 71 ++++++- .../interaction_env_handler_add_nodes.go | 4 + .../interaction_env_handler_store_liveness.go | 196 ++++++++++++++++++ pkg/raft/rawnode_test.go | 2 +- pkg/raft/testdata/async_storage_writes.txt | 46 ++-- .../async_storage_writes_append_aba_race.txt | 27 +-- pkg/raft/testdata/campaign.txt | 12 +- .../testdata/campaign_learner_must_vote.txt | 14 +- pkg/raft/testdata/checkquorum.txt | 20 +- .../confchange_disable_validation.txt | 2 +- .../testdata/confchange_drop_if_unapplied.txt | 4 +- .../testdata/confchange_v1_add_single.txt | 4 +- .../testdata/confchange_v1_remove_leader.txt | 12 +- .../confchange_v1_remove_leader_stepdown.txt | 6 +- .../confchange_v2_add_double_auto.txt | 10 +- .../confchange_v2_add_double_implicit.txt | 6 +- .../confchange_v2_add_single_auto.txt | 4 +- .../confchange_v2_add_single_explicit.txt | 8 +- .../testdata/confchange_v2_replace_leader.txt | 36 ++-- .../confchange_v2_replace_leader_stepdown.txt | 6 +- pkg/raft/testdata/forget_leader.txt | 8 +- .../forget_leader_prevote_checkquorum.txt | 14 +- pkg/raft/testdata/fortification_basic.txt | 159 ++++++++++++++ ...ication_leader_does_not_support_itself.txt | 160 ++++++++++++++ pkg/raft/testdata/lagging_commit.txt | 6 +- pkg/raft/testdata/prevote.txt | 28 +-- pkg/raft/testdata/prevote_checkquorum.txt | 40 ++-- pkg/raft/testdata/probe_and_replicate.txt | 40 ++-- pkg/raft/testdata/single_node.txt | 4 +- .../snapshot_succeed_via_app_resp_behind.txt | 4 +- pkg/raft/testdata/store_liveness_basic.txt | 167 +++++++++++++++ 36 files changed, 988 insertions(+), 195 deletions(-) create mode 100644 pkg/raft/rafttest/interaction_env_handler_store_liveness.go create mode 100644 pkg/raft/testdata/fortification_basic.txt create mode 100644 pkg/raft/testdata/fortification_leader_does_not_support_itself.txt create mode 100644 pkg/raft/testdata/store_liveness_basic.txt diff --git a/pkg/kv/kvserver/replica_store_liveness.go b/pkg/kv/kvserver/replica_store_liveness.go index b900027732fa..8fb0891af80f 100644 --- a/pkg/kv/kvserver/replica_store_liveness.go +++ b/pkg/kv/kvserver/replica_store_liveness.go @@ -45,6 +45,11 @@ func (r *replicaRLockedStoreLiveness) SupportFor(replicaID raftpb.PeerID) (raftp if !ok { return 0, false } + // TODO(arul): we can remove this once we start to assign storeLiveness in the + // Store constructor. + if r.store.storeLiveness == nil { + return 0, false + } epoch, ok := r.store.storeLiveness.SupportFor(storeID) if !ok { return 0, false diff --git a/pkg/raft/node_test.go b/pkg/raft/node_test.go index dd574168283e..69299ac96ef0 100644 --- a/pkg/raft/node_test.go +++ b/pkg/raft/node_test.go @@ -453,13 +453,13 @@ func TestNodeStart(t *testing.T) { MustSync: true, }, { - HardState: raftpb.HardState{Term: 2, Commit: 2, Vote: 1, Lead: 1}, + HardState: raftpb.HardState{Term: 2, Commit: 2, Vote: 1, Lead: 1, LeadEpoch: 1}, Entries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}}, CommittedEntries: []raftpb.Entry{{Term: 2, Index: 2, Data: nil}}, MustSync: true, }, { - HardState: raftpb.HardState{Term: 2, Commit: 3, Vote: 1, Lead: 1}, + HardState: raftpb.HardState{Term: 2, Commit: 3, Vote: 1, Lead: 1, LeadEpoch: 1}, Entries: nil, CommittedEntries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}}, MustSync: false, diff --git a/pkg/raft/raft.go b/pkg/raft/raft.go index 3c1effecab98..0c54c1ffa583 100644 --- a/pkg/raft/raft.go +++ b/pkg/raft/raft.go @@ -518,19 +518,22 @@ func (r *raft) send(m pb.Message) { m.Term = r.Term } } - if m.Type == pb.MsgAppResp || m.Type == pb.MsgVoteResp || m.Type == pb.MsgPreVoteResp { + if m.Type == pb.MsgAppResp || m.Type == pb.MsgVoteResp || + m.Type == pb.MsgPreVoteResp || m.Type == pb.MsgFortifyLeaderResp { // If async storage writes are enabled, messages added to the msgs slice // are allowed to be sent out before unstable state (e.g. log entry // writes and election votes) have been durably synced to the local // disk. // - // For most message types, this is not an issue. However, response - // messages that relate to "voting" on either leader election or log - // appends require durability before they can be sent. It would be - // incorrect to publish a vote in an election before that vote has been + // For most message types, this is not an issue. However, response messages + // that relate to "voting" on either leader election or log appends or + // leader fortification require durability before they can be sent. It would + // be incorrect to publish a vote in an election before that vote has been // synced to stable storage locally. Similarly, it would be incorrect to - // acknowledge a log append to the leader before that entry has been - // synced to stable storage locally. + // acknowledge a log append to the leader before that entry has been synced + // to stable storage locally. Similarly, it would also be incorrect to + // promise fortification support to a leader without durably persisting the + // leader's epoch being supported. // // Per the Raft thesis, section 3.8 Persisted state and server restarts: // @@ -683,7 +686,16 @@ func (r *raft) sendFortify(to pb.PeerID) { if to == r.id { // We handle the case where the leader is trying to fortify itself specially. // Doing so avoids a self-addressed message. - // TODO(arul): do this handling. + epoch, live := r.storeLiveness.SupportFor(r.lead) + if live { + r.leadEpoch = epoch + // TODO(arul): For now, we're not recording any support on the leader. Do + // this once we implement handleFortifyResp correctly. + } else { + r.logger.Infof( + "%x leader at term %d does not support itself in the liveness fabric", r.id, r.Term, + ) + } return } r.send(pb.Message{To: to, Type: pb.MsgFortifyLeader}) @@ -1701,6 +1713,8 @@ func stepFollower(r *raft, m pb.Message) error { r.lead = m.From r.handleSnapshot(m) case pb.MsgFortifyLeader: + r.electionElapsed = 0 + r.lead = m.From r.handleFortify(m) case pb.MsgTransferLeader: if r.lead == None { @@ -1868,7 +1882,22 @@ func (r *raft) handleSnapshot(m pb.Message) { } func (r *raft) handleFortify(m pb.Message) { - // TODO(arul): currently a no-op; implement. + assertTrue(r.state == StateFollower, "leaders should locally fortify without sending a message") + + epoch, live := r.storeLiveness.SupportFor(r.lead) + if !live { + // The leader isn't supported by this peer in its liveness fabric. Reject + // the fortification request. + r.send(pb.Message{ + To: m.From, + Type: pb.MsgFortifyLeaderResp, + Reject: true, + }) + return + } + r.leadEpoch = epoch + // TODO(arul): for now, we reject the fortification request because the leader + // hasn't been taught how to handle it. r.send(pb.Message{ To: m.From, Type: pb.MsgFortifyLeaderResp, diff --git a/pkg/raft/rafttest/BUILD.bazel b/pkg/raft/rafttest/BUILD.bazel index efc5d3b9752d..ec636a88a9ec 100644 --- a/pkg/raft/rafttest/BUILD.bazel +++ b/pkg/raft/rafttest/BUILD.bazel @@ -24,6 +24,7 @@ go_library( "interaction_env_handler_set_randomized_election_timeout.go", "interaction_env_handler_stabilize.go", "interaction_env_handler_status.go", + "interaction_env_handler_store_liveness.go", "interaction_env_handler_tick.go", "interaction_env_handler_transfer_leadership.go", "interaction_env_logger.go", @@ -37,7 +38,9 @@ go_library( "//pkg/raft/raftpb", "//pkg/raft/raftstoreliveness", "//pkg/raft/tracker", + "//pkg/util/hlc", "@com_github_cockroachdb_datadriven//:datadriven", + "@com_github_cockroachdb_errors//:errors", "@com_github_stretchr_testify//require", ], ) diff --git a/pkg/raft/rafttest/interaction_env.go b/pkg/raft/rafttest/interaction_env.go index 07c2acd8ee07..663803bd2545 100644 --- a/pkg/raft/rafttest/interaction_env.go +++ b/pkg/raft/rafttest/interaction_env.go @@ -25,7 +25,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/raft" pb "github.com/cockroachdb/cockroach/pkg/raft/raftpb" - "github.com/cockroachdb/cockroach/pkg/raft/raftstoreliveness" ) // InteractionOpts groups the options for an InteractionEnv. @@ -54,6 +53,7 @@ type InteractionEnv struct { Options *InteractionOpts Nodes []Node Messages []pb.Message // in-flight messages + Fabric *livenessFabric Output *RedirectLogger } @@ -65,6 +65,7 @@ func NewInteractionEnv(opts *InteractionOpts) *InteractionEnv { } return &InteractionEnv{ Options: opts, + Fabric: newLivenessFabric(), Output: &RedirectLogger{ Builder: &strings.Builder{}, }, @@ -103,7 +104,6 @@ func raftConfigStub() raft.Config { HeartbeatTick: 1, MaxSizePerMsg: math.MaxUint64, MaxInflightMsgs: math.MaxInt32, - StoreLiveness: raftstoreliveness.AlwaysLive{}, } } diff --git a/pkg/raft/rafttest/interaction_env_handler.go b/pkg/raft/rafttest/interaction_env_handler.go index 2ffec2b40026..511385c75dc3 100644 --- a/pkg/raft/rafttest/interaction_env_handler.go +++ b/pkg/raft/rafttest/interaction_env_handler.go @@ -1,3 +1,6 @@ +// This code has been modified from its original form by Cockroach Labs, Inc. +// All modifications are Copyright 2024 Cockroach Labs, Inc. +// // Copyright 2019 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,6 +23,7 @@ import ( "testing" "github.com/cockroachdb/datadriven" + "github.com/cockroachdb/errors" ) // Handle is the entrypoint for data-driven interaction testing. Commands and @@ -192,6 +196,65 @@ func (env *InteractionEnv) Handle(t *testing.T, d datadriven.TestData) string { // Example: // report-unreachable 1 2 err = env.handleReportUnreachable(t, d) + case "store-liveness": + // Prints the global store liveness state. + // + // Example: + // store-liveness + if env.Fabric == nil { + err = errors.Newf("empty liveness fabric") + break + } + _, err = env.Output.WriteString(env.Fabric.String()) + case "bump-epoch": + // Bumps the epoch of a store. As a result, the store stops seeking support + // from all remote stores at the prior epoch. It instead (successfully) + // seeks support for the newer epoch. + // + // bump-epoch id + // Arguments are: + // id - id of the store whose epoch is being bumped. + // + // Example: + // bump-epoch 1 + err = env.handleBumpEpoch(t, d) + + case "withdraw-support": + // Withdraw support for another store (for_store) from a store (from_store). + // + // Note that after invoking "withdraw-support", a test may establish support + // from from_store for for_store at a higher epoch by calling + // "grant-support". + // + // withdraw-support from_id for_id + // Arguments are: + // from_id - id of the store who is withdrawing support. + // for_id - id of the store for which support is being withdrawn. + // + // Example: + // withdraw-support 1 2 + // Explanation: + // 1 (from_store) withdraws support for 2's (for_store) current epoch. + err = env.handleWithdrawSupport(t, d) + + case "grant-support": + // Grant support for another store (for_store) by forcing for_store to bump + // its epoch and using it to seek support from from_store at this new epoch. + // + // Note that from_store should not be supporting for_store already; if it + // is, an error will be returned. + // + // grant-support from_id for_id + // Arguments are: + // from_id - id of the store who is granting support. + // for_id - id of the store for which support is being granted. + // + // Example: + // grant-support 1 2 + // Explanation: + // 1 (from_store) grants support for 2 (for_store) at a higher epoch. + err = env.handleGrantSupport(t, d) + default: err = fmt.Errorf("unknown command") } @@ -211,12 +274,16 @@ func (env *InteractionEnv) Handle(t *testing.T, d datadriven.TestData) string { } func firstAsInt(t *testing.T, d datadriven.TestData) int { + return nthAsInt(t, d, 0) +} + +func nthAsInt(t *testing.T, d datadriven.TestData, n int) int { t.Helper() - n, err := strconv.Atoi(d.CmdArgs[0].Key) + ret, err := strconv.Atoi(d.CmdArgs[n].Key) if err != nil { t.Fatal(err) } - return n + return ret } func firstAsNodeIdx(t *testing.T, d datadriven.TestData) int { diff --git a/pkg/raft/rafttest/interaction_env_handler_add_nodes.go b/pkg/raft/rafttest/interaction_env_handler_add_nodes.go index 1d7f17599cfa..3f55a5eceb9c 100644 --- a/pkg/raft/rafttest/interaction_env_handler_add_nodes.go +++ b/pkg/raft/rafttest/interaction_env_handler_add_nodes.go @@ -126,6 +126,10 @@ func (env *InteractionEnv) AddNodes(n int, cfg raft.Config, snap pb.Snapshot) er } cfg := cfg // fork the config stub cfg.ID, cfg.Storage = id, s + + env.Fabric.addNode() + cfg.StoreLiveness = newStoreLiveness(env.Fabric, id) + if env.Options.OnConfig != nil { env.Options.OnConfig(&cfg) if cfg.ID != id { diff --git a/pkg/raft/rafttest/interaction_env_handler_store_liveness.go b/pkg/raft/rafttest/interaction_env_handler_store_liveness.go new file mode 100644 index 000000000000..0c461a586ed3 --- /dev/null +++ b/pkg/raft/rafttest/interaction_env_handler_store_liveness.go @@ -0,0 +1,196 @@ +// Copyright 2024 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package rafttest + +import ( + "fmt" + "strings" + "testing" + + pb "github.com/cockroachdb/cockroach/pkg/raft/raftpb" + "github.com/cockroachdb/cockroach/pkg/raft/raftstoreliveness" + "github.com/cockroachdb/cockroach/pkg/util/hlc" + "github.com/cockroachdb/datadriven" + "github.com/cockroachdb/errors" +) + +// livenessEntry is an entry in the liveness fabric. +type livenessEntry struct { + epoch pb.Epoch + isSupported bool +} + +// initLIvenessEntry is the initial liveness entry placed in the liveness fabric +// for new stores. +var initLivenessEntry = livenessEntry{ + epoch: 1, + isSupported: true, +} + +// livenessFabric is a global view of the store liveness state. +type livenessFabric struct { + // state is a 2D array, where state[i][j] represents store i's support for + // store j. Stores are 1-indexed. + state [][]livenessEntry +} + +// newLivenessFabric initializes and returns a livenessFabric. +func newLivenessFabric() *livenessFabric { + // Stores are 1-indexed. Pad our state with an empty, unusued throughout, + // liveness entry. This is useful in addNode below. + state := make([][]livenessEntry, 1) + state[0] = make([]livenessEntry, 1) + state[0][0] = initLivenessEntry + return &livenessFabric{ + state: state, + } +} + +// addNode adds another node (store) to the liveness fabric. +func (l *livenessFabric) addNode() { + // For every store that already exists, add another column for the node we're + // adding. + for i := range l.state { + l.state[i] = append(l.state[i], initLivenessEntry) + } + // Add a row for the node we're adding. + newNodeState := make([]livenessEntry, len(l.state)+1) // 1-indexed stores + for i := range l.state { + newNodeState[i] = livenessEntry{ + // NB: The most recent epoch nodes are using to seek support is stored at + // l.state[i][i].epoch. Start providing support at this epoch. + epoch: l.state[i][i].epoch, + isSupported: true, + } + } + l.state = append(l.state, newNodeState) + + // Finally, initialize the liveness entry for the node we've just added. It'll + // start off with epoch 1 and support itself. + l.state[len(l.state)-1][len(l.state)-1] = initLivenessEntry +} + +func (l *livenessFabric) String() string { + var buf strings.Builder + for i := range l.state { + for j := range l.state[i] { + if i == 0 && j == 0 { + buf.WriteString(" ") + continue + } + if i == 0 { + buf.WriteString(fmt.Sprintf(" %d", j)) + continue + } + if j == 0 { + buf.WriteString(fmt.Sprintf("%d", i)) + continue + } + + buf.WriteString(" ") + if l.state[i][j].isSupported { + buf.WriteString(fmt.Sprintf("%d", l.state[i][j].epoch)) + } else { + buf.WriteString("x") + } + } + buf.WriteString("\n") + } + return buf.String() +} + +// storeLiveness is a per-peer view of the store liveness state. +type storeLiveness struct { + livenessFabric *livenessFabric + nodeID pb.PeerID +} + +var _ raftstoreliveness.StoreLiveness = &storeLiveness{} + +func newStoreLiveness(livenessFabric *livenessFabric, nodeID pb.PeerID) *storeLiveness { + return &storeLiveness{ + nodeID: nodeID, + livenessFabric: livenessFabric, + } +} + +// SupportFor implements the StoreLiveness interface. +func (s *storeLiveness) SupportFor(id pb.PeerID) (pb.Epoch, bool) { + entry := s.livenessFabric.state[s.nodeID][id] + return entry.epoch, entry.isSupported +} + +// SupportFrom implements the StoreLiveness interface. +func (s *storeLiveness) SupportFrom(id pb.PeerID) (pb.Epoch, hlc.Timestamp, bool) { + entry := s.livenessFabric.state[id][s.nodeID] + if !entry.isSupported { + return 0, hlc.Timestamp{}, false + } + // TODO(arul): we may need to inject timestamps in here as well. + return entry.epoch, hlc.MaxTimestamp, entry.isSupported +} + +// SupportFromEnabled implements the StoreLiveness interface. +func (s *storeLiveness) SupportFromEnabled() bool { + return true +} + +// SupportExpired implements the StoreLiveness interface. +func (s *storeLiveness) SupportExpired(hlc.Timestamp) bool { + // TODO(arul): we may need to implement this if we start injecting timestamps. + return false +} + +// handleBumpEpoch handles the case where the epoch of a store is bumped and the +// store then seeks support from all other stores at this new epoch. +func (env *InteractionEnv) handleBumpEpoch(t *testing.T, d datadriven.TestData) error { + n := firstAsInt(t, d) + for i := range env.Fabric.state { + env.Fabric.state[i][n].epoch++ + } + _, err := env.Output.WriteString(env.Fabric.String()) + return err +} + +// handleWithdrawSupport handles the case where a store withdraws support for +// another store. The store for which support has been withdrawn may seek +// support from the store that withdrew support in the future by bumping its +// epoch. +func (env *InteractionEnv) handleWithdrawSupport(t *testing.T, d datadriven.TestData) error { + fromStore := nthAsInt(t, d, 0) + forStore := nthAsInt(t, d, 1) + // Bump forStore's epoch and mark it as unsupported. + env.Fabric.state[fromStore][forStore].epoch++ + env.Fabric.state[fromStore][forStore].isSupported = false + _, err := env.Output.WriteString(env.Fabric.String()) + return err +} + +// handleGrantSupport handles the case where where a store grants support for +// another store. To enable this, the store for whom support is being granted +// must seek support at a higher epoch. +func (env *InteractionEnv) handleGrantSupport(t *testing.T, d datadriven.TestData) error { + fromStore := nthAsInt(t, d, 0) + forStore := nthAsInt(t, d, 1) + if env.Fabric.state[fromStore][forStore].isSupported { + return errors.Newf("store %d is already supporting %d", fromStore, forStore) + } + // First, copy over the bumped epoch (while ensuring it doesn't regress) onto + // env.Fabric.state[forStore][forStore].epoch. + env.Fabric.state[forStore][forStore].epoch = max( + env.Fabric.state[forStore][forStore].epoch, env.Fabric.state[fromStore][forStore].epoch, + ) + // Then, provide support from fromStore for forStore at this new epoch. + env.Fabric.state[fromStore][forStore].epoch = env.Fabric.state[forStore][forStore].epoch + env.Fabric.state[fromStore][forStore].isSupported = true + _, err := env.Output.WriteString(env.Fabric.String()) + return err +} diff --git a/pkg/raft/rawnode_test.go b/pkg/raft/rawnode_test.go index 48fcf0ce95d0..2a035ec8b4d4 100644 --- a/pkg/raft/rawnode_test.go +++ b/pkg/raft/rawnode_test.go @@ -521,7 +521,7 @@ func TestRawNodeStart(t *testing.T) { } want := Ready{ SoftState: &SoftState{RaftState: StateLeader}, - HardState: pb.HardState{Term: 1, Commit: 3, Vote: 1, Lead: 1}, + HardState: pb.HardState{Term: 1, Commit: 3, Vote: 1, Lead: 1, LeadEpoch: 1}, Entries: nil, // emitted & checked in intermediate Ready cycle CommittedEntries: entries, MustSync: false, // since we're only applying, not appending diff --git a/pkg/raft/testdata/async_storage_writes.txt b/pkg/raft/testdata/async_storage_writes.txt index 3b56bb1d0306..79dd5398e707 100644 --- a/pkg/raft/testdata/async_storage_writes.txt +++ b/pkg/raft/testdata/async_storage_writes.txt @@ -84,7 +84,7 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:1 Vote:1 Commit:10 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:10 Lead:1 LeadEpoch:1 Entries: 1/11 EntryNormal "" Messages: @@ -110,45 +110,47 @@ stabilize AppendThread->1 MsgStorageAppendResp Term:0 Log:1/11 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:10 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:10 Lead:1 LeadEpoch:1 Entries: 1/11 EntryNormal "" Messages: - 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 2->AppendThread MsgStorageAppend Term:1 Log:1/11 Commit:10 Vote:1 Lead:1 Entries:[1/11 EntryNormal ""] Responses:[ + 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 2->1 MsgAppResp Term:1 Log:0/11 AppendThread->2 MsgStorageAppendResp Term:0 Log:1/11 ] > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:10 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:10 Lead:1 LeadEpoch:1 Entries: 1/11 EntryNormal "" Messages: - 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 3->AppendThread MsgStorageAppend Term:1 Log:1/11 Commit:10 Vote:1 Lead:1 Entries:[1/11 EntryNormal ""] Responses:[ + 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 3->1 MsgAppResp Term:1 Log:0/11 AppendThread->3 MsgStorageAppendResp Term:0 Log:1/11 ] > 1 receiving messages 1->1 MsgAppResp Term:1 Log:0/11 AppendThread->1 MsgStorageAppendResp Term:0 Log:1/11 - 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) - 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) > 2 processing append thread Processing: 2->AppendThread MsgStorageAppend Term:1 Log:1/11 Commit:10 Vote:1 Lead:1 Entries:[1/11 EntryNormal ""] Responses: + 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 2->1 MsgAppResp Term:1 Log:0/11 AppendThread->2 MsgStorageAppendResp Term:0 Log:1/11 > 3 processing append thread Processing: 3->AppendThread MsgStorageAppend Term:1 Log:1/11 Commit:10 Vote:1 Lead:1 Entries:[1/11 EntryNormal ""] Responses: + 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 3->1 MsgAppResp Term:1 Log:0/11 AppendThread->3 MsgStorageAppendResp Term:0 Log:1/11 > 1 receiving messages + 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 2->1 MsgAppResp Term:1 Log:0/11 + 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) 3->1 MsgAppResp Term:1 Log:0/11 > 2 receiving messages AppendThread->2 MsgStorageAppendResp Term:0 Log:1/11 @@ -156,7 +158,7 @@ stabilize AppendThread->3 MsgStorageAppendResp Term:0 Log:1/11 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:1 CommittedEntries: 1/11 EntryNormal "" Messages: @@ -181,7 +183,7 @@ stabilize ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/11 EntryNormal ""] > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:1 CommittedEntries: 1/11 EntryNormal "" Messages: @@ -193,7 +195,7 @@ stabilize ] > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:1 CommittedEntries: 1/11 EntryNormal "" Messages: @@ -383,7 +385,7 @@ process-ready 1 2 3 ---- > 1 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:1 Entries: 1/14 EntryNormal "prop_3" CommittedEntries: @@ -418,7 +420,7 @@ process-ready 1 2 3 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:1 Entries: 1/14 EntryNormal "prop_3" CommittedEntries: @@ -434,7 +436,7 @@ process-ready 1 2 3 ] > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:1 Entries: 1/14 EntryNormal "prop_3" CommittedEntries: @@ -487,7 +489,7 @@ process-ready 1 2 3 ---- > 1 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1 Entries: 1/15 EntryNormal "prop_4" CommittedEntries: @@ -522,7 +524,7 @@ process-ready 1 2 3 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1 Entries: 1/15 EntryNormal "prop_4" CommittedEntries: @@ -538,7 +540,7 @@ process-ready 1 2 3 ] > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1 Entries: 1/15 EntryNormal "prop_4" CommittedEntries: @@ -612,7 +614,7 @@ process-ready 1 2 3 ---- > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:14 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:14 Lead:1 LeadEpoch:1 CommittedEntries: 1/14 EntryNormal "prop_3" Messages: @@ -638,7 +640,7 @@ process-ready 1 2 3 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:14 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:14 Lead:1 LeadEpoch:1 CommittedEntries: 1/14 EntryNormal "prop_3" Messages: @@ -650,7 +652,7 @@ process-ready 1 2 3 ] > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:14 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:14 Lead:1 LeadEpoch:1 CommittedEntries: 1/14 EntryNormal "prop_3" Messages: @@ -720,7 +722,7 @@ process-ready 1 2 3 ---- > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:15 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:15 Lead:1 LeadEpoch:1 CommittedEntries: 1/15 EntryNormal "prop_4" Messages: @@ -746,7 +748,7 @@ process-ready 1 2 3 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:15 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:15 Lead:1 LeadEpoch:1 CommittedEntries: 1/15 EntryNormal "prop_4" Messages: @@ -758,7 +760,7 @@ process-ready 1 2 3 ] > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:15 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:15 Lead:1 LeadEpoch:1 CommittedEntries: 1/15 EntryNormal "prop_4" Messages: diff --git a/pkg/raft/testdata/async_storage_writes_append_aba_race.txt b/pkg/raft/testdata/async_storage_writes_append_aba_race.txt index c498902c1488..c34774b8cd32 100644 --- a/pkg/raft/testdata/async_storage_writes_append_aba_race.txt +++ b/pkg/raft/testdata/async_storage_writes_append_aba_race.txt @@ -86,7 +86,7 @@ process-ready 3 ---- Ready MustSync=true: State:StateCandidate -HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:0 +HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:1 Messages: 3->1 MsgVote Term:2 Log:1/11 3->2 MsgVote Term:2 Log:1/11 @@ -117,21 +117,21 @@ process-ready 4 5 6 ---- > 4 handling Ready Ready MustSync=true: - HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:1 Messages: 4->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[ 4->3 MsgVoteResp Term:2 Log:0/0 ] > 5 handling Ready Ready MustSync=true: - HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:1 Messages: 5->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[ 5->3 MsgVoteResp Term:2 Log:0/0 ] > 6 handling Ready Ready MustSync=true: - HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:1 Messages: 6->AppendThread MsgStorageAppend Term:2 Log:0/0 Commit:11 Vote:3 Responses:[ 6->3 MsgVoteResp Term:2 Log:0/0 @@ -185,7 +185,7 @@ process-ready 3 ---- Ready MustSync=true: State:StateLeader -HardState Term:2 Vote:3 Commit:11 Lead:3 LeadEpoch:0 +HardState Term:2 Vote:3 Commit:11 Lead:3 LeadEpoch:1 Entries: 2/12 EntryNormal "" Messages: @@ -232,13 +232,13 @@ dropped: 3->7 MsgApp Term:2 Log:1/11 Commit:11 Entries:[2/12 EntryNormal ""] process-ready 1 ---- Ready MustSync=true: -HardState Term:2 Commit:11 Lead:3 LeadEpoch:0 +HardState Term:2 Commit:11 Lead:3 LeadEpoch:1 Entries: 2/12 EntryNormal "" Messages: -1->3 MsgFortifyLeaderResp Term:2 Log:0/0 Rejected (Hint: 0) 1->AppendThread MsgStorageAppend Term:2 Log:2/12 Commit:11 Lead:3 Entries:[2/12 EntryNormal ""] Responses:[ 1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0) + 1->3 MsgFortifyLeaderResp Term:2 Log:0/0 Rejected (Hint: 0) 1->3 MsgAppResp Term:2 Log:0/12 AppendThread->1 MsgStorageAppendResp Term:0 Log:2/12 ] @@ -260,7 +260,7 @@ process-ready 4 ---- Ready MustSync=true: State:StateCandidate -HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:0 +HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:1 Messages: 4->1 MsgVote Term:3 Log:1/11 4->2 MsgVote Term:3 Log:1/11 @@ -291,21 +291,21 @@ process-ready 5 6 7 ---- > 5 handling Ready Ready MustSync=true: - HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:1 Messages: 5->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[ 5->4 MsgVoteResp Term:3 Log:0/0 ] > 6 handling Ready Ready MustSync=true: - HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:1 Messages: 6->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[ 6->4 MsgVoteResp Term:3 Log:0/0 ] > 7 handling Ready Ready MustSync=true: - HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:3 Vote:4 Commit:11 Lead:0 LeadEpoch:1 Messages: 7->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Vote:4 Responses:[ 7->4 MsgVoteResp Term:3 Log:0/0 @@ -354,7 +354,7 @@ process-ready 4 ---- Ready MustSync=true: State:StateLeader -HardState Term:3 Vote:4 Commit:11 Lead:4 LeadEpoch:0 +HardState Term:3 Vote:4 Commit:11 Lead:4 LeadEpoch:1 Entries: 3/12 EntryNormal "" Messages: @@ -411,7 +411,7 @@ INFO 1 became follower at term 3 process-ready 1 ---- Ready MustSync=true: -HardState Term:3 Commit:11 Lead:4 LeadEpoch:0 +HardState Term:3 Commit:11 Lead:4 LeadEpoch:1 Messages: 1->4 MsgHeartbeatResp Term:3 Log:0/0 1->AppendThread MsgStorageAppend Term:3 Log:0/0 Commit:11 Lead:4 @@ -483,6 +483,7 @@ Processing: 1->AppendThread MsgStorageAppend Term:2 Log:2/12 Commit:11 Lead:3 Entries:[2/12 EntryNormal ""] Responses: 1->3 MsgVoteResp Term:2 Log:0/0 Rejected (Hint: 0) +1->3 MsgFortifyLeaderResp Term:2 Log:0/0 Rejected (Hint: 0) 1->3 MsgAppResp Term:2 Log:0/12 AppendThread->1 MsgStorageAppendResp Term:0 Log:2/12 diff --git a/pkg/raft/testdata/campaign.txt b/pkg/raft/testdata/campaign.txt index fd1b9bfe8389..ab6defa9e554 100644 --- a/pkg/raft/testdata/campaign.txt +++ b/pkg/raft/testdata/campaign.txt @@ -61,7 +61,7 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" Messages: @@ -77,7 +77,7 @@ stabilize 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" Messages: @@ -85,7 +85,7 @@ stabilize 2->1 MsgAppResp Term:1 Log:0/3 > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" Messages: @@ -98,7 +98,7 @@ stabilize 3->1 MsgAppResp Term:1 Log:0/3 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" Messages: @@ -110,14 +110,14 @@ stabilize 1->3 MsgApp Term:1 Log:1/3 Commit:3 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" Messages: 2->1 MsgAppResp Term:1 Log:0/3 > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/campaign_learner_must_vote.txt b/pkg/raft/testdata/campaign_learner_must_vote.txt index 911ea75f5aa5..a263fc5b9504 100644 --- a/pkg/raft/testdata/campaign_learner_must_vote.txt +++ b/pkg/raft/testdata/campaign_learner_must_vote.txt @@ -62,7 +62,7 @@ process-ready 2 ---- Ready MustSync=true: State:StateCandidate -HardState Term:2 Vote:2 Commit:4 Lead:0 LeadEpoch:0 +HardState Term:2 Vote:2 Commit:4 Lead:0 LeadEpoch:1 Messages: 2->1 MsgVote Term:2 Log:1/4 2->3 MsgVote Term:2 Log:1/4 @@ -82,7 +82,7 @@ stabilize 3 INFO 3 [logterm: 1, index: 3, vote: 0] cast MsgVote for 2 [logterm: 1, index: 4] at term 2 > 3 handling Ready Ready MustSync=true: - HardState Term:2 Vote:2 Commit:3 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:3 Lead:0 LeadEpoch:1 Messages: 3->2 MsgVoteResp Term:2 Log:0/0 @@ -96,7 +96,7 @@ stabilize 2 3 > 2 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:2 Vote:2 Commit:4 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:4 Lead:2 LeadEpoch:1 Entries: 2/5 EntryNormal "" Messages: @@ -110,7 +110,7 @@ stabilize 2 3 DEBUG 3 [logterm: 0, index: 4] rejected MsgApp [logterm: 1, index: 4] from 2 > 3 handling Ready Ready MustSync=true: - HardState Term:2 Vote:2 Commit:3 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:3 Lead:2 LeadEpoch:1 Messages: 3->2 MsgFortifyLeaderResp Term:2 Log:0/0 Rejected (Hint: 0) 3->2 MsgAppResp Term:2 Log:1/4 Rejected (Hint: 3) @@ -133,7 +133,7 @@ stabilize 2 3 ] > 3 handling Ready Ready MustSync=true: - HardState Term:2 Vote:2 Commit:4 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:4 Lead:2 LeadEpoch:1 Entries: 1/4 EntryConfChangeV2 v3 2/5 EntryNormal "" @@ -146,7 +146,7 @@ stabilize 2 3 3->2 MsgAppResp Term:2 Log:0/5 > 2 handling Ready Ready MustSync=false: - HardState Term:2 Vote:2 Commit:5 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:5 Lead:2 LeadEpoch:1 CommittedEntries: 2/5 EntryNormal "" Messages: @@ -155,7 +155,7 @@ stabilize 2 3 2->3 MsgApp Term:2 Log:2/5 Commit:5 > 3 handling Ready Ready MustSync=false: - HardState Term:2 Vote:2 Commit:5 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:5 Lead:2 LeadEpoch:1 CommittedEntries: 2/5 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/checkquorum.txt b/pkg/raft/testdata/checkquorum.txt index 204ccfa8d676..7d694f612a45 100644 --- a/pkg/raft/testdata/checkquorum.txt +++ b/pkg/raft/testdata/checkquorum.txt @@ -37,7 +37,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StateCandidate - HardState Term:2 Vote:2 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:11 Lead:0 LeadEpoch:1 Messages: 2->1 MsgVote Term:2 Log:1/11 2->3 MsgVote Term:2 Log:1/11 @@ -128,7 +128,7 @@ stabilize INFO 1 [term: 2] ignored a MsgHeartbeatResp message with lower term from 3 [term: 1] > 1 handling Ready Ready MustSync=true: - HardState Term:2 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Commit:11 Lead:0 LeadEpoch:1 # Other nodes can now successfully campaign. Note that we haven't ticked 3, so # it won't grant votes. @@ -142,7 +142,7 @@ INFO 2 [logterm: 1, index: 11] sent MsgVote request to 3 at term 3 process-ready 2 ---- Ready MustSync=true: -HardState Term:3 Vote:2 Commit:11 Lead:0 LeadEpoch:0 +HardState Term:3 Vote:2 Commit:11 Lead:0 LeadEpoch:1 Messages: 2->1 MsgVote Term:3 Log:1/11 2->3 MsgVote Term:3 Log:1/11 @@ -165,7 +165,7 @@ stabilize ---- > 1 handling Ready Ready MustSync=true: - HardState Term:3 Vote:2 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:11 Lead:0 LeadEpoch:1 Messages: 1->2 MsgVoteResp Term:3 Log:0/0 > 2 receiving messages @@ -176,7 +176,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:3 Vote:2 Commit:11 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:11 Lead:2 LeadEpoch:1 Entries: 3/12 EntryNormal "" Messages: @@ -194,7 +194,7 @@ stabilize 2->3 MsgApp Term:3 Log:1/11 Commit:11 Entries:[3/12 EntryNormal ""] > 1 handling Ready Ready MustSync=true: - HardState Term:3 Vote:2 Commit:11 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:11 Lead:2 LeadEpoch:1 Entries: 3/12 EntryNormal "" Messages: @@ -202,7 +202,7 @@ stabilize 1->2 MsgAppResp Term:3 Log:0/12 > 3 handling Ready Ready MustSync=true: - HardState Term:3 Commit:11 Lead:2 LeadEpoch:0 + HardState Term:3 Commit:11 Lead:2 LeadEpoch:1 Entries: 3/12 EntryNormal "" Messages: @@ -215,7 +215,7 @@ stabilize 3->2 MsgAppResp Term:3 Log:0/12 > 2 handling Ready Ready MustSync=false: - HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:1 CommittedEntries: 3/12 EntryNormal "" Messages: @@ -227,14 +227,14 @@ stabilize 2->3 MsgApp Term:3 Log:3/12 Commit:12 > 1 handling Ready Ready MustSync=false: - HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:1 CommittedEntries: 3/12 EntryNormal "" Messages: 1->2 MsgAppResp Term:3 Log:0/12 > 3 handling Ready Ready MustSync=false: - HardState Term:3 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:3 Commit:12 Lead:2 LeadEpoch:1 CommittedEntries: 3/12 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/confchange_disable_validation.txt b/pkg/raft/testdata/confchange_disable_validation.txt index 373de2045f26..98ef9892a61b 100644 --- a/pkg/raft/testdata/confchange_disable_validation.txt +++ b/pkg/raft/testdata/confchange_disable_validation.txt @@ -42,7 +42,7 @@ stabilize 1 1/5 EntryConfChangeV2 l2 l3 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/4 EntryNormal "foo" > 1 handling Ready diff --git a/pkg/raft/testdata/confchange_drop_if_unapplied.txt b/pkg/raft/testdata/confchange_drop_if_unapplied.txt index 832e57fe0877..10fb2395d900 100644 --- a/pkg/raft/testdata/confchange_drop_if_unapplied.txt +++ b/pkg/raft/testdata/confchange_drop_if_unapplied.txt @@ -43,7 +43,7 @@ stabilize 1 ---- > 1 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 Entries: 1/5 EntryNormal "" CommittedEntries: @@ -51,7 +51,7 @@ stabilize 1 INFO 1 switched to configuration voters=(1)&&(1) learners=(2 3) > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/5 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/confchange_v1_add_single.txt b/pkg/raft/testdata/confchange_v1_add_single.txt index f24a13a64ae1..84b926de2c39 100644 --- a/pkg/raft/testdata/confchange_v1_add_single.txt +++ b/pkg/raft/testdata/confchange_v1_add_single.txt @@ -42,13 +42,13 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" 1/4 EntryConfChange v2 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" 1/4 EntryConfChange v2 diff --git a/pkg/raft/testdata/confchange_v1_remove_leader.txt b/pkg/raft/testdata/confchange_v1_remove_leader.txt index 83fc3b523283..59e8b9d51725 100644 --- a/pkg/raft/testdata/confchange_v1_remove_leader.txt +++ b/pkg/raft/testdata/confchange_v1_remove_leader.txt @@ -96,7 +96,7 @@ stabilize 1 2->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/4 EntryConfChange r1 1/5 EntryNormal "foo" @@ -122,7 +122,7 @@ stabilize 2 1->2 MsgApp Term:1 Log:1/6 Commit:5 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 Entries: 1/6 EntryNormal "bar" CommittedEntries: @@ -153,7 +153,7 @@ stabilize 1->3 MsgApp Term:1 Log:1/6 Commit:5 > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 Entries: 1/4 EntryConfChange r1 1/5 EntryNormal "foo" @@ -176,7 +176,7 @@ stabilize 3->1 MsgAppResp Term:1 Log:0/6 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:1 CommittedEntries: 1/6 EntryNormal "bar" Messages: @@ -188,14 +188,14 @@ stabilize 1->3 MsgApp Term:1 Log:1/6 Commit:6 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:1 CommittedEntries: 1/6 EntryNormal "bar" Messages: 2->1 MsgAppResp Term:1 Log:0/6 > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:1 CommittedEntries: 1/6 EntryNormal "bar" Messages: diff --git a/pkg/raft/testdata/confchange_v1_remove_leader_stepdown.txt b/pkg/raft/testdata/confchange_v1_remove_leader_stepdown.txt index 007444ada490..50b32c3901e3 100644 --- a/pkg/raft/testdata/confchange_v1_remove_leader_stepdown.txt +++ b/pkg/raft/testdata/confchange_v1_remove_leader_stepdown.txt @@ -95,7 +95,7 @@ stabilize 1 2->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/4 EntryConfChange r1 1/5 EntryNormal "foo" @@ -125,7 +125,7 @@ stabilize 2 1->2 MsgApp Term:1 Log:1/6 Commit:5 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 Entries: 1/6 EntryNormal "bar" CommittedEntries: @@ -156,7 +156,7 @@ stabilize 1->3 MsgApp Term:1 Log:1/6 Commit:5 > 3 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 Entries: 1/4 EntryConfChange r1 1/5 EntryNormal "foo" diff --git a/pkg/raft/testdata/confchange_v2_add_double_auto.txt b/pkg/raft/testdata/confchange_v2_add_double_auto.txt index 76e33b474e1f..b1f083cd7b5d 100644 --- a/pkg/raft/testdata/confchange_v2_add_double_auto.txt +++ b/pkg/raft/testdata/confchange_v2_add_double_auto.txt @@ -43,7 +43,7 @@ process-ready 1 ---- Ready MustSync=true: State:StateLeader -HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 +HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 v3 @@ -55,7 +55,7 @@ Entries: process-ready 1 ---- Ready MustSync=false: -HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 +HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 v3 @@ -133,7 +133,7 @@ stabilize 1 2 2->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: @@ -267,7 +267,7 @@ stabilize 1 3->1 MsgAppResp Term:1 Log:0/6 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:1 CommittedEntries: 1/6 EntryConfChangeV2 r2 r3 Messages: @@ -343,7 +343,7 @@ stabilize 3->1 MsgAppResp Term:1 Log:0/9 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:9 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:9 Lead:1 LeadEpoch:1 CommittedEntries: 1/7 EntryNormal "foo" 1/8 EntryNormal "bar" diff --git a/pkg/raft/testdata/confchange_v2_add_double_implicit.txt b/pkg/raft/testdata/confchange_v2_add_double_implicit.txt index 3c747cd8ecb0..8d08ec33512c 100644 --- a/pkg/raft/testdata/confchange_v2_add_double_implicit.txt +++ b/pkg/raft/testdata/confchange_v2_add_double_implicit.txt @@ -45,13 +45,13 @@ stabilize 1 2 > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 @@ -116,7 +116,7 @@ stabilize 1 2 2->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: diff --git a/pkg/raft/testdata/confchange_v2_add_single_auto.txt b/pkg/raft/testdata/confchange_v2_add_single_auto.txt index e032be859863..e2c9026bd48b 100644 --- a/pkg/raft/testdata/confchange_v2_add_single_auto.txt +++ b/pkg/raft/testdata/confchange_v2_add_single_auto.txt @@ -43,13 +43,13 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 diff --git a/pkg/raft/testdata/confchange_v2_add_single_explicit.txt b/pkg/raft/testdata/confchange_v2_add_single_explicit.txt index 8413305402d8..9b6ea40396ea 100644 --- a/pkg/raft/testdata/confchange_v2_add_single_explicit.txt +++ b/pkg/raft/testdata/confchange_v2_add_single_explicit.txt @@ -43,13 +43,13 @@ stabilize 1 2 > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:1 Entries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/3 EntryNormal "" 1/4 EntryConfChangeV2 v2 @@ -136,7 +136,7 @@ stabilize 2->1 MsgAppResp Term:1 Log:0/6 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:6 Lead:1 LeadEpoch:1 CommittedEntries: 1/5 EntryNormal "" 1/6 EntryConfChangeV2 @@ -187,7 +187,7 @@ stabilize 2->1 MsgAppResp Term:1 Log:0/7 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:7 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:7 Lead:1 LeadEpoch:1 CommittedEntries: 1/7 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/confchange_v2_replace_leader.txt b/pkg/raft/testdata/confchange_v2_replace_leader.txt index b11afdc0a93e..0b07247a83fc 100644 --- a/pkg/raft/testdata/confchange_v2_replace_leader.txt +++ b/pkg/raft/testdata/confchange_v2_replace_leader.txt @@ -77,7 +77,7 @@ stabilize 3->1 MsgAppResp Term:1 Log:0/4 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/4 EntryConfChangeV2 r1 v4 Messages: @@ -94,7 +94,7 @@ stabilize 1->4 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 r1 v4] > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/4 EntryConfChangeV2 r1 v4 Messages: @@ -102,7 +102,7 @@ stabilize INFO 2 switched to configuration voters=(2 3 4)&&(1 2 3) > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/4 EntryConfChangeV2 r1 v4 Messages: @@ -201,17 +201,17 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:2 Vote:4 Commit:4 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:4 Lead:0 LeadEpoch:1 Messages: 1->4 MsgVoteResp Term:2 Log:0/0 > 2 handling Ready Ready MustSync=true: - HardState Term:2 Vote:4 Commit:4 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:4 Lead:0 LeadEpoch:1 Messages: 2->4 MsgVoteResp Term:2 Log:0/0 > 3 handling Ready Ready MustSync=true: - HardState Term:2 Vote:4 Commit:4 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:4 Lead:0 LeadEpoch:1 Messages: 3->4 MsgVoteResp Term:2 Log:0/0 > 4 receiving messages @@ -226,7 +226,7 @@ stabilize > 4 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:1 Entries: 2/5 EntryNormal "" Messages: @@ -247,7 +247,7 @@ stabilize 4->3 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""] > 1 handling Ready Ready MustSync=true: - HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:1 Entries: 2/5 EntryNormal "" Messages: @@ -255,7 +255,7 @@ stabilize 1->4 MsgAppResp Term:2 Log:0/5 > 2 handling Ready Ready MustSync=true: - HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:1 Entries: 2/5 EntryNormal "" Messages: @@ -263,7 +263,7 @@ stabilize 2->4 MsgAppResp Term:2 Log:0/5 > 3 handling Ready Ready MustSync=true: - HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:4 Lead:4 LeadEpoch:1 Entries: 2/5 EntryNormal "" Messages: @@ -278,7 +278,7 @@ stabilize 3->4 MsgAppResp Term:2 Log:0/5 > 4 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:1 CommittedEntries: 2/5 EntryNormal "" Messages: @@ -293,21 +293,21 @@ stabilize 4->3 MsgApp Term:2 Log:2/5 Commit:5 > 1 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:1 CommittedEntries: 2/5 EntryNormal "" Messages: 1->4 MsgAppResp Term:2 Log:0/5 > 2 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:1 CommittedEntries: 2/5 EntryNormal "" Messages: 2->4 MsgAppResp Term:2 Log:0/5 > 3 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:5 Lead:4 LeadEpoch:1 CommittedEntries: 2/5 EntryNormal "" Messages: @@ -371,7 +371,7 @@ stabilize 3->4 MsgAppResp Term:2 Log:0/6 > 4 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:1 CommittedEntries: 2/6 EntryConfChangeV2 Messages: @@ -387,7 +387,7 @@ stabilize 4->3 MsgApp Term:2 Log:2/6 Commit:6 > 1 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:1 CommittedEntries: 2/6 EntryConfChangeV2 Messages: @@ -395,7 +395,7 @@ stabilize INFO 1 switched to configuration voters=(2 3 4) > 2 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:1 CommittedEntries: 2/6 EntryConfChangeV2 Messages: @@ -403,7 +403,7 @@ stabilize INFO 2 switched to configuration voters=(2 3 4) > 3 handling Ready Ready MustSync=false: - HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:0 + HardState Term:2 Vote:4 Commit:6 Lead:4 LeadEpoch:1 CommittedEntries: 2/6 EntryConfChangeV2 Messages: diff --git a/pkg/raft/testdata/confchange_v2_replace_leader_stepdown.txt b/pkg/raft/testdata/confchange_v2_replace_leader_stepdown.txt index 333031ef386c..267d553835c1 100644 --- a/pkg/raft/testdata/confchange_v2_replace_leader_stepdown.txt +++ b/pkg/raft/testdata/confchange_v2_replace_leader_stepdown.txt @@ -112,7 +112,7 @@ stabilize 4->1 MsgAppResp Term:1 Log:0/5 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: @@ -132,7 +132,7 @@ stabilize State:StateFollower > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: @@ -140,7 +140,7 @@ stabilize INFO 2 switched to configuration voters=(2 3 4) > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:5 Lead:1 LeadEpoch:1 CommittedEntries: 1/5 EntryConfChangeV2 Messages: diff --git a/pkg/raft/testdata/forget_leader.txt b/pkg/raft/testdata/forget_leader.txt index 058ea66e308a..65875ca3a87a 100644 --- a/pkg/raft/testdata/forget_leader.txt +++ b/pkg/raft/testdata/forget_leader.txt @@ -81,10 +81,10 @@ stabilize 1->4 MsgHeartbeat Term:1 Log:0/0 Commit:11 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:1 > 4 handling Ready Ready MustSync=true: - HardState Term:1 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:1 Commit:11 Lead:0 LeadEpoch:1 > 2 receiving messages 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 > 3 receiving messages @@ -93,7 +93,7 @@ stabilize 1->4 MsgHeartbeat Term:1 Log:0/0 Commit:11 > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:1 Messages: 2->1 MsgHeartbeatResp Term:1 Log:0/0 > 3 handling Ready @@ -102,7 +102,7 @@ stabilize 3->1 MsgHeartbeatResp Term:1 Log:0/0 > 4 handling Ready Ready MustSync=true: - HardState Term:1 Commit:11 Lead:1 LeadEpoch:0 + HardState Term:1 Commit:11 Lead:1 LeadEpoch:1 Messages: 4->1 MsgHeartbeatResp Term:1 Log:0/0 > 1 receiving messages diff --git a/pkg/raft/testdata/forget_leader_prevote_checkquorum.txt b/pkg/raft/testdata/forget_leader_prevote_checkquorum.txt index 1df1ab2fd182..a929b9f1ca21 100644 --- a/pkg/raft/testdata/forget_leader_prevote_checkquorum.txt +++ b/pkg/raft/testdata/forget_leader_prevote_checkquorum.txt @@ -36,7 +36,7 @@ stabilize 3 > 3 handling Ready Ready MustSync=true: State:StatePreCandidate - HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:1 Messages: 3->1 MsgPreVote Term:2 Log:1/11 3->2 MsgPreVote Term:2 Log:1/11 @@ -74,7 +74,7 @@ stabilize > 3 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:1 Messages: 3->1 MsgHeartbeatResp Term:1 Log:0/0 > 1 receiving messages @@ -111,7 +111,7 @@ stabilize 3 > 3 handling Ready Ready MustSync=true: State:StatePreCandidate - HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:1 Messages: 3->1 MsgPreVote Term:2 Log:1/11 3->2 MsgPreVote Term:2 Log:1/11 @@ -122,7 +122,7 @@ stabilize 2 ---- > 2 handling Ready Ready MustSync=true: - HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:1 > 2 receiving messages 3->2 MsgPreVote Term:2 Log:1/11 INFO 2 [logterm: 1, index: 11, vote: 1] cast MsgPreVote for 3 [logterm: 1, index: 11] at term 1 @@ -143,7 +143,7 @@ stabilize 3 > 3 handling Ready Ready MustSync=true: State:StateCandidate - HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:1 Messages: 3->1 MsgVote Term:2 Log:1/11 3->2 MsgVote Term:2 Log:1/11 @@ -208,7 +208,7 @@ process-ready 1 ---- Ready MustSync=true: State:StatePreCandidate -HardState Term:2 Commit:12 Lead:0 LeadEpoch:0 +HardState Term:2 Commit:12 Lead:0 LeadEpoch:1 Messages: 1->2 MsgPreVote Term:3 Log:2/12 1->3 MsgPreVote Term:3 Log:2/12 @@ -219,7 +219,7 @@ stabilize 2 ---- > 2 handling Ready Ready MustSync=true: - HardState Term:2 Vote:3 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:12 Lead:0 LeadEpoch:1 > 2 receiving messages 1->2 MsgPreVote Term:3 Log:2/12 INFO 2 [logterm: 2, index: 13, vote: 3] rejected MsgPreVote from 1 [logterm: 2, index: 12] at term 2 diff --git a/pkg/raft/testdata/fortification_basic.txt b/pkg/raft/testdata/fortification_basic.txt new file mode 100644 index 000000000000..20e3713aef50 --- /dev/null +++ b/pkg/raft/testdata/fortification_basic.txt @@ -0,0 +1,159 @@ +# Basic tests for leader fortification. + +log-level info +---- +ok + +add-nodes 3 voters=(1,2,3) index=2 +---- +INFO 1 switched to configuration voters=(1 2 3) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] +INFO 2 switched to configuration voters=(1 2 3) +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] +INFO 3 switched to configuration voters=(1 2 3) +INFO 3 became follower at term 0 +INFO newRaft 3 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +# Muck around with StoreLiveness to make it somewhat interesting. +bump-epoch 1 +---- + 1 2 3 +1 2 1 1 +2 2 1 1 +3 2 1 1 + +withdraw-support 1 1 +---- + 1 2 3 +1 x 1 1 +2 2 1 1 +3 2 1 1 + +grant-support 1 1 +---- + 1 2 3 +1 3 1 1 +2 2 1 1 +3 2 1 1 + +withdraw-support 3 1 +---- + 1 2 3 +1 3 1 1 +2 2 1 1 +3 x 1 1 + + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 +INFO 1 [logterm: 1, index: 2] sent MsgVote request to 2 at term 1 +INFO 1 [logterm: 1, index: 2] sent MsgVote request to 3 at term 1 + +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + State:StateCandidate + HardState Term:1 Vote:1 Commit:2 Lead:0 LeadEpoch:0 + Messages: + 1->2 MsgVote Term:1 Log:1/2 + 1->3 MsgVote Term:1 Log:1/2 + INFO 1 received MsgVoteResp from 1 at term 1 + INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections +> 2 receiving messages + 1->2 MsgVote Term:1 Log:1/2 + INFO 2 [term: 0] received a MsgVote message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + INFO 2 [logterm: 1, index: 2, vote: 0] cast MsgVote for 1 [logterm: 1, index: 2] at term 1 +> 3 receiving messages + 1->3 MsgVote Term:1 Log:1/2 + INFO 3 [term: 0] received a MsgVote message with higher term from 1 [term: 1] + INFO 3 became follower at term 1 + INFO 3 [logterm: 1, index: 2, vote: 0] cast MsgVote for 1 [logterm: 1, index: 2] at term 1 +> 2 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:0 LeadEpoch:0 + Messages: + 2->1 MsgVoteResp Term:1 Log:0/0 +> 3 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:0 LeadEpoch:0 + Messages: + 3->1 MsgVoteResp Term:1 Log:0/0 +> 1 receiving messages + 2->1 MsgVoteResp Term:1 Log:0/0 + INFO 1 received MsgVoteResp from 2 at term 1 + INFO 1 has received 2 MsgVoteResp votes and 0 vote rejections + INFO 1 became leader at term 1 + 3->1 MsgVoteResp Term:1 Log:0/0 +> 1 handling Ready + Ready MustSync=true: + State:StateLeader + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:3 + Entries: + 1/3 EntryNormal "" + Messages: + 1->2 MsgFortifyLeader Term:1 Log:0/0 + 1->3 MsgFortifyLeader Term:1 Log:0/0 + 1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] + 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] +> 2 receiving messages + 1->2 MsgFortifyLeader Term:1 Log:0/0 + 1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] +> 3 receiving messages + 1->3 MsgFortifyLeader Term:1 Log:0/0 + 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] +> 2 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:2 + Entries: + 1/3 EntryNormal "" + Messages: + 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 2->1 MsgAppResp Term:1 Log:0/3 +> 3 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + Entries: + 1/3 EntryNormal "" + Messages: + 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 3->1 MsgAppResp Term:1 Log:0/3 +> 1 receiving messages + 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 2->1 MsgAppResp Term:1 Log:0/3 + 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 3->1 MsgAppResp Term:1 Log:0/3 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:3 + CommittedEntries: + 1/3 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:3 + 1->3 MsgApp Term:1 Log:1/3 Commit:3 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:3 +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/3 Commit:3 +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:2 + CommittedEntries: + 1/3 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:0 + CommittedEntries: + 1/3 EntryNormal "" + Messages: + 3->1 MsgAppResp Term:1 Log:0/3 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/3 + 3->1 MsgAppResp Term:1 Log:0/3 diff --git a/pkg/raft/testdata/fortification_leader_does_not_support_itself.txt b/pkg/raft/testdata/fortification_leader_does_not_support_itself.txt new file mode 100644 index 000000000000..f11b0785c604 --- /dev/null +++ b/pkg/raft/testdata/fortification_leader_does_not_support_itself.txt @@ -0,0 +1,160 @@ +# This file tests the scenario where the leader doesn't support itself in the +# liveness fabric for whatever reason. + +log-level info +---- +ok + +add-nodes 3 voters=(1,2,3) index=2 +---- +INFO 1 switched to configuration voters=(1 2 3) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] +INFO 2 switched to configuration voters=(1 2 3) +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] +INFO 3 switched to configuration voters=(1 2 3) +INFO 3 became follower at term 0 +INFO newRaft 3 [peers: [1,2,3], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +# Make StoreLiveness interesting. Then, withdraw support for 1 from 1. +bump-epoch 1 +---- + 1 2 3 +1 2 1 1 +2 2 1 1 +3 2 1 1 + +withdraw-support 2 1 +---- + 1 2 3 +1 2 1 1 +2 x 1 1 +3 2 1 1 + +grant-support 2 1 +---- + 1 2 3 +1 3 1 1 +2 3 1 1 +3 2 1 1 + +withdraw-support 1 1 +---- + 1 2 3 +1 x 1 1 +2 3 1 1 +3 2 1 1 + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 +INFO 1 [logterm: 1, index: 2] sent MsgVote request to 2 at term 1 +INFO 1 [logterm: 1, index: 2] sent MsgVote request to 3 at term 1 + +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + State:StateCandidate + HardState Term:1 Vote:1 Commit:2 Lead:0 LeadEpoch:0 + Messages: + 1->2 MsgVote Term:1 Log:1/2 + 1->3 MsgVote Term:1 Log:1/2 + INFO 1 received MsgVoteResp from 1 at term 1 + INFO 1 has received 1 MsgVoteResp votes and 0 vote rejections +> 2 receiving messages + 1->2 MsgVote Term:1 Log:1/2 + INFO 2 [term: 0] received a MsgVote message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + INFO 2 [logterm: 1, index: 2, vote: 0] cast MsgVote for 1 [logterm: 1, index: 2] at term 1 +> 3 receiving messages + 1->3 MsgVote Term:1 Log:1/2 + INFO 3 [term: 0] received a MsgVote message with higher term from 1 [term: 1] + INFO 3 became follower at term 1 + INFO 3 [logterm: 1, index: 2, vote: 0] cast MsgVote for 1 [logterm: 1, index: 2] at term 1 +> 2 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:0 LeadEpoch:0 + Messages: + 2->1 MsgVoteResp Term:1 Log:0/0 +> 3 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:0 LeadEpoch:0 + Messages: + 3->1 MsgVoteResp Term:1 Log:0/0 +> 1 receiving messages + 2->1 MsgVoteResp Term:1 Log:0/0 + INFO 1 received MsgVoteResp from 2 at term 1 + INFO 1 has received 2 MsgVoteResp votes and 0 vote rejections + INFO 1 became leader at term 1 + INFO 1 leader at term 1 does not support itself in the liveness fabric + 3->1 MsgVoteResp Term:1 Log:0/0 +> 1 handling Ready + Ready MustSync=true: + State:StateLeader + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:0 + Entries: + 1/3 EntryNormal "" + Messages: + 1->2 MsgFortifyLeader Term:1 Log:0/0 + 1->3 MsgFortifyLeader Term:1 Log:0/0 + 1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] + 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] +> 2 receiving messages + 1->2 MsgFortifyLeader Term:1 Log:0/0 + 1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] +> 3 receiving messages + 1->3 MsgFortifyLeader Term:1 Log:0/0 + 1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""] +> 2 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:3 + Entries: + 1/3 EntryNormal "" + Messages: + 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 2->1 MsgAppResp Term:1 Log:0/3 +> 3 handling Ready + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:2 Lead:1 LeadEpoch:2 + Entries: + 1/3 EntryNormal "" + Messages: + 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 3->1 MsgAppResp Term:1 Log:0/3 +> 1 receiving messages + 2->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 2->1 MsgAppResp Term:1 Log:0/3 + 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) + 3->1 MsgAppResp Term:1 Log:0/3 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:0 + CommittedEntries: + 1/3 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:3 + 1->3 MsgApp Term:1 Log:1/3 Commit:3 +> 2 receiving messages + 1->2 MsgApp Term:1 Log:1/3 Commit:3 +> 3 receiving messages + 1->3 MsgApp Term:1 Log:1/3 Commit:3 +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:3 + CommittedEntries: + 1/3 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 +> 3 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:2 + CommittedEntries: + 1/3 EntryNormal "" + Messages: + 3->1 MsgAppResp Term:1 Log:0/3 +> 1 receiving messages + 2->1 MsgAppResp Term:1 Log:0/3 + 3->1 MsgAppResp Term:1 Log:0/3 diff --git a/pkg/raft/testdata/lagging_commit.txt b/pkg/raft/testdata/lagging_commit.txt index c23146b78740..16914f3346f7 100644 --- a/pkg/raft/testdata/lagging_commit.txt +++ b/pkg/raft/testdata/lagging_commit.txt @@ -78,7 +78,7 @@ stabilize 1 2 2->1 MsgAppResp Term:1 Log:0/13 > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1 CommittedEntries: 1/12 EntryNormal "data1" 1/13 EntryNormal "data2" @@ -92,7 +92,7 @@ stabilize 1 2 1->2 MsgApp Term:1 Log:1/13 Commit:13 > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1 CommittedEntries: 1/12 EntryNormal "data1" 1/13 EntryNormal "data2" @@ -164,7 +164,7 @@ stabilize 1 3 1->3 MsgApp Term:1 Log:1/13 Commit:13 > 3 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:13 Lead:1 LeadEpoch:1 CommittedEntries: 1/12 EntryNormal "data1" 1/13 EntryNormal "data2" diff --git a/pkg/raft/testdata/prevote.txt b/pkg/raft/testdata/prevote.txt index 8dcb3eac935d..973468911a02 100644 --- a/pkg/raft/testdata/prevote.txt +++ b/pkg/raft/testdata/prevote.txt @@ -67,7 +67,7 @@ process-ready 3 ---- Ready MustSync=true: State:StatePreCandidate -HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:0 +HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:1 Messages: 3->1 MsgPreVote Term:2 Log:1/11 3->2 MsgPreVote Term:2 Log:1/11 @@ -87,7 +87,7 @@ stabilize ---- > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:1 CommittedEntries: 1/12 EntryNormal "prop_1" Messages: @@ -108,7 +108,7 @@ stabilize 2->3 MsgPreVoteResp Term:1 Log:0/0 Rejected (Hint: 0) > 2 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:1 CommittedEntries: 1/12 EntryNormal "prop_1" Messages: @@ -116,7 +116,7 @@ stabilize > 3 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:12 Lead:1 LeadEpoch:1 Entries: 1/12 EntryNormal "prop_1" CommittedEntries: @@ -142,7 +142,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StatePreCandidate - HardState Term:1 Vote:1 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:12 Lead:0 LeadEpoch:1 Messages: 2->1 MsgPreVote Term:2 Log:1/12 2->3 MsgPreVote Term:2 Log:1/12 @@ -173,7 +173,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StateCandidate - HardState Term:2 Vote:2 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:12 Lead:0 LeadEpoch:1 Messages: 2->1 MsgVote Term:2 Log:1/12 2->3 MsgVote Term:2 Log:1/12 @@ -192,12 +192,12 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:2 Vote:2 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:12 Lead:0 LeadEpoch:1 Messages: 1->2 MsgVoteResp Term:2 Log:0/0 > 3 handling Ready Ready MustSync=true: - HardState Term:2 Vote:2 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:12 Lead:0 LeadEpoch:1 Messages: 3->2 MsgVoteResp Term:2 Log:0/0 > 2 receiving messages @@ -209,7 +209,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:2 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:12 Lead:2 LeadEpoch:1 Entries: 2/13 EntryNormal "" Messages: @@ -225,7 +225,7 @@ stabilize 2->3 MsgApp Term:2 Log:1/12 Commit:12 Entries:[2/13 EntryNormal ""] > 1 handling Ready Ready MustSync=true: - HardState Term:2 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:12 Lead:2 LeadEpoch:1 Entries: 2/13 EntryNormal "" Messages: @@ -233,7 +233,7 @@ stabilize 1->2 MsgAppResp Term:2 Log:0/13 > 3 handling Ready Ready MustSync=true: - HardState Term:2 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:12 Lead:2 LeadEpoch:1 Entries: 2/13 EntryNormal "" Messages: @@ -246,7 +246,7 @@ stabilize 3->2 MsgAppResp Term:2 Log:0/13 > 2 handling Ready Ready MustSync=false: - HardState Term:2 Vote:2 Commit:13 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:13 Lead:2 LeadEpoch:1 CommittedEntries: 2/13 EntryNormal "" Messages: @@ -258,14 +258,14 @@ stabilize 2->3 MsgApp Term:2 Log:2/13 Commit:13 > 1 handling Ready Ready MustSync=false: - HardState Term:2 Vote:2 Commit:13 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:13 Lead:2 LeadEpoch:1 CommittedEntries: 2/13 EntryNormal "" Messages: 1->2 MsgAppResp Term:2 Log:0/13 > 3 handling Ready Ready MustSync=false: - HardState Term:2 Vote:2 Commit:13 Lead:2 LeadEpoch:0 + HardState Term:2 Vote:2 Commit:13 Lead:2 LeadEpoch:1 CommittedEntries: 2/13 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/prevote_checkquorum.txt b/pkg/raft/testdata/prevote_checkquorum.txt index 9b0da76aa083..decde104d0fc 100644 --- a/pkg/raft/testdata/prevote_checkquorum.txt +++ b/pkg/raft/testdata/prevote_checkquorum.txt @@ -36,7 +36,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StatePreCandidate - HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:1 Messages: 2->1 MsgPreVote Term:2 Log:1/11 2->3 MsgPreVote Term:2 Log:1/11 @@ -70,7 +70,7 @@ process-ready 3 ---- Ready MustSync=true: State:StatePreCandidate -HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:0 +HardState Term:1 Vote:1 Commit:11 Lead:0 LeadEpoch:1 Messages: 3->1 MsgPreVote Term:2 Log:1/11 3->2 MsgPreVote Term:2 Log:1/11 @@ -103,7 +103,7 @@ stabilize > 3 handling Ready Ready MustSync=true: State:StateCandidate - HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:1 Messages: 3->1 MsgVote Term:2 Log:1/11 3->2 MsgVote Term:2 Log:1/11 @@ -120,7 +120,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:0 LeadEpoch:1 Messages: 2->3 MsgVoteResp Term:2 Log:0/0 > 3 receiving messages @@ -131,7 +131,7 @@ stabilize > 3 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:2 Vote:3 Commit:11 Lead:3 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:3 LeadEpoch:1 Entries: 2/12 EntryNormal "" Messages: @@ -150,7 +150,7 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:2 Commit:11 Lead:3 LeadEpoch:0 + HardState Term:2 Commit:11 Lead:3 LeadEpoch:1 Entries: 2/12 EntryNormal "" Messages: @@ -158,7 +158,7 @@ stabilize 1->3 MsgAppResp Term:2 Log:0/12 > 2 handling Ready Ready MustSync=true: - HardState Term:2 Vote:3 Commit:11 Lead:3 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:11 Lead:3 LeadEpoch:1 Entries: 2/12 EntryNormal "" Messages: @@ -171,7 +171,7 @@ stabilize 2->3 MsgAppResp Term:2 Log:0/12 > 3 handling Ready Ready MustSync=false: - HardState Term:2 Vote:3 Commit:12 Lead:3 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:12 Lead:3 LeadEpoch:1 CommittedEntries: 2/12 EntryNormal "" Messages: @@ -183,14 +183,14 @@ stabilize 3->2 MsgApp Term:2 Log:2/12 Commit:12 > 1 handling Ready Ready MustSync=false: - HardState Term:2 Commit:12 Lead:3 LeadEpoch:0 + HardState Term:2 Commit:12 Lead:3 LeadEpoch:1 CommittedEntries: 2/12 EntryNormal "" Messages: 1->3 MsgAppResp Term:2 Log:0/12 > 2 handling Ready Ready MustSync=false: - HardState Term:2 Vote:3 Commit:12 Lead:3 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:12 Lead:3 LeadEpoch:1 CommittedEntries: 2/12 EntryNormal "" Messages: @@ -219,7 +219,7 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StatePreCandidate - HardState Term:2 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:2 Commit:12 Lead:0 LeadEpoch:1 Messages: 1->2 MsgPreVote Term:3 Log:2/12 1->3 MsgPreVote Term:3 Log:2/12 @@ -244,7 +244,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StatePreCandidate - HardState Term:2 Vote:3 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:2 Vote:3 Commit:12 Lead:0 LeadEpoch:1 Messages: 2->1 MsgPreVote Term:3 Log:2/12 2->3 MsgPreVote Term:3 Log:2/12 @@ -270,7 +270,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StateCandidate - HardState Term:3 Vote:2 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:12 Lead:0 LeadEpoch:1 Messages: 2->1 MsgVote Term:3 Log:2/12 2->3 MsgVote Term:3 Log:2/12 @@ -287,7 +287,7 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:3 Vote:2 Commit:12 Lead:0 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:12 Lead:0 LeadEpoch:1 Messages: 1->2 MsgVoteResp Term:3 Log:0/0 > 2 receiving messages @@ -298,7 +298,7 @@ stabilize > 2 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:1 Entries: 3/13 EntryNormal "" Messages: @@ -316,7 +316,7 @@ stabilize 2->3 MsgApp Term:3 Log:2/12 Commit:12 Entries:[3/13 EntryNormal ""] > 1 handling Ready Ready MustSync=true: - HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:12 Lead:2 LeadEpoch:1 Entries: 3/13 EntryNormal "" Messages: @@ -325,7 +325,7 @@ stabilize > 3 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:3 Commit:12 Lead:2 LeadEpoch:0 + HardState Term:3 Commit:12 Lead:2 LeadEpoch:1 Entries: 3/13 EntryNormal "" Messages: @@ -338,7 +338,7 @@ stabilize 3->2 MsgAppResp Term:3 Log:0/13 > 2 handling Ready Ready MustSync=false: - HardState Term:3 Vote:2 Commit:13 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:13 Lead:2 LeadEpoch:1 CommittedEntries: 3/13 EntryNormal "" Messages: @@ -350,14 +350,14 @@ stabilize 2->3 MsgApp Term:3 Log:3/13 Commit:13 > 1 handling Ready Ready MustSync=false: - HardState Term:3 Vote:2 Commit:13 Lead:2 LeadEpoch:0 + HardState Term:3 Vote:2 Commit:13 Lead:2 LeadEpoch:1 CommittedEntries: 3/13 EntryNormal "" Messages: 1->2 MsgAppResp Term:3 Log:0/13 > 3 handling Ready Ready MustSync=false: - HardState Term:3 Commit:13 Lead:2 LeadEpoch:0 + HardState Term:3 Commit:13 Lead:2 LeadEpoch:1 CommittedEntries: 3/13 EntryNormal "" Messages: diff --git a/pkg/raft/testdata/probe_and_replicate.txt b/pkg/raft/testdata/probe_and_replicate.txt index 4e7535c76684..9b8af246b5aa 100644 --- a/pkg/raft/testdata/probe_and_replicate.txt +++ b/pkg/raft/testdata/probe_and_replicate.txt @@ -370,7 +370,7 @@ stabilize 1 > 1 handling Ready Ready MustSync=true: State:StateCandidate - HardState Term:8 Vote:1 Commit:18 Lead:0 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:18 Lead:0 LeadEpoch:1 Messages: 1->2 MsgVote Term:8 Log:6/20 1->3 MsgVote Term:8 Log:6/20 @@ -415,34 +415,34 @@ stabilize 2 3 4 5 6 7 INFO 7 [logterm: 3, index: 21, vote: 0] cast MsgVote for 1 [logterm: 6, index: 20] at term 8 > 2 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:18 Lead:0 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:18 Lead:0 LeadEpoch:1 Messages: 2->1 MsgVoteResp Term:8 Log:0/0 > 3 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:14 Lead:0 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:14 Lead:0 LeadEpoch:1 Messages: 3->1 MsgVoteResp Term:8 Log:0/0 > 4 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:8 Commit:18 Lead:0 LeadEpoch:0 + HardState Term:8 Commit:18 Lead:0 LeadEpoch:1 Messages: 4->1 MsgVoteResp Term:8 Log:0/0 Rejected (Hint: 0) > 5 handling Ready Ready MustSync=true: State:StateFollower - HardState Term:8 Commit:18 Lead:0 LeadEpoch:0 + HardState Term:8 Commit:18 Lead:0 LeadEpoch:1 Messages: 5->1 MsgVoteResp Term:8 Log:0/0 Rejected (Hint: 0) > 6 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:15 Lead:0 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:15 Lead:0 LeadEpoch:1 Messages: 6->1 MsgVoteResp Term:8 Log:0/0 > 7 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:13 Lead:0 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:13 Lead:0 LeadEpoch:1 Messages: 7->1 MsgVoteResp Term:8 Log:0/0 @@ -469,7 +469,7 @@ stabilize 1 > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:8 Vote:1 Commit:18 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:18 Lead:1 LeadEpoch:1 Entries: 8/21 EntryNormal "" Messages: @@ -494,7 +494,7 @@ stabilize 1 2 1->2 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] > 2 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:18 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:18 Lead:1 LeadEpoch:1 Messages: 2->1 MsgFortifyLeaderResp Term:8 Log:0/0 Rejected (Hint: 0) 2->1 MsgAppResp Term:8 Log:6/20 Rejected (Hint: 19) @@ -530,7 +530,7 @@ stabilize 1 3 1->3 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] > 3 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:14 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:14 Lead:1 LeadEpoch:1 Messages: 3->1 MsgFortifyLeaderResp Term:8 Log:0/0 Rejected (Hint: 0) 3->1 MsgAppResp Term:8 Log:4/20 Rejected (Hint: 14) @@ -561,7 +561,7 @@ stabilize 1 3 ] > 3 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:18 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:18 Lead:1 LeadEpoch:1 Entries: 4/15 EntryNormal "prop_4_15" 5/16 EntryNormal "" @@ -589,7 +589,7 @@ stabilize 1 4 INFO replace the unstable entries from index 21 > 4 handling Ready Ready MustSync=true: - HardState Term:8 Commit:18 Lead:1 LeadEpoch:0 + HardState Term:8 Commit:18 Lead:1 LeadEpoch:1 Entries: 8/21 EntryNormal "" Messages: @@ -600,7 +600,7 @@ stabilize 1 4 4->1 MsgAppResp Term:8 Log:0/21 > 1 handling Ready Ready MustSync=false: - HardState Term:8 Vote:1 Commit:21 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:21 Lead:1 LeadEpoch:1 CommittedEntries: 6/19 EntryNormal "prop_6_19" 6/20 EntryNormal "prop_6_20" @@ -613,7 +613,7 @@ stabilize 1 4 1->4 MsgApp Term:8 Log:8/21 Commit:21 > 4 handling Ready Ready MustSync=false: - HardState Term:8 Commit:21 Lead:1 LeadEpoch:0 + HardState Term:8 Commit:21 Lead:1 LeadEpoch:1 CommittedEntries: 6/19 EntryNormal "prop_6_19" 6/20 EntryNormal "prop_6_20" @@ -630,7 +630,7 @@ stabilize 1 5 1->5 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] > 5 handling Ready Ready MustSync=true: - HardState Term:8 Commit:18 Lead:1 LeadEpoch:0 + HardState Term:8 Commit:18 Lead:1 LeadEpoch:1 Messages: 5->1 MsgFortifyLeaderResp Term:8 Log:0/0 Rejected (Hint: 0) 5->1 MsgAppResp Term:8 Log:6/20 Rejected (Hint: 18) @@ -655,7 +655,7 @@ stabilize 1 5 INFO replace the unstable entries from index 19 > 5 handling Ready Ready MustSync=true: - HardState Term:8 Commit:21 Lead:1 LeadEpoch:0 + HardState Term:8 Commit:21 Lead:1 LeadEpoch:1 Entries: 6/19 EntryNormal "prop_6_19" 6/20 EntryNormal "prop_6_20" @@ -676,7 +676,7 @@ stabilize 1 6 1->6 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] > 6 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:15 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:15 Lead:1 LeadEpoch:1 Messages: 6->1 MsgFortifyLeaderResp Term:8 Log:0/0 Rejected (Hint: 0) 6->1 MsgAppResp Term:8 Log:4/20 Rejected (Hint: 17) @@ -707,7 +707,7 @@ stabilize 1 6 INFO replace the unstable entries from index 16 > 6 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:21 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:21 Lead:1 LeadEpoch:1 Entries: 5/16 EntryNormal "" 5/17 EntryNormal "prop_5_17" @@ -734,7 +734,7 @@ stabilize 1 7 1->7 MsgApp Term:8 Log:6/20 Commit:18 Entries:[8/21 EntryNormal ""] > 7 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:13 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:13 Lead:1 LeadEpoch:1 Messages: 7->1 MsgFortifyLeaderResp Term:8 Log:0/0 Rejected (Hint: 0) 7->1 MsgAppResp Term:8 Log:3/20 Rejected (Hint: 20) @@ -769,7 +769,7 @@ stabilize 1 7 INFO replace the unstable entries from index 14 > 7 handling Ready Ready MustSync=true: - HardState Term:8 Vote:1 Commit:21 Lead:1 LeadEpoch:0 + HardState Term:8 Vote:1 Commit:21 Lead:1 LeadEpoch:1 Entries: 4/14 EntryNormal "" 4/15 EntryNormal "prop_4_15" diff --git a/pkg/raft/testdata/single_node.txt b/pkg/raft/testdata/single_node.txt index 23a85c198f8e..f819261495b8 100644 --- a/pkg/raft/testdata/single_node.txt +++ b/pkg/raft/testdata/single_node.txt @@ -25,11 +25,11 @@ stabilize > 1 handling Ready Ready MustSync=true: State:StateLeader - HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:3 Lead:1 LeadEpoch:1 Entries: 1/4 EntryNormal "" > 1 handling Ready Ready MustSync=false: - HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:0 + HardState Term:1 Vote:1 Commit:4 Lead:1 LeadEpoch:1 CommittedEntries: 1/4 EntryNormal "" diff --git a/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt b/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt index 8ce4226d3227..23d8e09639a3 100644 --- a/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt +++ b/pkg/raft/testdata/snapshot_succeed_via_app_resp_behind.txt @@ -143,8 +143,8 @@ dropped: 1->3 MsgSnap Term:1 Log:0/0 stabilize 3 ---- > 3 handling Ready - Ready MustSync=false: - HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:0 + Ready MustSync=true: + HardState Term:1 Vote:1 Commit:11 Lead:1 LeadEpoch:1 Snapshot Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false Messages: 3->1 MsgFortifyLeaderResp Term:1 Log:0/0 Rejected (Hint: 0) diff --git a/pkg/raft/testdata/store_liveness_basic.txt b/pkg/raft/testdata/store_liveness_basic.txt new file mode 100644 index 000000000000..88f696ea068d --- /dev/null +++ b/pkg/raft/testdata/store_liveness_basic.txt @@ -0,0 +1,167 @@ +# This file has some basic tests that manipulate the store liveness fabric. +# They're more to test the directives themselves than anything else. + +add-nodes 2 index=10 +---- +INFO 1 switched to configuration voters=() +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [], term: 0, commit: 10, applied: 10, lastindex: 10, lastterm: 1] +INFO 2 switched to configuration voters=() +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [], term: 0, commit: 10, applied: 10, lastindex: 10, lastterm: 1] + +store-liveness +---- + 1 2 +1 1 1 +2 1 1 + +add-nodes 1 +---- +INFO 3 switched to configuration voters=() +INFO 3 became follower at term 0 +INFO newRaft 3 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +store-liveness +---- + 1 2 3 +1 1 1 1 +2 1 1 1 +3 1 1 1 + +# Bump epoch of store 1. This will result in it seeking support from all other +# stores at epoch 2, which they'll provide successfully. +bump-epoch 1 +---- + 1 2 3 +1 2 1 1 +2 2 1 1 +3 2 1 1 + +# Withdraw support from 3 for 2. +withdraw-support 3 2 +---- + 1 2 3 +1 2 1 1 +2 2 1 1 +3 2 x 1 + +# Withdraw support from 2 for 3. +withdraw-support 2 3 +---- + 1 2 3 +1 2 1 1 +2 2 1 x +3 2 x 1 + +# Grant support from 2 for 3. This causes 3 to bump its epoch and successfully +# seek support from 2 at the newer epoch (2). +grant-support 2 3 +---- + 1 2 3 +1 2 1 1 +2 2 1 2 +3 2 x 2 + + +# Repeat this process of withdrawing support and granting support from 2 for 3 +# another time to distinguish 3s epoch from 1s epoch. + +withdraw-support 2 3 +---- + 1 2 3 +1 2 1 1 +2 2 1 x +3 2 x 2 + +grant-support 2 3 +---- + 1 2 3 +1 2 1 1 +2 2 1 3 +3 2 x 3 + +# Finally, add another node in the mix. The node should start seeking support +# from all other stores at epoch 1. Moreover, it should provide support to store +# 1 at epoch 2 and store 3 at epoch 3. +add-nodes 1 +---- +INFO 4 switched to configuration voters=() +INFO 4 became follower at term 0 +INFO newRaft 4 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +store-liveness +---- + 1 2 3 4 +1 2 1 1 1 +2 2 1 3 1 +3 2 x 3 1 +4 2 1 3 1 + +# Ensure a store can withdraw support from itself. + +withdraw-support 4 4 +---- + 1 2 3 4 +1 2 1 1 1 +2 2 1 3 1 +3 2 x 3 1 +4 2 1 3 x + +grant-support 4 4 +---- + 1 2 3 4 +1 2 1 1 1 +2 2 1 3 1 +3 2 x 3 1 +4 2 1 3 2 + +# Ensure epochs do not regress, and ensure that whenever support is granted, it +# happens at the latest epoch that for_store is seeking support at. +withdraw-support 1 2 +---- + 1 2 3 4 +1 2 x 1 1 +2 2 1 3 1 +3 2 x 3 1 +4 2 1 3 2 + +# Withdraw support from 1 for 2 again. Internally, it'll bump the epoch 1 is +# tracking for 2 twice (so it'll be 3 now). +withdraw-support 1 2 +---- + 1 2 3 4 +1 2 x 1 1 +2 2 1 3 1 +3 2 x 3 1 +4 2 1 3 2 + +# Withdraw support from 4 for 2 now. Internally, it'll bump 2's epoch to 2. +withdraw-support 4 2 +---- + 1 2 3 4 +1 2 x 1 1 +2 2 1 3 1 +3 2 x 3 1 +4 2 x 3 2 + +# Grant support from 1 for 2; this will happen at epoch 3, which should also be +# mirrored at state[2][2]. +grant-support 1 2 +---- + 1 2 3 4 +1 2 3 1 1 +2 2 3 3 1 +3 2 x 3 1 +4 2 x 3 2 + +# Grant support from 4 for 2. This could have happened at epoch 2, but it will +# happen at epoch 3 instead because of the previous step. Also, note that +# state[2][2] doesn't regress. +grant-support 4 2 +---- + 1 2 3 4 +1 2 3 1 1 +2 2 3 3 1 +3 2 x 3 1 +4 2 3 3 2