How to handle committed ConfChangeV2

[haraldng]

Is it possible to provide an example on how to handle a committed ConfChangeV2 Entry? The example in single_mem_node is still a TODO and in five_mem_node still uses the old ConfChange.

Thanks in advance

[BusyJay]

It depends on how would you like to exit joint state. If auto_leave is configured, then raft node should propose a leave joint command when enter join is applied, which is informed by apply_conf_change. All you need to do is maintaining the metadata according the the committed entries.

If auto_leave is set to false, then you need to propose a leave joint command, which is just an empty conf change, manually.

I hope tests code can help for now example is missing:

raft-rs/harness/tests/integration_cases/test_raw_node.rs

Lines 176 to 462 in 7c21f8d

	/// Tests the configuration change mechanism. Each test case sends a configuration
	/// change which is either simple or joint, verifies that it applies and that the
	/// resulting ConfState matches expectations, and for joint configurations makes
	/// sure that they are exited successfully.
	#[test]
	fn test_raw_node_propose_and_conf_change() {
	let l = default_logger();
	let mut test_cases: Vec<(Box<dyn ConfChangeI>, _, _)> = vec![
	// V1 config change.
	(
	Box::new(conf_change(ConfChangeType::AddNode, 2)),
	conf_state(vec![1, 2], vec![]),
	None,
	),
	];
	// Proposing the same as a V2 change works just the same, without entering
	// a joint config.
	let single = new_conf_change_single(2, ConfChangeType::AddNode);
	test_cases.push((
	Box::new(conf_change_v2(vec![single])),
	conf_state(vec![1, 2], vec![]),
	None,
	));
	// Ditto if we add it as a learner instead.
	let single = new_conf_change_single(2, ConfChangeType::AddLearnerNode);
	test_cases.push((
	Box::new(conf_change_v2(vec![single])),
	conf_state(vec![1], vec![2]),
	None,
	));
	// We can ask explicitly for joint consensus if we want it.
	let single = new_conf_change_single(2, ConfChangeType::AddLearnerNode);
	let mut cc = conf_change_v2(vec![single]);
	cc.set_transition(ConfChangeTransition::Explicit);
	let cs = conf_state_v2(vec![1], vec![2], vec![1], vec![], false);
	test_cases.push((Box::new(cc), cs, Some(conf_state(vec![1], vec![2]))));
	// Ditto, but with implicit transition (the harness checks this).
	let single = new_conf_change_single(2, ConfChangeType::AddLearnerNode);
	let mut cc = conf_change_v2(vec![single]);
	cc.set_transition(ConfChangeTransition::Implicit);
	let cs = conf_state_v2(vec![1], vec![2], vec![1], vec![], true);
	test_cases.push((Box::new(cc), cs, Some(conf_state(vec![1], vec![2]))));
	// Add a new node and demote n1. This exercises the interesting case in
	// which we really need joint config changes and also need LearnersNext.
	let cc = conf_change_v2(vec![
	new_conf_change_single(2, ConfChangeType::AddNode),
	new_conf_change_single(1, ConfChangeType::AddLearnerNode),
	new_conf_change_single(3, ConfChangeType::AddLearnerNode),
	]);
	let cs = conf_state_v2(vec![2], vec![3], vec![1], vec![1], true);
	test_cases.push((Box::new(cc), cs, Some(conf_state(vec![2], vec![1, 3]))));
	// Ditto explicit.
	let mut cc = conf_change_v2(vec![
	new_conf_change_single(2, ConfChangeType::AddNode),
	new_conf_change_single(1, ConfChangeType::AddLearnerNode),
	new_conf_change_single(3, ConfChangeType::AddLearnerNode),
	]);
	cc.set_transition(ConfChangeTransition::Explicit);
	let cs = conf_state_v2(vec![2], vec![3], vec![1], vec![1], false);
	test_cases.push((Box::new(cc), cs, Some(conf_state(vec![2], vec![1, 3]))));
	// Ditto implicit.
	let mut cc = conf_change_v2(vec![
	new_conf_change_single(2, ConfChangeType::AddNode),
	new_conf_change_single(1, ConfChangeType::AddLearnerNode),
	new_conf_change_single(3, ConfChangeType::AddLearnerNode),
	]);
	cc.set_transition(ConfChangeTransition::Implicit);
	let cs = conf_state_v2(vec![2], vec![3], vec![1], vec![1], true);
	test_cases.push((Box::new(cc), cs, Some(conf_state(vec![2], vec![1, 3]))));
	for (cc, exp, exp2) in test_cases {
	let s = new_storage();
	let mut raw_node = new_raw_node(1, vec![1], 10, 1, s.clone(), &l);
	raw_node.campaign().unwrap();
	let mut proposed = false;
	let mut ccdata = vec![];
	// Propose the ConfChange, wait until it applies, save the resulting ConfState.
	let mut cs = None;
	while cs.is_none() {
	let mut rd = raw_node.ready();
	s.wl().append(rd.entries()).unwrap();
	let mut handle_committed_entries =
	|rn: &mut RawNode<MemStorage>, committed_entries: Vec<Entry>| {
	for e in committed_entries {
	if e.get_entry_type() == EntryType::EntryConfChange {
	let mut cc = ConfChange::default();
	cc.merge_from_bytes(e.get_data()).unwrap();
	cs = Some(rn.apply_conf_change(&cc).unwrap());
	} else if e.get_entry_type() == EntryType::EntryConfChangeV2 {
	let mut cc = ConfChangeV2::default();
	cc.merge_from_bytes(e.get_data()).unwrap();
	cs = Some(rn.apply_conf_change(&cc).unwrap());
	}
	}
	};
	handle_committed_entries(&mut raw_node, rd.take_committed_entries());
	let is_leader = rd.ss().map_or(false, |ss| ss.leader_id == raw_node.raft.id);
	let mut light_rd = raw_node.advance(rd);
	handle_committed_entries(&mut raw_node, light_rd.take_committed_entries());
	raw_node.advance_apply();
	// Once we are the leader, propose a command and a ConfChange.
	if !proposed && is_leader {
	raw_node.propose(vec![], b"somedata".to_vec()).unwrap();
	if let Some(v1) = cc.as_v1() {
	ccdata = v1.write_to_bytes().unwrap();
	raw_node.propose_conf_change(vec![], v1.clone()).unwrap();
	} else {
	let v2 = cc.as_v2().clone().into_owned();
	ccdata = v2.write_to_bytes().unwrap();
	raw_node.propose_conf_change(vec![], v2).unwrap();
	}
	proposed = true;
	}
	}
	// Check that the last index is exactly the conf change we put in,
	// down to the bits. Note that this comes from the Storage, which
	// will not reflect any unstable entries that we'll only be presented
	// with in the next Ready.
	let last_index = s.last_index().unwrap();
	let entries = s.entries(last_index - 1, last_index + 1, NO_LIMIT).unwrap();
	assert_eq!(entries.len(), 2);
	assert_eq!(entries[0].get_data(), b"somedata");
	if cc.as_v1().is_some() {
	assert_eq!(entries[1].get_entry_type(), EntryType::EntryConfChange);
	} else {
	assert_eq!(entries[1].get_entry_type(), EntryType::EntryConfChangeV2);
	}
	assert_eq!(ccdata, entries[1].get_data());
	assert_eq!(exp, cs.unwrap());
	let conf_index = if cc.as_v2().enter_joint() == Some(true) {
	// If this is an auto-leaving joint conf change, it will have
	// appended the entry that auto-leaves, so add one to the last
	// index that forms the basis of our expectations on
	// pendingConfIndex. (Recall that lastIndex was taken from stable
	// storage, but this auto-leaving entry isn't on stable storage
	// yet).
	last_index + 1
	} else {
	last_index
	};
	assert_eq!(conf_index, raw_node.raft.pending_conf_index);
	// Move the RawNode along. If the ConfChange was simple, nothing else
	// should happen. Otherwise, we're in a joint state, which is either
	// left automatically or not. If not, we add the proposal that leaves
	// it manually.
	let mut rd = raw_node.ready();
	let mut context = vec![];
	if !exp.auto_leave {
	assert!(rd.entries().is_empty());
	if exp2.is_none() {
	continue;
	}
	context = b"manual".to_vec();
	let mut cc = conf_change_v2(vec![]);
	cc.set_context(context.clone().into());
	raw_node.propose_conf_change(vec![], cc).unwrap();
	rd = raw_node.ready();
	}
	// Check that the right ConfChange comes out.
	assert_eq!(rd.entries().len(), 1);
	assert_eq!(
	rd.entries()[0].get_entry_type(),
	EntryType::EntryConfChangeV2
	);
	let mut leave_cc = ConfChangeV2::default();
	leave_cc
	.merge_from_bytes(rd.entries()[0].get_data())
	.unwrap();
	assert_eq!(context, leave_cc.get_context(), "{:?}", cc.as_v2());
	// Lie and pretend the ConfChange applied. It won't do so because now
	// we require the joint quorum and we're only running one node.
	let cs = raw_node.apply_conf_change(&leave_cc).unwrap();
	assert_eq!(cs, exp2.unwrap());
	}
	}
	/// Tests the configuration change auto leave even leader lost leadership.
	#[test]
	fn test_raw_node_joint_auto_leave() {
	let l = default_logger();
	let single = new_conf_change_single(2, ConfChangeType::AddLearnerNode);
	let mut test_cc = conf_change_v2(vec![single]);
	test_cc.set_transition(ConfChangeTransition::Implicit);
	let exp_cs = conf_state_v2(vec![1], vec![2], vec![1], vec![], true);
	let exp_cs2 = conf_state(vec![1], vec![2]);
	let s = new_storage();
	let mut raw_node = new_raw_node(1, vec![1], 10, 1, s.clone(), &l);
	raw_node.campaign().unwrap();
	let mut proposed = false;
	let ccdata = test_cc.write_to_bytes().unwrap();
	// Propose the ConfChange, wait until it applies, save the resulting ConfState.
	let mut cs = None;
	while cs.is_none() {
	let mut rd = raw_node.ready();
	s.wl().append(rd.entries()).unwrap();
	let mut handle_committed_entries =
	|rn: &mut RawNode<MemStorage>, committed_entries: Vec<Entry>| {
	for e in committed_entries {
	if e.get_entry_type() == EntryType::EntryConfChangeV2 {
	let mut cc = ConfChangeV2::default();
	cc.merge_from_bytes(e.get_data()).unwrap();
	// Force it step down.
	let mut msg = new_message(1, 1, MessageType::MsgHeartbeatResponse, 0);
	msg.term = rn.raft.term + 1;
	rn.step(msg).unwrap();
	cs = Some(rn.apply_conf_change(&cc).unwrap());
	}
	}
	};
	handle_committed_entries(&mut raw_node, rd.take_committed_entries());
	let is_leader = rd.ss().map_or(false, |ss| ss.leader_id == raw_node.raft.id);
	let mut light_rd = raw_node.advance(rd);
	handle_committed_entries(&mut raw_node, light_rd.take_committed_entries());
	raw_node.advance_apply();
	// Once we are the leader, propose a command and a ConfChange.
	if !proposed && is_leader {
	raw_node.propose(vec![], b"somedata".to_vec()).unwrap();
	raw_node
	.propose_conf_change(vec![], test_cc.clone())
	.unwrap();
	proposed = true;
	}
	}
	// Check that the last index is exactly the conf change we put in,
	// down to the bits. Note that this comes from the Storage, which
	// will not reflect any unstable entries that we'll only be presented
	// with in the next Ready.
	let last_index = s.last_index().unwrap();
	let entries = s.entries(last_index - 1, last_index + 1, NO_LIMIT).unwrap();
	assert_eq!(entries.len(), 2);
	assert_eq!(entries[0].get_data(), b"somedata");
	assert_eq!(entries[1].get_entry_type(), EntryType::EntryConfChangeV2);
	assert_eq!(ccdata, entries[1].get_data());
	assert_eq!(exp_cs, cs.unwrap());
	assert_eq!(0, raw_node.raft.pending_conf_index);
	// Move the RawNode along. It should not leave joint because it's follower.
	let mut rd = raw_node.ready();
	assert!(rd.entries().is_empty());
	let _ = raw_node.advance(rd);
	// Make it leader again. It should leave joint automatically after moving apply index.
	raw_node.campaign().unwrap();
	rd = raw_node.ready();
	s.wl().append(rd.entries()).unwrap();
	let _ = raw_node.advance(rd);
	rd = raw_node.ready();
	s.wl().append(rd.entries()).unwrap();
	// Check that the right ConfChange comes out.
	assert_eq!(rd.entries().len(), 1);
	assert_eq!(
	rd.entries()[0].get_entry_type(),
	EntryType::EntryConfChangeV2
	);
	let mut leave_cc = ConfChangeV2::default();
	leave_cc
	.merge_from_bytes(rd.entries()[0].get_data())
	.unwrap();
	assert!(leave_cc.get_context().is_empty());
	// Lie and pretend the ConfChange applied. It won't do so because now
	// we require the joint quorum and we're only running one node.
	let cs = raw_node.apply_conf_change(&leave_cc).unwrap();
	assert_eq!(cs, exp_cs2);
	}

[haraldng]

I don't understand how to auto leave the joint state. What metadata need to be maintained after performing apply_conf_change and what actions need to be performed?