-
Notifications
You must be signed in to change notification settings - Fork 25k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add node REPLACE shutdown implementation #76247
Changes from 9 commits
d1b8218
fd0c597
41c6238
a96e596
238112a
2402803
ff30a2a
47500c7
9ded9f3
f3613bc
7b4c518
4ce00ef
27ebeeb
cb1c0c4
c2571ee
fe1889c
4f0da9c
d659b11
448968b
e83b630
df91ec1
5724f2a
c3f6c23
bc4c78b
21e9f9e
f7c407f
06320e3
4ab4496
e8773b7
658a721
d4c8650
fa92698
c27ea49
c237f95
f692e63
f31c949
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -294,7 +294,7 @@ public void testNodeReplacementOverridesFilters() throws Exception { | |
.put("index.number_of_replicas", 0); | ||
createIndex("myindex", nodeASettings.build()); | ||
final String nodeAId = getNodeId(nodeA); | ||
final String nodeB = "node_t1"; // TODO: fix this to so it's actually overrideable | ||
final String nodeB = "node_t2"; // TODO: fix this to so it's actually overrideable | ||
|
||
// Mark the nodeA as being replaced | ||
PutShutdownNodeAction.Request putShutdownRequest = new PutShutdownNodeAction.Request( | ||
|
@@ -314,6 +314,7 @@ public void testNodeReplacementOverridesFilters() throws Exception { | |
|
||
assertThat(getResp.getShutdownStatuses().get(0).migrationStatus().getStatus(), equalTo(STALLED)); | ||
|
||
final String nodeC = internalCluster().startNode(); | ||
internalCluster().startNode(Settings.builder().put("node.name", nodeB)); | ||
final String nodeBId = getNodeId(nodeB); | ||
|
||
|
@@ -339,8 +340,6 @@ public void testNodeReplacementOverridesFilters() throws Exception { | |
assertThat(shutdownStatus.getShutdownStatuses().get(0).migrationStatus().getStatus(), equalTo(COMPLETE)); | ||
}); | ||
|
||
final String nodeC = internalCluster().startNode(); | ||
|
||
createIndex("other", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 1).build()); | ||
|
||
ensureYellow("other"); | ||
|
@@ -382,6 +381,65 @@ public void testNodeReplacementOverridesFilters() throws Exception { | |
}, () -> fail("expected a 'NO' decision for nodeB but there was no explanation for that node")); | ||
} | ||
|
||
public void testNodeReplacementOnlyToTarget() throws Exception { | ||
String nodeA = internalCluster().startNode( | ||
Settings.builder().put("node.name", "node-a").put("cluster.routing.rebalance.enable", "none") | ||
); | ||
// Create an index and pin it to nodeA, when we replace it with nodeB, | ||
// it'll move the data, overridding the `_name` allocation filter | ||
Settings.Builder nodeASettings = Settings.builder().put("index.number_of_shards", 4).put("index.number_of_replicas", 0); | ||
createIndex("myindex", nodeASettings.build()); | ||
final String nodeAId = getNodeId(nodeA); | ||
final String nodeB = "node_t1"; // TODO: fix this to so it's actually overrideable | ||
final String nodeC = "node_t2"; // TODO: fix this to so it's actually overrideable | ||
|
||
// Mark the nodeA as being replaced | ||
PutShutdownNodeAction.Request putShutdownRequest = new PutShutdownNodeAction.Request( | ||
nodeAId, | ||
SingleNodeShutdownMetadata.Type.REPLACE, | ||
this.getTestName(), | ||
null, | ||
nodeB | ||
); | ||
AcknowledgedResponse putShutdownResponse = client().execute(PutShutdownNodeAction.INSTANCE, putShutdownRequest).get(); | ||
assertTrue(putShutdownResponse.isAcknowledged()); | ||
|
||
GetShutdownStatusAction.Response getResp = client().execute( | ||
GetShutdownStatusAction.INSTANCE, | ||
new GetShutdownStatusAction.Request(nodeAId) | ||
).get(); | ||
|
||
assertThat(getResp.getShutdownStatuses().get(0).migrationStatus().getStatus(), equalTo(STALLED)); | ||
|
||
internalCluster().startNode(Settings.builder().put("node.name", nodeB)); | ||
internalCluster().startNode(Settings.builder().put("node.name", nodeC)); | ||
Comment on lines
+412
to
+413
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I wonder if we can swap the order of starting nodes? Suppose we allowed allocating to both nodes (a bug). But we then risk the shards moving over to B (they only have to start the relocation) before node C have started? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We do have another test that starts nodeC prior to nodeB for the replacement (it was one of the changes you asked for), which I think satisfies this? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah, I did notice that and thanks for doing that. I was only adding this comment here because this test is named |
||
final String nodeBId = getNodeId(nodeB); | ||
final String nodeCId = getNodeId(nodeC); | ||
|
||
logger.info("--> NodeA: {} -- {}", nodeA, nodeAId); | ||
logger.info("--> NodeB: {} -- {}", nodeB, nodeBId); | ||
logger.info("--> NodeC: {} -- {}", nodeC, nodeCId); | ||
|
||
assertBusy(() -> { | ||
ClusterState state = client().admin().cluster().prepareState().clear().setRoutingTable(true).get().getState(); | ||
for (ShardRouting sr : state.routingTable().allShards("myindex")) { | ||
assertThat( | ||
"expected all shards for index to be on node B (" + nodeBId + ") but " + sr.toString() + " is on " + sr.currentNodeId(), | ||
sr.currentNodeId(), | ||
equalTo(nodeBId) | ||
); | ||
} | ||
}); | ||
|
||
assertBusy(() -> { | ||
GetShutdownStatusAction.Response shutdownStatus = client().execute( | ||
GetShutdownStatusAction.INSTANCE, | ||
new GetShutdownStatusAction.Request(nodeAId) | ||
).get(); | ||
assertThat(shutdownStatus.getShutdownStatuses().get(0).migrationStatus().getStatus(), equalTo(COMPLETE)); | ||
}); | ||
} | ||
|
||
private void indexRandomData() throws Exception { | ||
int numDocs = scaledRandomIntBetween(100, 1000); | ||
IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs]; | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can we update this comment, looks like a copy-paste error, since we do not pin the index here?