diff --git a/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java b/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java index 858108ccff560..bc77c0f286056 100644 --- a/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java +++ b/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.logging.ESLoggerFactory; import java.util.Arrays; import java.util.List; @@ -105,6 +106,9 @@ public List indicesCreated() { * Returns the indices deleted in this event */ public List indicesDeleted() { + if (newMaster()) { + return ImmutableList.of(); + } if (previousState == null) { return ImmutableList.of(); } @@ -165,4 +169,16 @@ public boolean nodesAdded() { public boolean nodesChanged() { return nodesRemoved() || nodesAdded(); } + + public boolean newMaster() { + String oldMaster = previousState().getNodes().masterNodeId(); + String newMaster = state().getNodes().masterNodeId(); + if (oldMaster == null && newMaster == null) { + return false; + } + if (oldMaster == null && newMaster != null) { + return true; + } + return previousState().getNodes().masterNodeId().equals(state().getNodes().masterNodeId()) == false; + } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 34793f214b576..282ec2c51e499 100644 --- a/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -19,6 +19,9 @@ package org.elasticsearch.gateway; +import com.carrotsearch.hppc.cursors.IntObjectCursor; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchIllegalStateException; import org.elasticsearch.Version; @@ -28,16 +31,16 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.DjbHashFunction; -import org.elasticsearch.cluster.routing.HashFunction; -import org.elasticsearch.cluster.routing.SimpleHashFunction; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.index.Index; import java.io.IOException; import java.nio.file.DirectoryStream; @@ -72,7 +75,7 @@ public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateSer if (DiscoveryNode.masterNode(settings) || DiscoveryNode.dataNode(settings)) { nodeEnv.ensureAtomicMoveSupported(); } - if (DiscoveryNode.masterNode(settings)) { + if (DiscoveryNode.masterNode(settings) || DiscoveryNode.dataNode(settings)) { try { ensureNoPre019State(); pre20Upgrade(); @@ -103,8 +106,8 @@ public void clusterChanged(ClusterChangedEvent event) { // we don't check if metaData changed, since we might be called several times and we need to check dangling... boolean success = true; - // only applied to master node, writing the global and index level states - if (state.nodes().localNode().masterNode()) { + // write the state if this node is a master eligible node or if it is a data node and has shards allocated on it + if (state.nodes().localNode().masterNode() || state.nodes().localNode().dataNode()) { // check if the global state changed? if (currentMetaData == null || !MetaData.isGlobalStateEquals(currentMetaData, newMetaData)) { try { @@ -116,6 +119,22 @@ public void clusterChanged(ClusterChangedEvent event) { // check and write changes in indices for (IndexMetaData indexMetaData : newMetaData) { + + boolean shardsAllocatedOnThisNodeInLastClusterState = true; + if (isDataOnlyNode(state)) { + boolean shardsCurrentlyAllocatedOnThisNode = shardsAllocatedOnLocalNode(state, indexMetaData); + shardsAllocatedOnThisNodeInLastClusterState = shardsAllocatedOnLocalNode(event.previousState(), indexMetaData); + + if (shardsCurrentlyAllocatedOnThisNode == false) { + // remove the index state for this index if it is only a data node + // only delete if the last shard was removed + if (shardsAllocatedOnThisNodeInLastClusterState) { + removeIndexState(indexMetaData); + } + // nothing left to do, we do not write the index state for data only nodes if they do not have shards allocated on them + continue; + } + } String writeReason = null; IndexMetaData currentIndexMetaData; if (currentMetaData == null) { @@ -132,6 +151,9 @@ public void clusterChanged(ClusterChangedEvent event) { writeReason = "freshly created"; } else if (currentIndexMetaData.version() != indexMetaData.version()) { writeReason = "version changed from [" + currentIndexMetaData.version() + "] to [" + indexMetaData.version() + "]"; + } else if (shardsAllocatedOnThisNodeInLastClusterState == false && isDataOnlyNode(state)) { + // shard was newly allocated because it was not allocated in last cluster state but is now + writeReason = "shard allocated on data only node"; } // we update the writeReason only if we really need to write it @@ -154,6 +176,43 @@ public void clusterChanged(ClusterChangedEvent event) { } } + protected boolean shardsAllocatedOnLocalNode(ClusterState state, IndexMetaData indexMetaData) { + boolean shardsAllocatedOnThisNode = false; + IndexRoutingTable indexRoutingTable = state.getRoutingTable().index(indexMetaData.index()); + if (indexRoutingTable == null) { + // nothing allocated ? + return false; + } + // iterate over shards and see if one is on our node + for (IntObjectCursor it : indexRoutingTable.shards()) { + IndexShardRoutingTable shardRoutingTable = (IndexShardRoutingTable) it.value; + for (ShardRouting shardRouting : shardRoutingTable.shards()) { + if (shardRouting.currentNodeId() != null && shardRouting.currentNodeId().equals(state.nodes().localNode().getId())) { + shardsAllocatedOnThisNode = true; + } + } + } + return shardsAllocatedOnThisNode; + } + + protected boolean isDataOnlyNode(ClusterState state) { + return ((state.nodes().localNode().masterNode() == false) && (state.nodes().localNode().dataNode() == true)); + } + + private void removeIndexState(IndexMetaData indexMetaData) { + final MetaDataStateFormat writer = indexStateFormat(format, formatParams, true); + try { + Path[] locations = nodeEnv.indexPaths(new Index(indexMetaData.index())); + Preconditions.checkArgument(locations != null, "Locations must not be null"); + Preconditions.checkArgument(locations.length > 0, "One or more locations required"); + writer.cleanupOldFiles(INDEX_STATE_FILE_PREFIX, null, locations); + logger.debug("successfully deleted state for {}", indexMetaData.getIndex()); + } catch (Throwable ex) { + logger.warn("[{}]: failed to delete index state", ex, indexMetaData.index()); + // and now what? + } + } + /** * Throws an IAE if a pre 0.19 state is detected */ diff --git a/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java index 56fb1a97505a2..e3c5da906c695 100644 --- a/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java +++ b/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java @@ -190,14 +190,14 @@ protected Directory newDirectory(Path dir) throws IOException { return new SimpleFSDirectory(dir); } - private void cleanupOldFiles(final String prefix, final String currentStateFile, Path[] locations) throws IOException { + protected void cleanupOldFiles(final String prefix, final String currentStateFile, Path[] locations) throws IOException { final DirectoryStream.Filter filter = new DirectoryStream.Filter() { @Override public boolean accept(Path entry) throws IOException { final String entryFileName = entry.getFileName().toString(); return Files.isRegularFile(entry) && entryFileName.startsWith(prefix) // only state files - && currentStateFile.equals(entryFileName) == false; // keep the current state file around + && entryFileName.equals(currentStateFile) == false; // keep the current state file around } }; // now clean up the old files diff --git a/src/test/java/org/elasticsearch/indices/recovery/MetaDataWriteDataNodesTests.java b/src/test/java/org/elasticsearch/indices/recovery/MetaDataWriteDataNodesTests.java new file mode 100644 index 0000000000000..aec1114ed67cf --- /dev/null +++ b/src/test/java/org/elasticsearch/indices/recovery/MetaDataWriteDataNodesTests.java @@ -0,0 +1,347 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices.recovery; + +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.gateway.GatewayMetaState; +import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.test.junit.annotations.TestLogging; +import org.junit.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.client.Requests.clusterHealthRequest; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; + +/** + * + */ +@ClusterScope(scope = Scope.TEST, numDataNodes = 0) +public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest { + + @Test + public void testMetaWrittenAlsoOnDataNode() throws Exception { + // this test checks that index state is written on data only nodes + startMasterNode("master_node", false); + startDataNode("red_node", "red", false); + assertAcked(prepareCreate("test").setSettings(ImmutableSettings.builder().put("index.number_of_replicas", 0))); + index("test", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); + waitForConcreteMappingsOnAll("test", "doc", "text"); + ensureGreen("test"); + assertIndexInMetaState("red_node", "test"); + assertIndexInMetaState("master_node", "test"); + //stop master node and start again with an empty data folder + ((InternalTestCluster) cluster()).stopCurrentMasterNode(); + startMasterNode("new_master_node", true); + client().admin().cluster().prepareHealth() + .setWaitForRelocatingShards(0) + .setWaitForEvents(Priority.LANGUID) + .setWaitForNodes("2") + .setWaitForGreenStatus().get(); + ensureGreen("test"); + // wait for mapping also on master becasue then we can be sure the state was written + waitForConcreteMappingsOnAll("test", "doc", "text"); + // check for meta data + assertIndexInMetaState("red_node", "test"); + assertIndexInMetaState("new_master_node", "test"); + // check if index and doc is still there + ensureGreen("test"); + assertTrue(client().prepareGet("test", "doc", "1").get().isExists()); + } + + @Test + public void testMetaWrittenOnlyForIndicesOnNodesThatHaveAShard() throws Exception { + // this test checks that the index state is only written to a data only node if they have a shard of that index allocated on the node + startMasterNode("master_node", false); + startDataNode("blue_node", "blue", false); + startDataNode("red_node", "red", false); + + assertAcked(prepareCreate("blue_index").setSettings(ImmutableSettings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "blue"))); + index("blue_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); + assertAcked(prepareCreate("red_index").setSettings(ImmutableSettings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red"))); + index("red_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); + ensureGreen(); + waitForConcreteMappingsOnAll("blue_index", "doc", "text"); + waitForConcreteMappingsOnAll("red_index", "doc", "text"); + assertIndexNotInMetaState("blue_node", "red_index"); + assertIndexNotInMetaState("red_node", "blue_index"); + assertIndexInMetaState("blue_node", "blue_index"); + assertIndexInMetaState("red_node", "red_index"); + assertIndexInMetaState("master_node", "red_index"); + assertIndexInMetaState("master_node", "blue_index"); + + // not the index state for blue_index should only be written on blue_node and the for red_index only on red_node + // we restart red node and master but with empty data folders + stopNode("red_node"); + ((InternalTestCluster) cluster()).stopCurrentMasterNode(); + startMasterNode("new_master_node", true); + startDataNode("new_red_node", "red", true); + + client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForRelocatingShards(0).setWaitForEvents(Priority.LANGUID).get(); + assertIndexNotInMetaState("blue_node", "red_index"); + assertIndexInMetaState("blue_node", "blue_index"); + assertIndexNotInMetaState("new_red_node", "red_index"); + assertIndexNotInMetaState("new_red_node", "blue_index"); + assertIndexNotInMetaState("new_master_node", "red_index"); + assertIndexInMetaState("new_master_node", "blue_index"); + // check that blue index is still there + assertFalse(client().admin().indices().prepareExists("red_index").get().isExists()); + assertTrue(client().prepareGet("blue_index", "doc", "1").get().isExists()); + // red index should be gone + // if the blue node had stored the index state then cluster health would be red and red_index would exist + assertFalse(client().admin().indices().prepareExists("red_index").get().isExists()); + ensureGreen(); + + } + + @Test + public void testMetaIsRemovedIfAllShardsFromIndexRemoved() throws Exception { + // this test checks that the index state is removed from a data only node once all shards have been allocated away from it + startMasterNode("master_node", false); + startDataNode("blue_node", "blue", false); + startDataNode("red_node", "red", false); + + // create blue_index on blue_node and same for red + client().admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForNodes("3")).get(); + assertAcked(prepareCreate("blue_index").setSettings(ImmutableSettings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "blue"))); + index("blue_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); + assertAcked(prepareCreate("red_index").setSettings(ImmutableSettings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red"))); + index("red_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); + + ensureGreen(); + assertIndexNotInMetaState("red_node", "blue_index"); + assertIndexNotInMetaState("blue_node", "red_index"); + assertIndexInMetaState("red_node", "red_index"); + assertIndexInMetaState("blue_node", "blue_index"); + assertIndexInMetaState("master_node", "red_index"); + assertIndexInMetaState("master_node", "blue_index"); + + // now relocate blue_index to red_node and red_index to blue_node + logger.debug("relocating indices..."); + client().admin().indices().prepareUpdateSettings("blue_index").setSettings(ImmutableSettings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red")).get(); + client().admin().indices().prepareUpdateSettings("red_index").setSettings(ImmutableSettings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "blue")).get(); + client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).get(); + ensureGreen(); + assertIndexNotInMetaState("red_node", "red_index"); + assertIndexNotInMetaState("blue_node", "blue_index"); + assertIndexInMetaState("red_node", "blue_index"); + assertIndexInMetaState("blue_node", "red_index"); + assertIndexInMetaState("master_node", "red_index"); + assertIndexInMetaState("master_node", "blue_index"); + waitForConcreteMappingsOnAll("blue_index", "doc", "text"); + waitForConcreteMappingsOnAll("red_index", "doc", "text"); + + //at this point the blue_index is on red node and the red_index on blue node + // now, when we start red and master node again but without data folder, the red index should be gone but the blue index should initialize fine + stopNode("red_node"); + ((InternalTestCluster) cluster()).stopCurrentMasterNode(); + startMasterNode("new_master_node", true); + startDataNode("new_red_node", "red", true); + client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForRelocatingShards(0).setWaitForEvents(Priority.LANGUID).get(); + assertIndexNotInMetaState("new_red_node", "blue_index"); + assertIndexNotInMetaState("blue_node", "blue_index"); + assertIndexNotInMetaState("new_red_node", "red_index"); + assertIndexInMetaState("blue_node", "red_index"); + assertIndexInMetaState("new_master_node", "red_index"); + assertIndexNotInMetaState("new_master_node", "blue_index"); + assertTrue(client().prepareGet("red_index", "doc", "1").get().isExists()); + // if the red_node had stored the index state then cluster health would be red and blue_index would exist + assertFalse(client().admin().indices().prepareExists("blue_index").get().isExists()); + } + + @Test + public void randomStartStopReloacteWithAndWithoutData() throws Exception { + // relocate shards to and fro and start master nodes randomly with or without data folder and see what happens + // not sure if this test is any good, it did not uncover anything + + Map runningMasterNodes = new HashMap<>(); + String[] masterNodeNames = {"master_node_1", "master_node_2"}; + for (String name : masterNodeNames) { + runningMasterNodes.put(name, true); + startMasterNode(name, false); + } + logger.debug("start data node"); + startDataNode("blue_node", "blue", false); + startDataNode("red_node", "red", false); + + client().admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForNodes("3")).get(); + assertAcked(prepareCreate("blue_index").setSettings(ImmutableSettings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "blue"))); + index("blue_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); + assertAcked(prepareCreate("red_index").setSettings(ImmutableSettings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red"))); + index("red_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); + + // green index can be anywhere + assertAcked(prepareCreate("green_index").setSettings(ImmutableSettings.builder().put("index.number_of_replicas", 0))); + index("green_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); + waitForConcreteMappingsOnAll("blue_index", "doc", "text"); + waitForConcreteMappingsOnAll("red_index", "doc", "text"); + waitForConcreteMappingsOnAll("green_index", "doc", "text"); + + String[] indexNames = {"blue_index", "red_index"}; + String[] colors = {"blue", "red"}; + ensureGreen(); + int numMasterNodesUp = masterNodeNames.length; + for (int i = 0; i < 10; i++) { + // 1. start stop node or relocate? + if (randomBoolean()) { + // we start or stop a master node + if (numMasterNodesUp > 0 && numMasterNodesUp < masterNodeNames.length) { + //we have a working cluster, we can either stop or start a master node + if (randomBoolean()) { + numMasterNodesUp = stopMasterNode(runningMasterNodes, numMasterNodesUp); + } else { + numMasterNodesUp = startMasterNode(runningMasterNodes, numMasterNodesUp); + } + } else if (numMasterNodesUp == masterNodeNames.length) { + // all masters are running, can only stop one + numMasterNodesUp = stopMasterNode(runningMasterNodes, numMasterNodesUp); + } else if (numMasterNodesUp == 0) { + // all masters are stopped, must start one + numMasterNodesUp = startMasterNode(runningMasterNodes, numMasterNodesUp); + } + + } else { + if (numMasterNodesUp > 0) { + // relocate a random index to red or blue + String index = indexNames[randomInt(1)]; + String color = colors[randomInt(1)]; + logger.info("move index {} to node with color {}", index, color); + ensureGreen(index); + client().admin().indices().prepareUpdateSettings(index).setSettings(ImmutableSettings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", color)).get(); + client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).get(); + ensureGreen(); + } + } + } + if (numMasterNodesUp == 0) { + numMasterNodesUp = startMasterNode(runningMasterNodes, numMasterNodesUp); + } + client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForRelocatingShards(0).setWaitForEvents(Priority.LANGUID).get(); + ensureGreen("blue_index"); + assertTrue(client().admin().indices().prepareExists("blue_index").get().isExists()); + ensureGreen("red_index"); + assertTrue(client().admin().indices().prepareExists("red_index").get().isExists()); + ensureGreen("green_index"); + assertTrue(client().admin().indices().prepareExists("green_index").get().isExists()); + assertTrue(client().prepareGet("red_index", "doc", "1").get().isExists()); + assertTrue(client().prepareGet("blue_index", "doc", "1").get().isExists()); + assertTrue(client().prepareGet("green_index", "doc", "1").get().isExists()); + } + + protected int stopMasterNode(Map runningMasterNodes, int numMasterNodesUp) throws IOException { + String runningMasterNode = getRunningMasterNode(runningMasterNodes); + logger.info("Stopping master eligible node {}", runningMasterNode); + stopNode(runningMasterNode); + runningMasterNodes.put(runningMasterNode, false); + numMasterNodesUp--; + return numMasterNodesUp; + } + + protected int startMasterNode(Map runningMasterNodes, int numMasterNodesUp) { + String downMasterNode = getDownMasterNode(runningMasterNodes); + boolean withNewDataPath = randomBoolean(); + logger.info("Starting master eligible node {} with {} data path", downMasterNode, (withNewDataPath ? "random empty " : "default")); + startMasterNode(downMasterNode, withNewDataPath); + runningMasterNodes.put(downMasterNode, true); + numMasterNodesUp++; + return numMasterNodesUp; + } + + private String getRunningMasterNode(Map masterNodes) { + for (Map.Entry masterNode : masterNodes.entrySet()) { + if (masterNode.getValue() == true) { + return masterNode.getKey(); + } + } + return null; + } + + private String getDownMasterNode(Map masterNodes) { + for (Map.Entry masterNode : masterNodes.entrySet()) { + if (masterNode.getValue() == false) { + return masterNode.getKey(); + } + } + return null; + } + + private void startDataNode(String name, String color, boolean newDataPath) { + ImmutableSettings.Builder settingsBuilder = ImmutableSettings.builder() + .put("node.data", true) + .put("node.master", false) + .put("node.color", color) + .put("node.name", name); + if (newDataPath) { + settingsBuilder.put("path.data", newTempDirPath().toString()); + } + internalCluster().startNode(settingsBuilder.build()); + } + + private void startMasterNode(String name, boolean newDataPath) { + ImmutableSettings.Builder settingsBuilder = ImmutableSettings.builder() + .put("node.data", false) + .put("node.master", true) + .put("node.name", name); + if (newDataPath) { + settingsBuilder.put("path.data", newTempDirPath().toString()); + } + internalCluster().startNode(settingsBuilder.build()); + } + + private void stopNode(String name) throws IOException { + ((InternalTestCluster) cluster()).stopNode(name); + } + + protected void assertIndexNotInMetaState(String nodeName, String indexName) throws Exception { + assertMetaState(nodeName, indexName, false); + } + + protected void assertIndexInMetaState(String nodeName, String indexName) throws Exception { + assertMetaState(nodeName, indexName, true); + } + + private void assertMetaState(String nodeName, String indexName, boolean shouldBe) throws Exception { + GatewayMetaState redNodeMetaState = ((InternalTestCluster) cluster()).getInstance(GatewayMetaState.class, nodeName); + MetaData redNodeMetaData = redNodeMetaState.loadMetaState(); + ImmutableOpenMap indices = redNodeMetaData.getIndices(); + boolean inMetaSate = false; + for (ObjectObjectCursor index : indices) { + inMetaSate = inMetaSate || index.key.equals(indexName); + } + if (shouldBe) { + assertTrue("expected " + indexName + " in meta state of node " + nodeName, inMetaSate); + } else { + assertFalse("expected " + indexName + " to not be in meta state of node " + nodeName, inMetaSate); + } + } +} diff --git a/src/test/java/org/elasticsearch/test/InternalTestCluster.java b/src/test/java/org/elasticsearch/test/InternalTestCluster.java index b83623118d0a4..b099174668505 100644 --- a/src/test/java/org/elasticsearch/test/InternalTestCluster.java +++ b/src/test/java/org/elasticsearch/test/InternalTestCluster.java @@ -569,7 +569,12 @@ private NodeAndClient buildNode(int nodeId, long seed, Settings settings, Versio assert Thread.holdsLock(this); ensureOpen(); settings = getSettings(nodeId, seed, settings); - String name = buildNodeName(nodeId); + String name; + if (settings.get("node.name") != null) { + name = settings.get("node.name"); + } else { + name = buildNodeName(nodeId); + } assert !nodes.containsKey(name); Settings finalSettings = settingsBuilder() .put(settings) @@ -1110,8 +1115,12 @@ public InetSocketAddress[] httpAddresses() { public synchronized boolean stopRandomDataNode() throws IOException { ensureOpen(); NodeAndClient nodeAndClient = getRandomNodeAndClient(new DataNodePredicate()); + return stopNode(nodeAndClient, true); + } + + private boolean stopNode(NodeAndClient nodeAndClient, boolean isRandomNode) throws IOException { if (nodeAndClient != null) { - logger.info("Closing random node [{}] ", nodeAndClient.name); + logger.info("Closing {}node [{}] ", (isRandomNode? "random ": ""), nodeAndClient.name); removeDisruptionSchemeFromNode(nodeAndClient); nodes.remove(nodeAndClient.name); nodeAndClient.close(); @@ -1120,6 +1129,15 @@ public synchronized boolean stopRandomDataNode() throws IOException { return false; } + /** + * Stops a node. Returns true if a node was found to stop, false otherwise. + */ + public synchronized boolean stopNode(String name) throws IOException { + ensureOpen(); + NodeAndClient nodeAndClient = nodes.get(name); + return stopNode(nodeAndClient, false); + } + /** * Stops a random node in the cluster that applies to the given filter or non if the non of the nodes applies to the * filter.