From 0715e5022e9ef4904e90afa2148d5135bcfe9fba Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 17 Apr 2019 16:36:04 +0100 Subject: [PATCH] Assert the stability of custom search preferences (#41150) Today the `?preference=custom_string_value` search preference will only change its choice of a shard copy if something changes the `IndexShardRoutingTable` for that specific shard. Users can use this behaviour to route searches to a consistent set of shard copies, which means they can reliably hit copies with hot caches, and use the other copies only for redundancy in case of failure. However we do not assert this property anywhere, so we might break it in future. This commit adds a test that shows that searches are routed consistently even if other indices are created/rebalanced/deleted. Relates https://discuss.elastic.co/t/176598, #41115, #26791 --- .../search/preference/SearchPreferenceIT.java | 71 +++++++++++++++++-- 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java b/server/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java index 366975071ce43..23c29ce9f4674 100644 --- a/server/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java +++ b/server/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java @@ -25,10 +25,13 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.OperationRouting; +import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.node.Node; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; @@ -42,10 +45,10 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasToString; -import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.not; @ESIntegTestCase.ClusterScope(minNumDataNodes = 2) public class SearchPreferenceIT extends ESIntegTestCase { @@ -57,7 +60,7 @@ public Settings nodeSettings(int nodeOrdinal) { } // see #2896 - public void testStopOneNodePreferenceWithRedState() throws InterruptedException, IOException { + public void testStopOneNodePreferenceWithRedState() throws IOException { assertAcked(prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", cluster().numDataNodes()+2) .put("index.number_of_replicas", 0))); ensureGreen(); @@ -87,7 +90,7 @@ public void testStopOneNodePreferenceWithRedState() throws InterruptedException, assertThat("_only_local", searchResponse.getFailedShards(), greaterThanOrEqualTo(0)); } - public void testNoPreferenceRandom() throws Exception { + public void testNoPreferenceRandom() { assertAcked(prepareCreate("test").setSettings( //this test needs at least a replica to make sure two consecutive searches go to two different copies of the same data Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(1, maximumNumberOfReplicas())) @@ -106,7 +109,7 @@ public void testNoPreferenceRandom() throws Exception { assertThat(firstNodeId, not(equalTo(secondNodeId))); } - public void testSimplePreference() throws Exception { + public void testSimplePreference() { client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 1}", XContentType.JSON).get(); ensureGreen(); @@ -123,7 +126,7 @@ public void testSimplePreference() throws Exception { assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L)); } - public void testThatSpecifyingNonExistingNodesReturnsUsefulError() throws Exception { + public void testThatSpecifyingNonExistingNodesReturnsUsefulError() { createIndex("test"); ensureGreen(); @@ -135,7 +138,7 @@ public void testThatSpecifyingNonExistingNodesReturnsUsefulError() throws Except } } - public void testNodesOnlyRandom() throws Exception { + public void testNodesOnlyRandom() { assertAcked(prepareCreate("test").setSettings( //this test needs at least a replica to make sure two consecutive searches go to two different copies of the same data Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(1, maximumNumberOfReplicas())))); @@ -193,4 +196,58 @@ private void assertSearchOnRandomNodes(SearchRequestBuilder request) { } assertThat(hitNodes.size(), greaterThan(1)); } + + public void testCustomPreferenceUnaffectedByOtherShardMovements() { + + /* + * Custom preferences can be used to encourage searches to go to a consistent set of shard copies, meaning that other copies' data + * is rarely touched and can be dropped from the filesystem cache. This works best if the set of shards searched doesn't change + * unnecessarily, so this test verifies a consistent routing even as other shards are created/relocated/removed. + */ + + assertAcked(prepareCreate("test").setSettings(Settings.builder().put(indexSettings()) + .put(SETTING_NUMBER_OF_REPLICAS, between(1, maximumNumberOfReplicas())) + .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE))); + ensureGreen(); + client().prepareIndex("test", "_doc").setSource("field1", "value1").get(); + refresh(); + + final String customPreference = randomAlphaOfLength(10); + + final String nodeId = client().prepareSearch("test").setQuery(matchAllQuery()).setPreference(customPreference) + .get().getHits().getAt(0).getShard().getNodeId(); + + assertSearchesSpecificNode("test", customPreference, nodeId); + + final int replicasInNewIndex = between(1, maximumNumberOfReplicas()); + assertAcked(prepareCreate("test2").setSettings( + Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, replicasInNewIndex))); + ensureGreen(); + + assertSearchesSpecificNode("test", customPreference, nodeId); + + assertAcked(client().admin().indices().prepareUpdateSettings("test2").setSettings(Settings.builder() + .put(SETTING_NUMBER_OF_REPLICAS, replicasInNewIndex - 1))); + + assertSearchesSpecificNode("test", customPreference, nodeId); + + assertAcked(client().admin().indices().prepareUpdateSettings("test2").setSettings(Settings.builder() + .put(SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._name", + internalCluster().getDataNodeInstance(Node.class).settings().get(Node.NODE_NAME_SETTING.getKey())))); + + ensureGreen(); + + assertSearchesSpecificNode("test", customPreference, nodeId); + + assertAcked(client().admin().indices().prepareDelete("test2")); + + assertSearchesSpecificNode("test", customPreference, nodeId); + } + + private static void assertSearchesSpecificNode(String index, String customPreference, String nodeId) { + final SearchResponse searchResponse = client().prepareSearch(index).setQuery(matchAllQuery()).setPreference(customPreference).get(); + assertThat(searchResponse.getHits().getHits().length, equalTo(1)); + assertThat(searchResponse.getHits().getAt(0).getShard().getNodeId(), equalTo(nodeId)); + } }