From 35a25ab8645f9d42db32bb9013630312e6892894 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 12 Apr 2019 15:14:01 +0100 Subject: [PATCH] Assert the stability of custom search preferences Today the `?preference=custom_string_value` search preference will only change its choice of a shard copy if something changes the `IndexShardRoutingTable` for that specific shard. Users can use this behaviour to route searches to a consistent set of shard copies, which means they can reliably hit copies with hot caches, and use the other copies only for redundancy in case of failure. However we do not assert this property anywhere, so we might break it in future. This commit adds a test that shows that searches are routed consistently even if other indices are created/rebalanced/deleted. Relates https://discuss.elastic.co/t/176598, #41115, #26791 --- .../search/preference/SearchPreferenceIT.java | 71 +++++++++++++++++-- 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java b/server/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java index 366975071ce43..f46c018e4e74f 100644 --- a/server/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java +++ b/server/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java @@ -25,7 +25,9 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.OperationRouting; +import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; @@ -42,10 +44,10 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasToString; -import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.not; @ESIntegTestCase.ClusterScope(minNumDataNodes = 2) public class SearchPreferenceIT extends ESIntegTestCase { @@ -57,7 +59,7 @@ public Settings nodeSettings(int nodeOrdinal) { } // see #2896 - public void testStopOneNodePreferenceWithRedState() throws InterruptedException, IOException { + public void testStopOneNodePreferenceWithRedState() throws IOException { assertAcked(prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", cluster().numDataNodes()+2) .put("index.number_of_replicas", 0))); ensureGreen(); @@ -87,7 +89,7 @@ public void testStopOneNodePreferenceWithRedState() throws InterruptedException, assertThat("_only_local", searchResponse.getFailedShards(), greaterThanOrEqualTo(0)); } - public void testNoPreferenceRandom() throws Exception { + public void testNoPreferenceRandom() { assertAcked(prepareCreate("test").setSettings( //this test needs at least a replica to make sure two consecutive searches go to two different copies of the same data Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(1, maximumNumberOfReplicas())) @@ -106,7 +108,7 @@ public void testNoPreferenceRandom() throws Exception { assertThat(firstNodeId, not(equalTo(secondNodeId))); } - public void testSimplePreference() throws Exception { + public void testSimplePreference() { client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 1}", XContentType.JSON).get(); ensureGreen(); @@ -123,7 +125,7 @@ public void testSimplePreference() throws Exception { assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L)); } - public void testThatSpecifyingNonExistingNodesReturnsUsefulError() throws Exception { + public void testThatSpecifyingNonExistingNodesReturnsUsefulError() { createIndex("test"); ensureGreen(); @@ -135,7 +137,7 @@ public void testThatSpecifyingNonExistingNodesReturnsUsefulError() throws Except } } - public void testNodesOnlyRandom() throws Exception { + public void testNodesOnlyRandom() { assertAcked(prepareCreate("test").setSettings( //this test needs at least a replica to make sure two consecutive searches go to two different copies of the same data Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(1, maximumNumberOfReplicas())))); @@ -193,4 +195,59 @@ private void assertSearchOnRandomNodes(SearchRequestBuilder request) { } assertThat(hitNodes.size(), greaterThan(1)); } + + public void testCustomPreferenceUnaffectedByIndexCreation() { + + /* + * Custom preferences can be used to encourage searches to go to a consistent set of shard copies, meaning that other copies' data + * is rarely touched and can be dropped from the filesystem cache. This works best if the set of shards searched doesn't change + * unnecessarily, so this test verifies a consistent routing even as other shards are created/relocated/removed. + */ + + final String testIndex = "test"; + + assertAcked(prepareCreate(testIndex).setSettings(Settings.builder().put(indexSettings()) + .put(SETTING_NUMBER_OF_REPLICAS, between(1, maximumNumberOfReplicas())) + .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE))); + ensureGreen(); + client().prepareIndex(testIndex, "_doc").setSource("field1", "value1").get(); + refresh(); + + final String customPreference = randomAlphaOfLength(10); + + final String nodeId = client().prepareSearch(testIndex).setQuery(matchAllQuery()).setPreference(customPreference) + .get().getHits().getAt(0).getShard().getNodeId(); + + assertSearchesSpecificNode(testIndex, customPreference, nodeId); + + final int replicasInNewIndex = between(1, maximumNumberOfReplicas()); + assertAcked(prepareCreate("test2").setSettings( + Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, replicasInNewIndex))); + ensureGreen(); + + assertSearchesSpecificNode(testIndex, customPreference, nodeId); + + assertAcked(client().admin().indices().prepareUpdateSettings("test2").setSettings(Settings.builder() + .put(SETTING_NUMBER_OF_REPLICAS, replicasInNewIndex - 1))); + + assertSearchesSpecificNode(testIndex, customPreference, nodeId); + + assertAcked(client().admin().indices().prepareUpdateSettings("test2").setSettings(Settings.builder() + .put(SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._name", internalCluster().getNodeNames()[0]))); + + ensureGreen(); + + assertSearchesSpecificNode(testIndex, customPreference, nodeId); + + assertAcked(client().admin().indices().prepareDelete("test2")); + + assertSearchesSpecificNode(testIndex, customPreference, nodeId); + } + + private static void assertSearchesSpecificNode(String index, String customPreference, String nodeId) { + final SearchResponse search2 = client().prepareSearch(index).setQuery(matchAllQuery()).setPreference(customPreference).get(); + assertThat(search2.getHits().getHits().length, equalTo(1)); + assertThat(search2.getHits().getAt(0).getShard().getNodeId(), equalTo(nodeId)); + } }