Assert the stability of custom search preferences (elastic#41150)

Today the `?preference=custom_string_value` search preference will only change
its choice of a shard copy if something changes the `IndexShardRoutingTable`
for that specific shard. Users can use this behaviour to route searches to a
consistent set of shard copies, which means they can reliably hit copies with
hot caches, and use the other copies only for redundancy in case of failure.
However we do not assert this property anywhere, so we might break it in
future.

This commit adds a test that shows that searches are routed consistently even
if other indices are created/rebalanced/deleted.

Relates https://discuss.elastic.co/t/176598, elastic#41115, elastic#26791

server/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java

@@ -25,10 +25,13 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.OperationRouting;
+import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -42,10 +45,10 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasToString;
-import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasToString;
+import static org.hamcrest.Matchers.not;
 
 @ESIntegTestCase.ClusterScope(minNumDataNodes = 2)
 public class SearchPreferenceIT extends ESIntegTestCase {
@@ -57,7 +60,7 @@ public Settings nodeSettings(int nodeOrdinal) {
     }
 
     // see #2896
-    public void testStopOneNodePreferenceWithRedState() throws InterruptedException, IOException {
+    public void testStopOneNodePreferenceWithRedState() throws IOException {
         assertAcked(prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", cluster().numDataNodes()+2)
                 .put("index.number_of_replicas", 0)));
         ensureGreen();
@@ -87,7 +90,7 @@ public void testStopOneNodePreferenceWithRedState() throws InterruptedException,
         assertThat("_only_local", searchResponse.getFailedShards(), greaterThanOrEqualTo(0));
     }
 
-    public void testNoPreferenceRandom() throws Exception {
+    public void testNoPreferenceRandom() {
         assertAcked(prepareCreate("test").setSettings(
                 //this test needs at least a replica to make sure two consecutive searches go to two different copies of the same data
                 Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(1, maximumNumberOfReplicas()))
@@ -106,7 +109,7 @@ public void testNoPreferenceRandom() throws Exception {
         assertThat(firstNodeId, not(equalTo(secondNodeId)));
     }
 
-    public void testSimplePreference() throws Exception {
+    public void testSimplePreference() {
         client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 1}", XContentType.JSON).get();
         ensureGreen();
 
@@ -123,7 +126,7 @@ public void testSimplePreference() throws Exception {
         assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
     }
 
-    public void testThatSpecifyingNonExistingNodesReturnsUsefulError() throws Exception {
+    public void testThatSpecifyingNonExistingNodesReturnsUsefulError() {
         createIndex("test");
         ensureGreen();
 
@@ -135,7 +138,7 @@ public void testThatSpecifyingNonExistingNodesReturnsUsefulError() throws Except
         }
     }
 
-    public void testNodesOnlyRandom() throws Exception {
+    public void testNodesOnlyRandom() {
         assertAcked(prepareCreate("test").setSettings(
             //this test needs at least a replica to make sure two consecutive searches go to two different copies of the same data
             Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(1, maximumNumberOfReplicas()))));
@@ -193,4 +196,58 @@ private void assertSearchOnRandomNodes(SearchRequestBuilder request) {
         }
         assertThat(hitNodes.size(), greaterThan(1));
     }
+
+    public void testCustomPreferenceUnaffectedByOtherShardMovements() {
+
+        /*
+         * Custom preferences can be used to encourage searches to go to a consistent set of shard copies, meaning that other copies' data
+         * is rarely touched and can be dropped from the filesystem cache. This works best if the set of shards searched doesn't change
+         * unnecessarily, so this test verifies a consistent routing even as other shards are created/relocated/removed.
+         */
+
+        assertAcked(prepareCreate("test").setSettings(Settings.builder().put(indexSettings())
+            .put(SETTING_NUMBER_OF_REPLICAS, between(1, maximumNumberOfReplicas()))
+            .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)));
+        ensureGreen();
+        client().prepareIndex("test", "_doc").setSource("field1", "value1").get();
+        refresh();
+
+        final String customPreference = randomAlphaOfLength(10);
+
+        final String nodeId = client().prepareSearch("test").setQuery(matchAllQuery()).setPreference(customPreference)
+            .get().getHits().getAt(0).getShard().getNodeId();
+
+        assertSearchesSpecificNode("test", customPreference, nodeId);
+
+        final int replicasInNewIndex = between(1, maximumNumberOfReplicas());
+        assertAcked(prepareCreate("test2").setSettings(
+            Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, replicasInNewIndex)));
+        ensureGreen();
+
+        assertSearchesSpecificNode("test", customPreference, nodeId);
+
+        assertAcked(client().admin().indices().prepareUpdateSettings("test2").setSettings(Settings.builder()
+            .put(SETTING_NUMBER_OF_REPLICAS, replicasInNewIndex - 1)));
+
+        assertSearchesSpecificNode("test", customPreference, nodeId);
+
+        assertAcked(client().admin().indices().prepareUpdateSettings("test2").setSettings(Settings.builder()
+            .put(SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._name",
+                internalCluster().getDataNodeInstance(Node.class).settings().get(Node.NODE_NAME_SETTING.getKey()))));
+
+        ensureGreen();
+
+        assertSearchesSpecificNode("test", customPreference, nodeId);
+
+        assertAcked(client().admin().indices().prepareDelete("test2"));
+
+        assertSearchesSpecificNode("test", customPreference, nodeId);
+    }
+
+    private static void assertSearchesSpecificNode(String index, String customPreference, String nodeId) {
+        final SearchResponse searchResponse = client().prepareSearch(index).setQuery(matchAllQuery()).setPreference(customPreference).get();
+        assertThat(searchResponse.getHits().getHits().length, equalTo(1));
+        assertThat(searchResponse.getHits().getAt(0).getShard().getNodeId(), equalTo(nodeId));
+    }
 }