From 3edc889d482f5a3e5c33a16d1e897a6a950fe0da Mon Sep 17 00:00:00 2001 From: Nhat Date: Mon, 25 Sep 2017 14:30:46 -0400 Subject: [PATCH 1/7] remove _primary and _replica shard preferences The shard preference `_primary`, `_replica` and its variants were useful for the asynchronous replication. However, with the current impl, they are no longer useful and should be removed. Closes #26335 --- .../search/NoopSearchRequestBuilder.java | 4 +- .../shards/ClusterSearchShardsRequest.java | 4 +- .../ClusterSearchShardsRequestBuilder.java | 4 +- .../elasticsearch/action/get/GetRequest.java | 4 +- .../action/get/GetRequestBuilder.java | 4 +- .../action/get/MultiGetRequest.java | 4 +- .../action/get/MultiGetRequestBuilder.java | 4 +- .../action/get/MultiGetShardRequest.java | 4 +- .../action/search/SearchRequest.java | 4 +- .../action/search/SearchRequestBuilder.java | 4 +- .../MultiTermVectorsShardRequest.java | 4 +- .../termvectors/TermVectorsRequest.java | 3 +- .../TermVectorsRequestBuilder.java | 4 +- .../routing/IndexShardRoutingTable.java | 68 ------------------- .../cluster/routing/OperationRouting.java | 8 --- .../cluster/routing/Preference.java | 30 -------- .../structure/RoutingIteratorTests.java | 61 +++-------------- .../index/translog/TruncateTranslogIT.java | 8 ++- .../indices/state/RareClusterStateIT.java | 3 +- .../basic/SearchWhileCreatingIndexIT.java | 26 +++++-- .../fetch/subphase/MatchedQueriesIT.java | 1 - .../functionscore/RandomScoreFunctionIT.java | 2 +- .../search/preference/SearchPreferenceIT.java | 47 +------------ .../search/profile/query/QueryProfilerIT.java | 2 - .../search/simple/SimpleSearchIT.java | 2 +- docs/reference/docs/get.asciidoc | 4 -- docs/reference/docs/index_.asciidoc | 5 +- .../search/request/preference.asciidoc | 17 +---- .../upgrades/FullClusterRestartIT.java | 35 +++++++--- .../elasticsearch/backwards/IndexingIT.java | 10 ++- .../test/bulk/20_list_of_strings.yml | 2 - .../test/client/RandomizingClient.java | 3 +- 32 files changed, 99 insertions(+), 286 deletions(-) diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/NoopSearchRequestBuilder.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/NoopSearchRequestBuilder.java index 5143bdd870594..529182aa98f7d 100644 --- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/NoopSearchRequestBuilder.java +++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/NoopSearchRequestBuilder.java @@ -142,8 +142,8 @@ public NoopSearchRequestBuilder setRouting(String... routing) { /** * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, or - * a custom value, which guarantees that the same order will be used across different requests. + * _local to prefer local shards or a custom value, which guarantees that the same order + * will be used across different requests. */ public NoopSearchRequestBuilder setPreference(String preference) { request.preference(preference); diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java index d8dfd71530922..d127829fa3584 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java @@ -146,8 +146,8 @@ public ClusterSearchShardsRequest routing(String... routings) { /** * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, or - * a custom value, which guarantees that the same order will be used across different requests. + * _local to prefer local shards or a custom value, which guarantees that the same order + * will be used across different requests. */ public ClusterSearchShardsRequest preference(String preference) { this.preference = preference; diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java index 7cb7ac1254c60..da31a79fc9bf0 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java @@ -55,8 +55,8 @@ public ClusterSearchShardsRequestBuilder setRouting(String... routing) { /** * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, or - * a custom value, which guarantees that the same order will be used across different requests. + * _local to prefer local shards or a custom value, which guarantees that the same order + * will be used across different requests. */ public ClusterSearchShardsRequestBuilder setPreference(String preference) { request.preference(preference); diff --git a/core/src/main/java/org/elasticsearch/action/get/GetRequest.java b/core/src/main/java/org/elasticsearch/action/get/GetRequest.java index 93045182f4c20..ea5dda45279e6 100644 --- a/core/src/main/java/org/elasticsearch/action/get/GetRequest.java +++ b/core/src/main/java/org/elasticsearch/action/get/GetRequest.java @@ -152,8 +152,8 @@ public GetRequest routing(String routing) { /** * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, or - * a custom value, which guarantees that the same order will be used across different requests. + * _local to prefer local shards or a custom value, which guarantees that the same order + * will be used across different requests. */ public GetRequest preference(String preference) { this.preference = preference; diff --git a/core/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java index 973b130bedbd2..1ca8dbde65200 100644 --- a/core/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java @@ -76,8 +76,8 @@ public GetRequestBuilder setRouting(String routing) { /** * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, or - * a custom value, which guarantees that the same order will be used across different requests. + * _local to prefer local shards or a custom value, which guarantees that the same order + * will be used across different requests. */ public GetRequestBuilder setPreference(String preference) { request.preference(preference); diff --git a/core/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java b/core/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java index 20a619cec2c70..420e0b448b052 100644 --- a/core/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java +++ b/core/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java @@ -284,8 +284,8 @@ public ActionRequestValidationException validate() { /** * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, or - * a custom value, which guarantees that the same order will be used across different requests. + * _local to prefer local shards or a custom value, which guarantees that the same order + * will be used across different requests. */ public MultiGetRequest preference(String preference) { this.preference = preference; diff --git a/core/src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java index a2cb204d5eabf..fd7a6ac88253e 100644 --- a/core/src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java @@ -58,8 +58,8 @@ public MultiGetRequestBuilder add(MultiGetRequest.Item item) { /** * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, or - * a custom value, which guarantees that the same order will be used across different requests. + * _local to prefer local shards or a custom value, which guarantees that the same order + * will be used across different requests. */ public MultiGetRequestBuilder setPreference(String preference) { request.preference(preference); diff --git a/core/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java b/core/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java index 25a624b2eb558..fea3cd1043c62 100644 --- a/core/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java +++ b/core/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java @@ -64,8 +64,8 @@ public int shardId() { /** * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, or - * a custom value, which guarantees that the same order will be used across different requests. + * _local to prefer local shards or a custom value, which guarantees that the same order + * will be used across different requests. */ public MultiGetShardRequest preference(String preference) { this.preference = preference; diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 030d19d8b6879..7bfa317c72c70 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -241,8 +241,8 @@ public SearchRequest routing(String... routings) { /** * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, or - * a custom value, which guarantees that the same order will be used across different requests. + * _local to prefer local shards or a custom value, which guarantees that the same order + * will be used across different requests. */ public SearchRequest preference(String preference) { this.preference = preference; diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java index 41e5babb64635..922e9be5fd75d 100644 --- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java @@ -144,8 +144,8 @@ public SearchRequestBuilder setRouting(String... routing) { /** * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, or - * a custom value, which guarantees that the same order will be used across different requests. + * _local to prefer local shards or a custom value, which guarantees that the same order + * will be used across different requests. */ public SearchRequestBuilder setPreference(String preference) { request.preference(preference); diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardRequest.java b/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardRequest.java index 6356c554991e6..8fdb6398ddccf 100644 --- a/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardRequest.java +++ b/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardRequest.java @@ -59,8 +59,8 @@ public int shardId() { /** * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, or - * a custom value, which guarantees that the same order will be used across different requests. + * _local to prefer local shards or a custom value, which guarantees that the same order + * will be used across different requests. */ public MultiTermVectorsShardRequest preference(String preference) { this.preference = preference; diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java index 0fe83e214463a..1886a8c2661ed 100644 --- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java +++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java @@ -294,8 +294,7 @@ public String preference() { /** * Sets the preference to execute the search. Defaults to randomize across - * shards. Can be set to _local to prefer local shards, - * _primary to execute only on primary shards, or a custom value, + * shards. Can be set to _local to prefer local shards or a custom value, * which guarantees that the same order will be used across different * requests. */ diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequestBuilder.java index 9aa3ebca759c3..47bd09b100857 100644 --- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequestBuilder.java @@ -99,8 +99,8 @@ public TermVectorsRequestBuilder setParent(String parent) { /** * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to - * _local to prefer local shards, _primary to execute only on primary shards, or - * a custom value, which guarantees that the same order will be used across different requests. + * _local to prefer local shards or a custom value, which guarantees that the same order + * will be used across different requests. */ public TermVectorsRequestBuilder setPreference(String preference) { request.preference(preference); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java index f8d42b3d8f5a0..a2d015a0dd13f 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java @@ -441,74 +441,6 @@ public ShardIterator primaryShardIt() { return new PlainShardIterator(shardId, primaryAsList); } - public ShardIterator primaryActiveInitializingShardIt() { - if (noPrimariesActive()) { - return new PlainShardIterator(shardId, NO_SHARDS); - } - return primaryShardIt(); - } - - public ShardIterator primaryFirstActiveInitializingShardsIt() { - ArrayList ordered = new ArrayList<>(activeShards.size() + allInitializingShards.size()); - // fill it in a randomized fashion - for (ShardRouting shardRouting : shuffler.shuffle(activeShards)) { - ordered.add(shardRouting); - if (shardRouting.primary()) { - // switch, its the matching node id - ordered.set(ordered.size() - 1, ordered.get(0)); - ordered.set(0, shardRouting); - } - } - // no need to worry about primary first here..., its temporal - if (!allInitializingShards.isEmpty()) { - ordered.addAll(allInitializingShards); - } - return new PlainShardIterator(shardId, ordered); - } - - public ShardIterator replicaActiveInitializingShardIt() { - // If the primaries are unassigned, return an empty list (there aren't - // any replicas to query anyway) - if (noPrimariesActive()) { - return new PlainShardIterator(shardId, NO_SHARDS); - } - - LinkedList ordered = new LinkedList<>(); - for (ShardRouting replica : shuffler.shuffle(replicas)) { - if (replica.active()) { - ordered.addFirst(replica); - } else if (replica.initializing()) { - ordered.addLast(replica); - } - } - return new PlainShardIterator(shardId, ordered); - } - - public ShardIterator replicaFirstActiveInitializingShardsIt() { - // If the primaries are unassigned, return an empty list (there aren't - // any replicas to query anyway) - if (noPrimariesActive()) { - return new PlainShardIterator(shardId, NO_SHARDS); - } - - ArrayList ordered = new ArrayList<>(activeShards.size() + allInitializingShards.size()); - // fill it in a randomized fashion with the active replicas - for (ShardRouting replica : shuffler.shuffle(replicas)) { - if (replica.active()) { - ordered.add(replica); - } - } - - // Add the primary shard - ordered.add(primary); - - // Add initializing shards last - if (!allInitializingShards.isEmpty()) { - ordered.addAll(allInitializingShards); - } - return new PlainShardIterator(shardId, ordered); - } - public ShardIterator onlyNodeActiveInitializingShardsIt(String nodeId) { ArrayList ordered = new ArrayList<>(activeShards.size() + allInitializingShards.size()); int seed = shuffler.nextSeed(); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java index 296eca476a6c5..87adb55704a25 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java @@ -198,14 +198,6 @@ private ShardIterator preferenceActiveShardIterator(IndexShardRoutingTable index return indexShard.preferNodeActiveInitializingShardsIt(nodesIds); case LOCAL: return indexShard.preferNodeActiveInitializingShardsIt(Collections.singleton(localNodeId)); - case PRIMARY: - return indexShard.primaryActiveInitializingShardIt(); - case REPLICA: - return indexShard.replicaActiveInitializingShardIt(); - case PRIMARY_FIRST: - return indexShard.primaryFirstActiveInitializingShardsIt(); - case REPLICA_FIRST: - return indexShard.replicaFirstActiveInitializingShardsIt(); case ONLY_LOCAL: return indexShard.onlyNodeActiveInitializingShardsIt(localNodeId); case ONLY_NODES: diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java b/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java index d4685d7aeadc1..9a55a99a51ca8 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java @@ -39,26 +39,6 @@ public enum Preference { */ LOCAL("_local"), - /** - * Route to primary shards - */ - PRIMARY("_primary"), - - /** - * Route to replica shards - */ - REPLICA("_replica"), - - /** - * Route to primary shards first - */ - PRIMARY_FIRST("_primary_first"), - - /** - * Route to replica shards first - */ - REPLICA_FIRST("_replica_first"), - /** * Route to the local shard only */ @@ -97,16 +77,6 @@ public static Preference parse(String preference) { return PREFER_NODES; case "_local": return LOCAL; - case "_primary": - return PRIMARY; - case "_replica": - return REPLICA; - case "_primary_first": - case "_primaryFirst": - return PRIMARY_FIRST; - case "_replica_first": - case "_replicaFirst": - return REPLICA_FIRST; case "_only_local": case "_onlyLocal": return ONLY_LOCAL; diff --git a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java index 172bcd6bd558b..747a0c57d724f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java @@ -50,6 +50,7 @@ import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -442,57 +443,15 @@ public void testReplicaShardPreferenceIters() throws Exception { clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)); - // When replicas haven't initialized, it comes back with the primary first, then initializing replicas - GroupShardsIterator shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_replica_first"); - assertThat(shardIterators.size(), equalTo(2)); // two potential shards - ShardIterator iter = shardIterators.iterator().next(); - assertThat(iter.size(), equalTo(3)); // three potential candidates for the shard - ShardRouting routing = iter.nextOrNull(); - assertNotNull(routing); - assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1))); - assertTrue(routing.primary()); // replicas haven't initialized yet, so primary is first - assertTrue(routing.started()); - routing = iter.nextOrNull(); - assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1))); - assertFalse(routing.primary()); - assertTrue(routing.initializing()); - routing = iter.nextOrNull(); - assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1))); - assertFalse(routing.primary()); - assertTrue(routing.initializing()); - - clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)); - - clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)); - - - shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_replica"); - assertThat(shardIterators.size(), equalTo(2)); // two potential shards - iter = shardIterators.iterator().next(); - assertThat(iter.size(), equalTo(2)); // two potential replicas for the shard - routing = iter.nextOrNull(); - assertNotNull(routing); - assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1))); - assertFalse(routing.primary()); - routing = iter.nextOrNull(); - assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1))); - assertFalse(routing.primary()); - - shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_replica_first"); - assertThat(shardIterators.size(), equalTo(2)); // two potential shards - iter = shardIterators.iterator().next(); - assertThat(iter.size(), equalTo(3)); // three potential candidates for the shard - routing = iter.nextOrNull(); - assertNotNull(routing); - assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1))); - assertFalse(routing.primary()); - routing = iter.nextOrNull(); - assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1))); - assertFalse(routing.primary()); - // finally the primary - routing = iter.nextOrNull(); - assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1))); - assertTrue(routing.primary()); + String[] removedPreferences = {"_primary", "_primary_first", "_replica", "_replica_first"}; + for (String pref : removedPreferences) { + try{ + operationRouting.searchShards(clusterState, new String[]{"test"}, null, pref); + fail("Should have failed because the shard preference [" + pref + "] is removed."); + }catch (IllegalArgumentException ex){ + assertThat(ex.getMessage(), containsString(pref)); + } + } } } diff --git a/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java b/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java index c2b394b219a20..df46a56882333 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java +++ b/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java @@ -210,7 +210,9 @@ public void testCorruptTranslogTruncation() throws Exception { logger.info("--> starting the replica node to test recovery"); internalCluster().startNode(); ensureGreen("test"); - assertHitCount(client().prepareSearch("test").setPreference("_replica").setQuery(matchAllQuery()).get(), numDocsToKeep); + for(String node : internalCluster().nodesInclude("test")){ + assertHitCount(client().prepareSearch("test").setPreference("_only_nodes:"+ node).setQuery(matchAllQuery()).get(), numDocsToKeep); + } final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").setActiveOnly(false).get(); final RecoveryState replicaRecoveryState = recoveryResponse.shardRecoveryStates().get("test").stream() .filter(recoveryState -> recoveryState.getPrimary() == false).findFirst().get(); @@ -308,7 +310,9 @@ public void testCorruptTranslogTruncationOfReplica() throws Exception { logger.info("--> starting the replica node to test recovery"); internalCluster().startNode(); ensureGreen("test"); - assertHitCount(client().prepareSearch("test").setPreference("_replica").setQuery(matchAllQuery()).get(), totalDocs); + for(String node : internalCluster().nodesInclude("test")){ + assertHitCount(client().prepareSearch("test").setPreference("_only_nodes:"+node).setQuery(matchAllQuery()).get(), totalDocs); + } final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").setActiveOnly(false).get(); final RecoveryState replicaRecoveryState = recoveryResponse.shardRecoveryStates().get("test").stream() diff --git a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java index 185f27d39c8fe..bf213b51475fb 100644 --- a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java @@ -406,8 +406,7 @@ public void onFailure(Exception e) { } }); - // Wait for document to be indexed on primary - assertBusy(() -> assertTrue(client().prepareGet("index", "type", "1").setPreference("_primary").get().isExists())); + assertBusy(() -> assertTrue(client().prepareGet("index", "type", "1").get().isExists())); // The mappings have not been propagated to the replica yet as a consequence the document count not be indexed // We wait on purpose to make sure that the document is not indexed because the shard operation is stalled diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java index 1ed396672b5d0..f853aebb746cb 100644 --- a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java +++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; @@ -73,13 +74,21 @@ private void searchWhileCreatingIndex(boolean createIndex, int numberOfReplicas) logger.info("using preference {}", preference); // we want to make sure that while recovery happens, and a replica gets recovered, its properly refreshed - ClusterHealthStatus status = client().admin().cluster().prepareHealth("test").get().getStatus();; + ClusterHealthStatus status = client().admin().cluster().prepareHealth("test").get().getStatus(); + while (status != ClusterHealthStatus.GREEN) { // first, verify that search on the primary search works - SearchResponse searchResponse = client().prepareSearch("test").setPreference("_primary").setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet(); - assertHitCount(searchResponse, 1); + for (IndexShardRoutingTable shardRoutingTable : clusterService().state().routingTable().index("test")) { + String primaryNode = shardRoutingTable.primaryShard().currentNodeId(); + SearchResponse searchResponse = client().prepareSearch("test") + .setPreference("_only_nodes:" + primaryNode) + .setQuery(QueryBuilders.termQuery("field", "test")) + .execute().actionGet(); + assertHitCount(searchResponse, 1); + break; + } Client client = client(); - searchResponse = client.prepareSearch("test").setPreference(preference + Integer.toString(counter++)).setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet(); + SearchResponse searchResponse = client.prepareSearch("test").setPreference(preference + Integer.toString(counter++)).setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet(); if (searchResponse.getHits().getTotalHits() != 1) { refresh(); SearchResponse searchResponseAfterRefresh = client.prepareSearch("test").setPreference(preference).setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet(); @@ -93,8 +102,13 @@ private void searchWhileCreatingIndex(boolean createIndex, int numberOfReplicas) status = client().admin().cluster().prepareHealth("test").get().getStatus(); internalCluster().ensureAtLeastNumDataNodes(numberOfReplicas + 1); } - SearchResponse searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet(); - assertHitCount(searchResponse, 1); + + for(String node : internalCluster().nodesInclude("test")){ + SearchResponse searchResponse = client().prepareSearch("test") + .setPreference("_only_nodes:" + node) + .setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet(); + assertHitCount(searchResponse, 1); + } cluster().wipeIndices("test"); } } diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java index 763518804e277..761f9798d7286 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java @@ -301,7 +301,6 @@ public void testMatchedWithShould() throws Exception { .should(queryStringQuery("dolor").queryName("dolor")) .should(queryStringQuery("elit").queryName("elit")) ) - .setPreference("_primary") .get(); assertHitCount(searchResponse, 2L); diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java index 31366c2534cb2..257089c90545f 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/RandomScoreFunctionIT.java @@ -107,7 +107,7 @@ public void testConsistentHitsWithSameSeed() throws Exception { for (int o = 0; o < outerIters; o++) { final int seed = randomInt(); String preference = randomRealisticUnicodeOfLengthBetween(1, 10); // at least one char!! - // randomPreference should not start with '_' (reserved for known preference types (e.g. _shards, _primary) + // randomPreference should not start with '_' (reserved for known preference types (e.g. _shards) while (preference.startsWith("_")) { preference = randomRealisticUnicodeOfLengthBetween(1, 10); } diff --git a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java index 6478446a1a254..9b0ed13ad325b 100644 --- a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java +++ b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java @@ -44,7 +44,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -67,7 +66,7 @@ public void testStopOneNodePreferenceWithRedState() throws InterruptedException, refresh(); internalCluster().stopRandomDataNode(); client().admin().cluster().prepareHealth().setWaitForStatus(ClusterHealthStatus.RED).execute().actionGet(); - String[] preferences = new String[] {"_primary", "_local", "_primary_first", "_prefer_nodes:somenode", "_prefer_nodes:server2", "_prefer_nodes:somenode,server2"}; + String[] preferences = new String[] {"_local", "_prefer_nodes:somenode", "_prefer_nodes:server2", "_prefer_nodes:somenode,server2"}; for (String pref : preferences) { logger.info("--> Testing out preference={}", pref); SearchResponse searchResponse = client().prepareSearch().setSize(0).setPreference(pref).execute().actionGet(); @@ -113,54 +112,14 @@ public void testSimplePreference() throws Exception { client().prepareIndex("test", "type1").setSource("field1", "value1").execute().actionGet(); refresh(); - SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_local").execute().actionGet(); - assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_local").execute().actionGet(); - assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); - - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_primary").execute().actionGet(); - assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_primary").execute().actionGet(); - assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); - - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_replica").execute().actionGet(); - assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_replica").execute().actionGet(); + SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_replica_first").execute().actionGet(); - assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_replica_first").execute().actionGet(); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_local").execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setPreference("1234").execute().actionGet(); assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setPreference("1234").execute().actionGet(); - assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L)); - } - - public void testReplicaPreference() throws Exception { - client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 0}", XContentType.JSON).get(); - ensureGreen(); - - client().prepareIndex("test", "type1").setSource("field1", "value1").execute().actionGet(); - refresh(); - - try { - client().prepareSearch().setQuery(matchAllQuery()).setPreference("_replica").execute().actionGet(); - fail("should have failed because there are no replicas"); - } catch (Exception e) { - // pass - } - - SearchResponse resp = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_replica_first").execute().actionGet(); - assertThat(resp.getHits().getTotalHits(), equalTo(1L)); - - client().admin().indices().prepareUpdateSettings("test").setSettings("{\"number_of_replicas\": 1}", XContentType.JSON).get(); - ensureGreen("test"); - - resp = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_replica").execute().actionGet(); - assertThat(resp.getHits().getTotalHits(), equalTo(1L)); } public void testThatSpecifyingNonExistingNodesReturnsUsefulError() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java index d5198485351b1..14378fdb1c8a9 100644 --- a/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java +++ b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java @@ -134,14 +134,12 @@ public void testProfileMatchesRegular() throws Exception { .setQuery(q) .setProfile(false) .addSort("_id", SortOrder.ASC) - .setPreference("_primary") .setSearchType(SearchType.QUERY_THEN_FETCH); SearchRequestBuilder profile = client().prepareSearch("test") .setQuery(q) .setProfile(true) .addSort("_id", SortOrder.ASC) - .setPreference("_primary") .setSearchType(SearchType.QUERY_THEN_FETCH); MultiSearchResponse.Item[] responses = client().prepareMultiSearch() diff --git a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java index 88c403a1d7fb5..9eacb0e81bd29 100644 --- a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -79,7 +79,7 @@ public void testSearchRandomPreference() throws InterruptedException, ExecutionE int iters = scaledRandomIntBetween(10, 20); for (int i = 0; i < iters; i++) { String randomPreference = randomUnicodeOfLengthBetween(0, 4); - // randomPreference should not start with '_' (reserved for known preference types (e.g. _shards, _primary) + // randomPreference should not start with '_' (reserved for known preference types (e.g. _shards) while (randomPreference.startsWith("_")) { randomPreference = randomUnicodeOfLengthBetween(0, 4); } diff --git a/docs/reference/docs/get.asciidoc b/docs/reference/docs/get.asciidoc index 2a252595dd59a..11b2347e7f31b 100644 --- a/docs/reference/docs/get.asciidoc +++ b/docs/reference/docs/get.asciidoc @@ -275,10 +275,6 @@ replicas. The `preference` can be set to: -`_primary`:: - The operation will go and be executed only on the primary - shards. - `_local`:: The operation will prefer to be executed on a local allocated shard if possible. diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index 8e18f3034e82b..7875f011abee1 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -91,8 +91,7 @@ will control the version of the document the operation is intended to be executed against. A good example of a use case for versioning is performing a transactional read-then-update. Specifying a `version` from the document initially read ensures no changes have happened in the -meantime (when reading in order to update, it is recommended to set -`preference` to `_primary`). For example: +meantime. For example: [source,js] -------------------------------------------------- @@ -242,7 +241,7 @@ The result of the above index operation is: [[index-routing]] === Routing -By default, shard placement — or `routing` — is controlled by using a +By default, shard placement ? or `routing` ? is controlled by using a hash of the document's id value. For more explicit control, the value fed into the hash function used by the router can be directly specified on a per-operation basis using the `routing` parameter. For example: diff --git a/docs/reference/search/request/preference.asciidoc b/docs/reference/search/request/preference.asciidoc index d0f60d700a82c..dbd9055ff8c86 100644 --- a/docs/reference/search/request/preference.asciidoc +++ b/docs/reference/search/request/preference.asciidoc @@ -7,21 +7,6 @@ search. By default, the operation is randomized among the available shard copies The `preference` is a query string parameter which can be set to: [horizontal] -`_primary`:: - The operation will go and be executed only on the primary - shards. - -`_primary_first`:: - The operation will go and be executed on the primary - shard, and if not available (failover), will execute on other shards. - -`_replica`:: - The operation will go and be executed only on a replica shard. - -`_replica_first`:: - The operation will go and be executed only on a replica shard, and if - not available (failover), will execute on other shards. - `_local`:: The operation will prefer to be executed on a local allocated shard if possible. @@ -33,7 +18,7 @@ The `preference` is a query string parameter which can be set to: `_shards:2,3`:: Restricts the operation to the specified shards. (`2` and `3` in this case). This preference can be combined with other - preferences but it has to appear first: `_shards:2,3|_primary` + preferences but it has to appear first: `_shards:2,3|_local` `_only_nodes`:: Restricts the operation to nodes specified in <> diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index c7e708418c92c..afc27b3e2706c 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -25,6 +25,7 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.Version; import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -37,12 +38,15 @@ import org.junit.Before; import java.io.IOException; +import java.util.ArrayList; import java.util.Base64; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -227,17 +231,15 @@ public void testNewReplicasWork() throws Exception { Map recoverRsp = toMap(client().performRequest("GET", "/" + index + "/_recovery")); logger.debug("--> recovery status:\n{}", recoverRsp); - Map responseBody = toMap(client().performRequest("GET", "/" + index + "/_search", - Collections.singletonMap("preference", "_primary"))); - assertNoFailures(responseBody); - int foundHits1 = (int) XContentMapValues.extractValue("hits.total", responseBody); - - responseBody = toMap(client().performRequest("GET", "/" + index + "/_search", - Collections.singletonMap("preference", "_replica"))); - assertNoFailures(responseBody); - int foundHits2 = (int) XContentMapValues.extractValue("hits.total", responseBody); - assertEquals(foundHits1, foundHits2); - // TODO: do something more with the replicas! index? + Set counts = new HashSet<>(); + for(String node : dataNodes(index, client())){ + Map responseBody = toMap(client().performRequest("GET", "/" + index + "/_search", + Collections.singletonMap("preference", "_only_nodes:"+node))); + assertNoFailures(responseBody); + int hits = (int) XContentMapValues.extractValue("hits.total", responseBody); + counts.add(hits); + } + assertEquals("All nodes should have a consistent number of documents", 1, counts.size()); } } @@ -940,4 +942,15 @@ private void refresh() throws IOException { logger.debug("Refreshing [{}]", index); client().performRequest("POST", "/" + index + "/_refresh"); } + + private List dataNodes(String index, RestClient client) throws IOException { + Response response = client.performRequest("GET", index + "/_stats", singletonMap("level", "shards")); + List nodes = new ArrayList<>(); + List shardStats = ObjectPath.createFromResponse(response).evaluate("indices." + index + ".shards.0"); + for (Object shard : shardStats) { + final String nodeId = ObjectPath.evaluate(shard, "routing.node"); + nodes.add(nodeId); + } + return nodes; + } } diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java index b5076868f04f0..c17002576fa1f 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java @@ -188,9 +188,6 @@ public void testIndexVersionPropagation() throws Exception { assertVersion(index, 5, "_only_nodes:" + shard.getNode().getNodeName(), finalVersionForDoc5); assertCount(index, "_only_nodes:" + shard.getNode().getNodeName(), 5); } - // the number of documents on the primary and on the recovered replica should match the number of indexed documents - assertCount(index, "_primary", 5); - assertCount(index, "_replica", 5); } } @@ -249,9 +246,10 @@ public void testSeqNoCheckpoints() throws Exception { updateIndexSetting(index, Settings.builder().put("index.number_of_replicas", 1)); ensureGreen(); assertOK(client().performRequest("POST", index + "/_refresh")); - // the number of documents on the primary and on the recovered replica should match the number of indexed documents - assertCount(index, "_primary", numDocs); - assertCount(index, "_replica", numDocs); + + for(Shard shard : buildShards(index, nodes, newNodeClient)){ + assertCount(index, "_only_nodes:" + shard.node.nodeName, numDocs); + } assertSeqNoOnShards(index, nodes, numDocs, newNodeClient); } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/20_list_of_strings.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/20_list_of_strings.yml index e25626cf3ae28..def91f4280722 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/20_list_of_strings.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/20_list_of_strings.yml @@ -11,8 +11,6 @@ - do: count: - # we count through the primary in case there is a replica that has not yet fully recovered - preference: _primary index: test_index - match: {count: 2} diff --git a/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java b/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java index b144898d643d0..e1a6ba030fde8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.unit.TimeValue; import java.util.Arrays; -import java.util.EnumSet; import java.util.Random; import java.util.concurrent.TimeUnit; @@ -52,7 +51,7 @@ public RandomizingClient(Client client, Random random) { SearchType.DFS_QUERY_THEN_FETCH, SearchType.QUERY_THEN_FETCH)); if (random.nextInt(10) == 0) { - defaultPreference = RandomPicks.randomFrom(random, EnumSet.of(Preference.PRIMARY_FIRST, Preference.LOCAL)).type(); + defaultPreference = Preference.LOCAL.type(); } else if (random.nextInt(10) == 0) { String s = TestUtil.randomRealisticUnicodeString(random, 1, 10); defaultPreference = s.startsWith("_") ? null : s; // '_' is a reserved character From adc7aa00c7ec2dd19d3cd62eb52c0d02e94934c8 Mon Sep 17 00:00:00 2001 From: Nhat Date: Tue, 26 Sep 2017 10:58:38 -0400 Subject: [PATCH 2/7] truncate a long line --- .../org/elasticsearch/index/translog/TruncateTranslogIT.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java b/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java index df46a56882333..0032d007000ae 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java +++ b/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java @@ -33,6 +33,7 @@ import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.GroupShardsIterator; @@ -211,7 +212,8 @@ public void testCorruptTranslogTruncation() throws Exception { internalCluster().startNode(); ensureGreen("test"); for(String node : internalCluster().nodesInclude("test")){ - assertHitCount(client().prepareSearch("test").setPreference("_only_nodes:"+ node).setQuery(matchAllQuery()).get(), numDocsToKeep); + SearchRequestBuilder q = client().prepareSearch("test").setPreference("_only_nodes:" + node).setQuery(matchAllQuery()); + assertHitCount(q.get(), numDocsToKeep); } final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").setActiveOnly(false).get(); final RecoveryState replicaRecoveryState = recoveryResponse.shardRecoveryStates().get("test").stream() From c442730ca6b9efa7f3fbc14dfa308ce2c5cfb344 Mon Sep 17 00:00:00 2001 From: Nhat Date: Wed, 4 Oct 2017 09:46:25 -0400 Subject: [PATCH 3/7] use expectThrows --- .../structure/RoutingIteratorTests.java | 35 +++++++------------ 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java index 747a0c57d724f..e6c890b72b91d 100644 --- a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java @@ -416,10 +416,6 @@ public void testShardsAndPreferNodeRouting() { } public void testReplicaShardPreferenceIters() throws Exception { - AllocationService strategy = createAllocationService(Settings.builder() - .put("cluster.routing.allocation.node_concurrent_recoveries", 10) - .build()); - OperationRouting operationRouting = new OperationRouting(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); @@ -431,26 +427,21 @@ public void testReplicaShardPreferenceIters() throws Exception { .addAsNew(metaData.index("test")) .build(); - ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build(); - - clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder() - .add(newNode("node1")) - .add(newNode("node2")) - .add(newNode("node3")) - .localNodeId("node1") - ).build(); - clusterState = strategy.reroute(clusterState, "reroute"); - - clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)); + final ClusterState clusterState = ClusterState + .builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metaData(metaData) + .routingTable(routingTable) + .nodes(DiscoveryNodes.builder() + .add(newNode("node1")) + .add(newNode("node2")) + .add(newNode("node3")) + .localNodeId("node1")) + .build(); - String[] removedPreferences = {"_primary", "_primary_first", "_replica", "_replica_first"}; + String[] removedPreferences = { "_primary", "_primary_first", "_replica", "_replica_first" }; for (String pref : removedPreferences) { - try{ - operationRouting.searchShards(clusterState, new String[]{"test"}, null, pref); - fail("Should have failed because the shard preference [" + pref + "] is removed."); - }catch (IllegalArgumentException ex){ - assertThat(ex.getMessage(), containsString(pref)); - } + expectThrows(IllegalArgumentException.class, + () -> operationRouting.searchShards(clusterState, new String[]{"test"}, null, pref)); } } From e5b2b246bacc4f90c123ea6857f48b3ed479813e Mon Sep 17 00:00:00 2001 From: Nhat Date: Wed, 4 Oct 2017 09:49:21 -0400 Subject: [PATCH 4/7] adjust code format --- .../cluster/structure/RoutingIteratorTests.java | 2 +- .../elasticsearch/index/translog/TruncateTranslogIT.java | 6 +++--- .../search/basic/SearchWhileCreatingIndexIT.java | 2 +- .../org/elasticsearch/upgrades/FullClusterRestartIT.java | 4 ++-- .../test/java/org/elasticsearch/backwards/IndexingIT.java | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java index e6c890b72b91d..6fd11aa91dce6 100644 --- a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java @@ -438,7 +438,7 @@ public void testReplicaShardPreferenceIters() throws Exception { .localNodeId("node1")) .build(); - String[] removedPreferences = { "_primary", "_primary_first", "_replica", "_replica_first" }; + String[] removedPreferences = {"_primary", "_primary_first", "_replica", "_replica_first"}; for (String pref : removedPreferences) { expectThrows(IllegalArgumentException.class, () -> operationRouting.searchShards(clusterState, new String[]{"test"}, null, pref)); diff --git a/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java b/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java index 0032d007000ae..b0d4c238679e8 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java +++ b/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java @@ -211,7 +211,7 @@ public void testCorruptTranslogTruncation() throws Exception { logger.info("--> starting the replica node to test recovery"); internalCluster().startNode(); ensureGreen("test"); - for(String node : internalCluster().nodesInclude("test")){ + for (String node : internalCluster().nodesInclude("test")) { SearchRequestBuilder q = client().prepareSearch("test").setPreference("_only_nodes:" + node).setQuery(matchAllQuery()); assertHitCount(q.get(), numDocsToKeep); } @@ -312,8 +312,8 @@ public void testCorruptTranslogTruncationOfReplica() throws Exception { logger.info("--> starting the replica node to test recovery"); internalCluster().startNode(); ensureGreen("test"); - for(String node : internalCluster().nodesInclude("test")){ - assertHitCount(client().prepareSearch("test").setPreference("_only_nodes:"+node).setQuery(matchAllQuery()).get(), totalDocs); + for (String node : internalCluster().nodesInclude("test")) { + assertHitCount(client().prepareSearch("test").setPreference("_only_nodes:" + node).setQuery(matchAllQuery()).get(), totalDocs); } final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").setActiveOnly(false).get(); diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java index f853aebb746cb..709ab82d749f2 100644 --- a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java +++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java @@ -103,7 +103,7 @@ private void searchWhileCreatingIndex(boolean createIndex, int numberOfReplicas) internalCluster().ensureAtLeastNumDataNodes(numberOfReplicas + 1); } - for(String node : internalCluster().nodesInclude("test")){ + for (String node : internalCluster().nodesInclude("test")) { SearchResponse searchResponse = client().prepareSearch("test") .setPreference("_only_nodes:" + node) .setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet(); diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index afc27b3e2706c..22859859f2521 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -232,9 +232,9 @@ public void testNewReplicasWork() throws Exception { logger.debug("--> recovery status:\n{}", recoverRsp); Set counts = new HashSet<>(); - for(String node : dataNodes(index, client())){ + for (String node : dataNodes(index, client())) { Map responseBody = toMap(client().performRequest("GET", "/" + index + "/_search", - Collections.singletonMap("preference", "_only_nodes:"+node))); + Collections.singletonMap("preference", "_only_nodes:" + node))); assertNoFailures(responseBody); int hits = (int) XContentMapValues.extractValue("hits.total", responseBody); counts.add(hits); diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java index c17002576fa1f..f779ba653554d 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java @@ -247,7 +247,7 @@ public void testSeqNoCheckpoints() throws Exception { ensureGreen(); assertOK(client().performRequest("POST", index + "/_refresh")); - for(Shard shard : buildShards(index, nodes, newNodeClient)){ + for (Shard shard : buildShards(index, nodes, newNodeClient)) { assertCount(index, "_only_nodes:" + shard.node.nodeName, numDocs); } assertSeqNoOnShards(index, nodes, numDocs, newNodeClient); From 88669338308c784d3ecd21ca3170a3f37cbad233 Mon Sep 17 00:00:00 2001 From: Nhat Date: Wed, 4 Oct 2017 10:09:26 -0400 Subject: [PATCH 5/7] add migration note --- docs/reference/migration/migrate_7_0/cluster.asciidoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/reference/migration/migrate_7_0/cluster.asciidoc b/docs/reference/migration/migrate_7_0/cluster.asciidoc index 12e6916e00145..d59aad0c3a692 100644 --- a/docs/reference/migration/migrate_7_0/cluster.asciidoc +++ b/docs/reference/migration/migrate_7_0/cluster.asciidoc @@ -11,3 +11,6 @@ cluster names may no longer contain `:`. The default value for the `wait_for_active_shards` parameter of the open index API is changed from 0 to 1, which means that the command will now by default wait for all primary shards of the opened index to be allocated. + +==== shard preferences `_primary`, `_primary_first`, `_replica`, and `_replica_first` are removed +These shard preferences are removed in favour of the `_prefer_nodes` and `_only_nodes` preferences. From 0460a0fcb290ee13a5797c747d4c7f51808a1e59 Mon Sep 17 00:00:00 2001 From: Nhat Date: Wed, 4 Oct 2017 12:47:14 -0400 Subject: [PATCH 6/7] loosen the preferences with _prefer_nodes --- .../elasticsearch/search/basic/SearchWhileCreatingIndexIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java index 709ab82d749f2..1ff2d6922f98b 100644 --- a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java +++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java @@ -105,7 +105,7 @@ private void searchWhileCreatingIndex(boolean createIndex, int numberOfReplicas) for (String node : internalCluster().nodesInclude("test")) { SearchResponse searchResponse = client().prepareSearch("test") - .setPreference("_only_nodes:" + node) + .setPreference("_prefer_nodes:" + node) .setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet(); assertHitCount(searchResponse, 1); } From 9c022a62493b9eae56341d56c60bdf2f536aebbf Mon Sep 17 00:00:00 2001 From: Nhat Date: Wed, 4 Oct 2017 18:35:42 -0400 Subject: [PATCH 7/7] address comments --- .../elasticsearch/search/preference/SearchPreferenceIT.java | 2 +- docs/reference/migration/migrate_7_0/cluster.asciidoc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java index 9b0ed13ad325b..8cbb626b6770e 100644 --- a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java +++ b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java @@ -66,7 +66,7 @@ public void testStopOneNodePreferenceWithRedState() throws InterruptedException, refresh(); internalCluster().stopRandomDataNode(); client().admin().cluster().prepareHealth().setWaitForStatus(ClusterHealthStatus.RED).execute().actionGet(); - String[] preferences = new String[] {"_local", "_prefer_nodes:somenode", "_prefer_nodes:server2", "_prefer_nodes:somenode,server2"}; + String[] preferences = new String[]{"_local", "_prefer_nodes:somenode", "_prefer_nodes:server2", "_prefer_nodes:somenode,server2"}; for (String pref : preferences) { logger.info("--> Testing out preference={}", pref); SearchResponse searchResponse = client().prepareSearch().setSize(0).setPreference(pref).execute().actionGet(); diff --git a/docs/reference/migration/migrate_7_0/cluster.asciidoc b/docs/reference/migration/migrate_7_0/cluster.asciidoc index d59aad0c3a692..e9584074d73d2 100644 --- a/docs/reference/migration/migrate_7_0/cluster.asciidoc +++ b/docs/reference/migration/migrate_7_0/cluster.asciidoc @@ -6,11 +6,11 @@ Due to cross-cluster search using `:` to separate a cluster and index name, cluster names may no longer contain `:`. -==== new default for `wait_for_active_shards` parameter of the open index command +==== New default for `wait_for_active_shards` parameter of the open index command The default value for the `wait_for_active_shards` parameter of the open index API is changed from 0 to 1, which means that the command will now by default wait for all primary shards of the opened index to be allocated. -==== shard preferences `_primary`, `_primary_first`, `_replica`, and `_replica_first` are removed +==== Shard preferences `_primary`, `_primary_first`, `_replica`, and `_replica_first` are removed These shard preferences are removed in favour of the `_prefer_nodes` and `_only_nodes` preferences.