Merge branch 'master' into check-for-close

* master:
  update Lucene version for 6.0-RC2 version
  Calculate and cache result when advanceExact is called (elastic#26920)
  Test query builder bwc against previous supported versions instead of just the current version.
  Set minimum_master_nodes on rolling-upgrade test (elastic#26911)
  Return List instead of an array from settings (elastic#26903)
  remove _primary and _replica shard preferences (elastic#26791)
  fixing typo in datehistogram-aggregation.asciidoc (elastic#26924)
  [API] Added the `terminate_after` parameter to the REST spec for "Count" API
  Setup debug logging for qa.full-cluster-restart
  Enable BWC testing against other remotes

TESTING.asciidoc

@@ -472,28 +472,30 @@ is tested depends on the branch. On master, this will test against the current
 stable branch. On the stable branch, it will test against the latest release
 branch. Finally, on a release branch, it will test against the most recent release.
 
-=== BWC Testing against a specific branch
+=== BWC Testing against a specific remote/branch
 
 Sometimes a backward compatibility change spans two versions. A common case is a new functionality
 that needs a BWC bridge in and an unreleased versioned of a release branch (for example, 5.x).
-To test the changes, you can instruct gradle to build the BWC version from a local branch instead of
-pulling the release branch from GitHub. You do so using the `tests.bwc.refspec` system property:
+To test the changes, you can instruct gradle to build the BWC version from a another remote/branch combination instead of
+pulling the release branch from GitHub. You do so using the `tests.bwc.remote` and `tests.bwc.refspec` system properties:
 
 -------------------------------------------------
-gradle check -Dtests.bwc.refspec=origin/index_req_bwc_5.x
+gradle check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec=index_req_bwc_5.x
 -------------------------------------------------
 
-The branch needs to be available on the local clone that the BWC makes of the repository you run the
-tests from. Using the `origin` remote is a handy trick to make sure that a branch is available
-and is up to date in the case of multiple runs.
+The branch needs to be available on the remote that the BWC makes of the
+repository you run the tests from. Using the remote is a handy trick to make
+sure that a branch is available and is up to date in the case of multiple runs.
 
 Example:
 
-Say you need to make a change to `master` and have a BWC layer in `5.x`. You will need to:
-. Create a branch called `index_req_change` off `master`. This will contain your change.
+Say you need to make a change to `master` and have a BWC layer in `5.x`. You
+will need to:
+. Create a branch called `index_req_change` off your remote `${remote}`. This
+will contain your change.
 . Create a branch called `index_req_bwc_5.x` off `5.x`. This will contain your bwc layer.
-. If not running the tests locally, push both branches to your remote repository.
-. Run the tests with `gradle check -Dtests.bwc.refspec=origin/index_req_bwc_5.x`
+. Push both branches to your remote repository.
+. Run the tests with `gradle check -Dtests.bwc.remote=${remote} -Dtests.bwc.refspec=index_req_bwc_5.x`.
 
 == Coverage analysis
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -63,13 +63,11 @@ class ClusterConfiguration {
     boolean debug = false
 
     /**
-     * if <code>true</code> each node will be configured with <tt>discovery.zen.minimum_master_nodes</tt> set
-     * to the total number of nodes in the cluster. This will also cause that each node has `0s` state recovery
-     * timeout which can lead to issues if for instance an existing clusterstate is expected to be recovered
-     * before any tests start
+     * Configuration of the setting <tt>discovery.zen.minimum_master_nodes</tt> on the nodes.
+     * In case of more than one node, this defaults to (number of nodes / 2) + 1
      */
     @Input
-    boolean useMinimumMasterNodes = true
+    Closure<Integer> minimumMasterNodes = { getNumNodes() > 1 ? getNumNodes().intdiv(2) + 1 : -1 }
 
     @Input
     String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -311,13 +311,14 @@ class ClusterFormationTasks {
                 // Define a node attribute so we can test that it exists
                 'node.attr.testattr'           : 'test'
         ]
-        // we set min master nodes to the total number of nodes in the cluster and
-        // basically skip initial state recovery to allow the cluster to form using a realistic master election
-        // this means all nodes must be up, join the seed node and do a master election. This will also allow new and
-        // old nodes in the BWC case to become the master
-        if (node.config.useMinimumMasterNodes && node.config.numNodes > 1) {
-            esConfig['discovery.zen.minimum_master_nodes'] = node.config.numNodes
-            esConfig['discovery.initial_state_timeout'] = '0s' // don't wait for state.. just start up quickly
+        int minimumMasterNodes = node.config.minimumMasterNodes.call()
+        if (minimumMasterNodes > 0) {
+            esConfig['discovery.zen.minimum_master_nodes'] = minimumMasterNodes
+        }
+        if (node.config.numNodes > 1) {
+            // don't wait for state.. just start up quickly
+            // this will also allow new and old nodes in the BWC case to become the master
+            esConfig['discovery.initial_state_timeout'] = '0s'
         }
         esConfig['node.max_local_storage_nodes'] = node.config.numNodes
         esConfig['http.port'] = node.config.httpPort

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/NoopSearchRequestBuilder.java

@@ -142,8 +142,8 @@ public NoopSearchRequestBuilder setRouting(String... routing) {
 
     /**
      * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards, or
-     * a custom value, which guarantees that the same order will be used across different requests.
+     * <tt>_local</tt> to prefer local shards or a custom value, which guarantees that the same order
+     * will be used across different requests.
      */
     public NoopSearchRequestBuilder setPreference(String preference) {
         request.preference(preference);

core/src/main/java/org/elasticsearch/Version.java

@@ -115,7 +115,7 @@ public class Version implements Comparable<Version> {
         new Version(V_6_0_0_rc1_ID, org.apache.lucene.util.Version.LUCENE_7_0_0);
     public static final int V_6_0_0_rc2_ID = 6000052;
     public static final Version V_6_0_0_rc2 =
-        new Version(V_6_0_0_rc2_ID, org.apache.lucene.util.Version.LUCENE_7_0_0);
+        new Version(V_6_0_0_rc2_ID, org.apache.lucene.util.Version.LUCENE_7_0_1);
     public static final int V_6_1_0_ID = 6010099;
     public static final Version V_6_1_0 =
         new Version(V_6_1_0_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);

core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java

@@ -146,8 +146,8 @@ public ClusterSearchShardsRequest routing(String... routings) {
 
     /**
      * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards, or
-     * a custom value, which guarantees that the same order will be used across different requests.
+     * <tt>_local</tt> to prefer local shards or a custom value, which guarantees that the same order
+     * will be used across different requests.
      */
     public ClusterSearchShardsRequest preference(String preference) {
         this.preference = preference;

core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestBuilder.java

@@ -55,8 +55,8 @@ public ClusterSearchShardsRequestBuilder setRouting(String... routing) {
 
     /**
      * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards, or
-     * a custom value, which guarantees that the same order will be used across different requests.
+     * <tt>_local</tt> to prefer local shards or a custom value, which guarantees that the same order
+     * will be used across different requests.
      */
     public ClusterSearchShardsRequestBuilder setPreference(String preference) {
         request.preference(preference);

core/src/main/java/org/elasticsearch/action/get/GetRequest.java

@@ -152,8 +152,8 @@ public GetRequest routing(String routing) {
 
     /**
      * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards, or
-     * a custom value, which guarantees that the same order will be used across different requests.
+     * <tt>_local</tt> to prefer local shards or a custom value, which guarantees that the same order
+     * will be used across different requests.
      */
     public GetRequest preference(String preference) {
         this.preference = preference;

core/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java

@@ -76,8 +76,8 @@ public GetRequestBuilder setRouting(String routing) {
 
     /**
      * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards, or
-     * a custom value, which guarantees that the same order will be used across different requests.
+     * <tt>_local</tt> to prefer local shards or a custom value, which guarantees that the same order
+     * will be used across different requests.
      */
     public GetRequestBuilder setPreference(String preference) {
         request.preference(preference);

core/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java

@@ -284,8 +284,8 @@ public ActionRequestValidationException validate() {
 
     /**
      * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards, or
-     * a custom value, which guarantees that the same order will be used across different requests.
+     * <tt>_local</tt> to prefer local shards or a custom value, which guarantees that the same order
+     * will be used across different requests.
      */
     public MultiGetRequest preference(String preference) {
         this.preference = preference;

core/src/main/java/org/elasticsearch/action/get/MultiGetRequestBuilder.java

@@ -58,8 +58,8 @@ public MultiGetRequestBuilder add(MultiGetRequest.Item item) {
 
     /**
      * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards, or
-     * a custom value, which guarantees that the same order will be used across different requests.
+     * <tt>_local</tt> to prefer local shards or a custom value, which guarantees that the same order
+     * will be used across different requests.
      */
     public MultiGetRequestBuilder setPreference(String preference) {
         request.preference(preference);

core/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java

@@ -64,8 +64,8 @@ public int shardId() {
 
     /**
      * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards, or
-     * a custom value, which guarantees that the same order will be used across different requests.
+     * <tt>_local</tt> to prefer local shards or a custom value, which guarantees that the same order
+     * will be used across different requests.
      */
     public MultiGetShardRequest preference(String preference) {
         this.preference = preference;

core/src/main/java/org/elasticsearch/action/search/SearchRequest.java

@@ -241,8 +241,8 @@ public SearchRequest routing(String... routings) {
 
     /**
      * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards, or
-     * a custom value, which guarantees that the same order will be used across different requests.
+     * <tt>_local</tt> to prefer local shards or a custom value, which guarantees that the same order
+     * will be used across different requests.
      */
     public SearchRequest preference(String preference) {
         this.preference = preference;

core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java

@@ -144,8 +144,8 @@ public SearchRequestBuilder setRouting(String... routing) {
 
     /**
      * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards, or
-     * a custom value, which guarantees that the same order will be used across different requests.
+     * <tt>_local</tt> to prefer local shards or a custom value, which guarantees that the same order
+     * will be used across different requests.
      */
     public SearchRequestBuilder setPreference(String preference) {
         request.preference(preference);

core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsShardRequest.java

@@ -59,8 +59,8 @@ public int shardId() {
 
     /**
      * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards, or
-     * a custom value, which guarantees that the same order will be used across different requests.
+     * <tt>_local</tt> to prefer local shards or a custom value, which guarantees that the same order
+     * will be used across different requests.
      */
     public MultiTermVectorsShardRequest preference(String preference) {
         this.preference = preference;

core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java

@@ -294,8 +294,7 @@ public String preference() {
 
     /**
      * Sets the preference to execute the search. Defaults to randomize across
-     * shards. Can be set to <tt>_local</tt> to prefer local shards,
-     * <tt>_primary</tt> to execute only on primary shards, or a custom value,
+     * shards. Can be set to <tt>_local</tt> to prefer local shards or a custom value,
      * which guarantees that the same order will be used across different
      * requests.
      */

core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequestBuilder.java

@@ -99,8 +99,8 @@ public TermVectorsRequestBuilder setParent(String parent) {
 
     /**
      * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards, or
-     * a custom value, which guarantees that the same order will be used across different requests.
+     * <tt>_local</tt> to prefer local shards or a custom value, which guarantees that the same order
+     * will be used across different requests.
      */
     public TermVectorsRequestBuilder setPreference(String preference) {
         request.preference(preference);

core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java

@@ -441,74 +441,6 @@ public ShardIterator primaryShardIt() {
         return new PlainShardIterator(shardId, primaryAsList);
     }
 
-    public ShardIterator primaryActiveInitializingShardIt() {
-        if (noPrimariesActive()) {
-            return new PlainShardIterator(shardId, NO_SHARDS);
-        }
-        return primaryShardIt();
-    }
-
-    public ShardIterator primaryFirstActiveInitializingShardsIt() {
-        ArrayList<ShardRouting> ordered = new ArrayList<>(activeShards.size() + allInitializingShards.size());
-        // fill it in a randomized fashion
-        for (ShardRouting shardRouting : shuffler.shuffle(activeShards)) {
-            ordered.add(shardRouting);
-            if (shardRouting.primary()) {
-                // switch, its the matching node id
-                ordered.set(ordered.size() - 1, ordered.get(0));
-                ordered.set(0, shardRouting);
-            }
-        }
-        // no need to worry about primary first here..., its temporal
-        if (!allInitializingShards.isEmpty()) {
-            ordered.addAll(allInitializingShards);
-        }
-        return new PlainShardIterator(shardId, ordered);
-    }
-
-    public ShardIterator replicaActiveInitializingShardIt() {
-        // If the primaries are unassigned, return an empty list (there aren't
-        // any replicas to query anyway)
-        if (noPrimariesActive()) {
-            return new PlainShardIterator(shardId, NO_SHARDS);
-        }
-
-        LinkedList<ShardRouting> ordered = new LinkedList<>();
-        for (ShardRouting replica : shuffler.shuffle(replicas)) {
-            if (replica.active()) {
-                ordered.addFirst(replica);
-            } else if (replica.initializing()) {
-                ordered.addLast(replica);
-            }
-        }
-        return new PlainShardIterator(shardId, ordered);
-    }
-
-    public ShardIterator replicaFirstActiveInitializingShardsIt() {
-        // If the primaries are unassigned, return an empty list (there aren't
-        // any replicas to query anyway)
-        if (noPrimariesActive()) {
-            return new PlainShardIterator(shardId, NO_SHARDS);
-        }
-
-        ArrayList<ShardRouting> ordered = new ArrayList<>(activeShards.size() + allInitializingShards.size());
-        // fill it in a randomized fashion with the active replicas
-        for (ShardRouting replica : shuffler.shuffle(replicas)) {
-            if (replica.active()) {
-                ordered.add(replica);
-            }
-        }
-
-        // Add the primary shard
-        ordered.add(primary);
-
-        // Add initializing shards last
-        if (!allInitializingShards.isEmpty()) {
-            ordered.addAll(allInitializingShards);
-        }
-        return new PlainShardIterator(shardId, ordered);
-    }
-
     public ShardIterator onlyNodeActiveInitializingShardsIt(String nodeId) {
         ArrayList<ShardRouting> ordered = new ArrayList<>(activeShards.size() + allInitializingShards.size());
         int seed = shuffler.nextSeed();

core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java

@@ -198,14 +198,6 @@ private ShardIterator preferenceActiveShardIterator(IndexShardRoutingTable index
                     return indexShard.preferNodeActiveInitializingShardsIt(nodesIds);
                 case LOCAL:
                     return indexShard.preferNodeActiveInitializingShardsIt(Collections.singleton(localNodeId));
-                case PRIMARY:
-                    return indexShard.primaryActiveInitializingShardIt();
-                case REPLICA:
-                    return indexShard.replicaActiveInitializingShardIt();
-                case PRIMARY_FIRST:
-                    return indexShard.primaryFirstActiveInitializingShardsIt();
-                case REPLICA_FIRST:
-                    return indexShard.replicaFirstActiveInitializingShardsIt();
                 case ONLY_LOCAL:
                     return indexShard.onlyNodeActiveInitializingShardsIt(localNodeId);
                 case ONLY_NODES:

core/src/main/java/org/elasticsearch/cluster/routing/Preference.java

@@ -39,26 +39,6 @@ public enum  Preference {
      */
     LOCAL("_local"),
 
-    /**
-     * Route to primary shards
-     */
-    PRIMARY("_primary"),
-
-    /**
-     * Route to replica shards
-     */
-    REPLICA("_replica"),
-
-    /**
-     * Route to primary shards first
-     */
-    PRIMARY_FIRST("_primary_first"),
-
-    /**
-     * Route to replica shards first
-     */
-    REPLICA_FIRST("_replica_first"),
-
     /**
      * Route to the local shard only
      */
@@ -97,16 +77,6 @@ public static Preference parse(String preference) {
                 return PREFER_NODES;
             case "_local":
                 return LOCAL;
-            case "_primary":
-                return PRIMARY;
-            case "_replica":
-                return REPLICA;
-            case "_primary_first":
-            case "_primaryFirst":
-                return PRIMARY_FIRST;
-            case "_replica_first":
-            case "_replicaFirst":
-                return REPLICA_FIRST;
             case "_only_local":
             case "_onlyLocal":
                 return ONLY_LOCAL;