Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x:
  Fix compil after backport
  Fix index prefixes to work with span_multi (#31066)
  Delete superfluous UpdateRequest test
  Disable MultiMatchQueryBuilderTests.
  Disable UpdateRequestTest#testUnknownFieldParsing.
  Update get.asciidoc (#31084)
  When the 'strict' mapping option is enabled, make sure to validate split_queries_on_whitespace.
  Transport client: Don't validate node in handshake (#30737) (#31080)
  In the internal highlighter APIs, use the field type as opposed to the mapper. (#31039)
  [DOCS] Removes redundant authorization pages
  Upgrade to Lucene-7.4.0-snapshot-0a7c3f462f (#31073)

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 6.4.0
-lucene            = 7.4.0-snapshot-1cbadda4d3
+lucene            = 7.4.0-snapshot-0a7c3f462f
 
 # optional dependencies
 spatial4j         = 0.7

docs/reference/docs/get.asciidoc

@@ -167,7 +167,7 @@ The result of the above get operation is:
 // TESTRESPONSE
 
 
-Field values fetched from the document it self are always returned as an array.
+Field values fetched from the document itself are always returned as an array.
 Since the `counter` field is not stored the get request simply ignores it when trying to get the `stored_fields.`
 
 It is also possible to retrieve metadata fields like the `_routing` field:

docs/reference/query-dsl/span-multi-term-query.asciidoc

@@ -36,3 +36,11 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+
+WARNING: By default `span_multi queries are rewritten to a `span_or` query
+containing **all** the expanded terms. This can be expensive if the number of expanded
+terms is large. To avoid an unbounded expansion you can set the
+<<query-dsl-multi-term-rewrite,rewrite method>> of the multi term query to `top_terms_*`
+rewrite. Or, if you use `span_multi` on `prefix` query only, you can
+activate the <<index-prefix-config,`index_prefixes`>> field option of the `text` field instead. This will
+rewrite any prefix query on the field to a a single term query that matches the indexed prefix.

modules/lang-expression/licenses/lucene-expressions-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+bf2cfa0551ebdf08a2cf3079f3c74643bd9dbb76

plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+82d83fcac1d9c8948aa0247fc9c87f177ddbd59b

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+73fd4364f2931e7c8303b5927e140a7d21116c36

plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+0a2c4417fa9a8be078864f590a5a66b98d551cf5

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+6fa179924f139a30fc0e5399256e1a44562ed32b

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+5ed135d34d7868b71a725257a46dc8d8735a15d4

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+875911b36b99c2103719f94559878a0ecb862fb6

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+e7191628df8cb72382a20da79224aef677117849

rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml

@@ -1,8 +1,8 @@
----
-"search with index prefixes":
+setup:
   - skip:
-      version: " - 6.99.99"
+      version: " - 6.2.99"
       reason: index_prefixes is only available as of 6.3.0
+
   - do:
       indices.create:
         index:  test
@@ -27,6 +27,11 @@
       indices.refresh:
         index: [test]
 
+---
+"search with index prefixes":
+  - skip:
+      version: " - 6.2.99"
+      reason: index_prefixes is only available as of 6.3.0
   - do:
       search:
         index: test
@@ -57,3 +62,23 @@
 
   - match: {hits.total: 1}
   - match: {hits.hits.0._score: 1}
+
+---
+"search index prefixes with span_multi":
+  - skip:
+      version: " - 6.2.99"
+      reason: span_multi throws an exception with prefix fields on < versions
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            span_near:
+              clauses: [
+                { "span_term": { "text": "short" } },
+                { "span_multi": { "match": { "prefix": { "text": "word" } } } }
+              ]
+
+  - match: {hits.total: 1}
+

server/licenses/lucene-analyzers-common-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+8cd761f40c4a89ed977167f0518d12e409eaf3d8

server/licenses/lucene-backward-codecs-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+8c93ed67599d345b9359586248ab92342d7d3033

server/licenses/lucene-core-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+003ed080e5184661e606091cd321c229798b22f8

server/licenses/lucene-grouping-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+0b4be9f96edfd3dbcff5aa9b3f0914e86eb9cc51

server/licenses/lucene-highlighter-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+a5dcceb5bc017cee6ab5d3ee1943aca1ac6fe074

server/licenses/lucene-join-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+b59e7441f121da969bef8eef2c0c61743b4230a8

server/licenses/lucene-memory-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+46736dbb07b432f0a7c1b3080f62932c483e5cb9

server/licenses/lucene-misc-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+ee203718d525da0c6258a51a5a32d877089fe5af

server/licenses/lucene-queries-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+cf17a332d8e42a45e8f013d5df408f4391d2620a

server/licenses/lucene-queryparser-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+04832303d70502d2ece44501cb1716f42e24fe35

server/licenses/lucene-sandbox-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+639313e3a9573779b6a28b45a7f57fc1f73ffa46

server/licenses/lucene-spatial-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+6144b493ba3588a638858d0058054758acc619b9

server/licenses/lucene-spatial-extras-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+9d00c6b8bbbbb496aecd555406267fee9e0af914

server/licenses/lucene-spatial3d-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+159cdb6d36845690cb1972d02cc0b472bb14b7f3

server/licenses/lucene-suggest-7.4.0-snapshot-0a7c3f462f.jar.sha1

@@ -0,0 +1 @@
+af1dd0218d58990cca5c1592d9722e67d233c996

server/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java

@@ -21,7 +21,7 @@
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
@@ -56,6 +56,7 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.Closeable;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -89,6 +90,7 @@ final class TransportClientNodesService extends AbstractComponent implements Clo
     private final Object mutex = new Object();
 
     private volatile List<DiscoveryNode> nodes = Collections.emptyList();
+    // Filtered nodes are nodes whose cluster name does not match the configured cluster name
     private volatile List<DiscoveryNode> filteredNodes = Collections.emptyList();
 
     private final AtomicInteger tempNodeIdGenerator = new AtomicInteger();
@@ -268,7 +270,7 @@ public static class RetryListener<Response> implements ActionListener<Response>
         private volatile int i;
 
         RetryListener(NodeListenerCallback<Response> callback, ActionListener<Response> listener,
-                             List<DiscoveryNode> nodes, int index, TransportClient.HostFailureListener hostFailureListener) {
+                      List<DiscoveryNode> nodes, int index, TransportClient.HostFailureListener hostFailureListener) {
             this.callback = callback;
             this.listener = listener;
             this.nodes = nodes;
@@ -361,10 +363,10 @@ public void sample() {
         protected abstract void doSample();
 
         /**
-         * validates a set of potentially newly discovered nodes and returns an immutable
-         * list of the nodes that has passed.
+         * Establishes the node connections. If validateInHandshake is set to true, the connection will fail if
+         * node returned in the handshake response is different than the discovery node.
          */
-        protected List<DiscoveryNode> validateNewNodes(Set<DiscoveryNode> nodes) {
+        List<DiscoveryNode> establishNodeConnections(Set<DiscoveryNode> nodes) {
             for (Iterator<DiscoveryNode> it = nodes.iterator(); it.hasNext(); ) {
                 DiscoveryNode node = it.next();
                 if (!transportService.nodeConnected(node)) {
@@ -380,7 +382,6 @@ protected List<DiscoveryNode> validateNewNodes(Set<DiscoveryNode> nodes) {
 
             return Collections.unmodifiableList(new ArrayList<>(nodes));
         }
-
     }
 
     class ScheduledNodeSampler implements Runnable {
@@ -402,14 +403,16 @@ class SimpleNodeSampler extends NodeSampler {
         @Override
         protected void doSample() {
             HashSet<DiscoveryNode> newNodes = new HashSet<>();
-            HashSet<DiscoveryNode> newFilteredNodes = new HashSet<>();
+            ArrayList<DiscoveryNode> newFilteredNodes = new ArrayList<>();
             for (DiscoveryNode listedNode : listedNodes) {
                 try (Transport.Connection connection = transportService.openConnection(listedNode, LISTED_NODES_PROFILE)){
                     final PlainTransportFuture<LivenessResponse> handler = new PlainTransportFuture<>(
                         new FutureTransportResponseHandler<LivenessResponse>() {
                             @Override
-                            public LivenessResponse newInstance() {
-                                return new LivenessResponse();
+                            public LivenessResponse read(StreamInput in) throws IOException {
+                                LivenessResponse response = new LivenessResponse();
+                                response.readFrom(in);
+                                return response;
                             }
                         });
                     transportService.sendRequest(connection, TransportLivenessAction.NAME, new LivenessRequest(),
@@ -435,8 +438,8 @@ public LivenessResponse newInstance() {
                 }
             }
 
-            nodes = validateNewNodes(newNodes);
-            filteredNodes = Collections.unmodifiableList(new ArrayList<>(newFilteredNodes));
+            nodes = establishNodeConnections(newNodes);
+            filteredNodes = Collections.unmodifiableList(newFilteredNodes);
         }
     }
 
@@ -557,7 +560,7 @@ public void handleException(TransportException e) {
                 }
             }
 
-            nodes = validateNewNodes(newNodes);
+            nodes = establishNodeConnections(newNodes);
             filteredNodes = Collections.unmodifiableList(new ArrayList<>(newFilteredNodes));
         }
     }

server/src/main/java/org/elasticsearch/index/engine/TranslogLeafReader.java

@@ -37,7 +37,6 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.index.fielddata.AbstractSortedDocValues;
-import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.ParentFieldMapper;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
@@ -58,16 +57,16 @@ final class TranslogLeafReader extends LeafReader {
     private final Translog.Index operation;
     private static final FieldInfo FAKE_SOURCE_FIELD
         = new FieldInfo(SourceFieldMapper.NAME, 1, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(),
-        0,0);
+        0, 0, false);
     private static final FieldInfo FAKE_ROUTING_FIELD
         = new FieldInfo(RoutingFieldMapper.NAME, 2, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(),
-        0,0);
+        0, 0, false);
     private static final FieldInfo FAKE_ID_FIELD
         = new FieldInfo(IdFieldMapper.NAME, 3, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(),
-        0,0);
+        0, 0, false);
     private static final FieldInfo FAKE_UID_FIELD
         = new FieldInfo(UidFieldMapper.NAME, 4, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(),
-        0,0);
+        0, 0, false);
     private final Version indexVersionCreated;
 
     TranslogLeafReader(Translog.Index operation, Version indexVersionCreated) {

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -222,6 +222,12 @@ public void checkCompatibility(MappedFieldType otherFT, List<String> conflicts,
             if (Objects.equals(normalizer, other.normalizer) == false) {
                 conflicts.add("mapper [" + name() + "] has different [normalizer]");
             }
+            if (strict) {
+                if (splitQueriesOnWhitespace != other.splitQueriesOnWhitespace) {
+                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types" +
+                        " to true to update [split_queries_on_whitespace] across all types.");
+                }
+            }
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -41,6 +41,7 @@
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.Settings;
@@ -176,7 +177,16 @@ public TextFieldMapper build(BuilderContext context) {
                 if (fieldType().isSearchable() == false) {
                     throw new IllegalArgumentException("Cannot set index_prefixes on unindexed field [" + name() + "]");
                 }
-                if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
+                // Copy the index options of the main field to allow phrase queries on
+                // the prefix field.
+                if (context.indexCreatedVersion().onOrAfter(Version.V_6_4_0)) {
+                    if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS) {
+                        // frequencies are not needed because prefix queries always use a constant score
+                        prefixFieldType.setIndexOptions(IndexOptions.DOCS);
+                    } else {
+                        prefixFieldType.setIndexOptions(fieldType.indexOptions());
+                    }
+                } else if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
                     prefixFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
                 }
                 if (fieldType.storeTermVectorOffsets()) {