Fail yaml and doc tests when they receive unexpected Warning header

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -119,6 +119,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                 current.println("      reason: $test.skipTest")
             }
             if (test.setup != null) {
+                // Insert a setup defined outside of the docs
                 String setup = setups[test.setup]
                 if (setup == null) {
                     throw new InvalidUserDataException("Couldn't find setup "
@@ -136,13 +137,23 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
             response.contents.eachLine { current.println("        $it") }
         }
 
-        void emitDo(String method, String pathAndQuery,
-                String body, String catchPart, boolean inSetup) {
+        void emitDo(String method, String pathAndQuery, String body,
+                String catchPart, List warnings, boolean inSetup) {
             def (String path, String query) = pathAndQuery.tokenize('?')
             current.println("  - do:")
             if (catchPart != null) {
                 current.println("      catch: $catchPart")
             }
+            if (false == warnings.isEmpty()) {
+                current.println("      warnings:")
+                for (String warning in warnings) {
+                    // Escape " because we're going to quote the warning
+                    String escaped = warning.replaceAll('"', '\\\\"')
+                    /* Quote the warning in case it starts with [ which makes
+                     * it look too much like an array. */
+                    current.println("         - \"$escaped\"")
+                }
+            }
             current.println("      raw:")
             current.println("        method: $method")
             current.println("        path: \"$path\"")
@@ -200,7 +211,8 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                     // Leading '/'s break the generated paths
                     pathAndQuery = pathAndQuery.substring(1)
                 }
-                emitDo(method, pathAndQuery, body, catchPart, inSetup)
+                emitDo(method, pathAndQuery, body, catchPart, snippet.warnings,
+                    inSetup)
             }
         }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

@@ -37,8 +37,9 @@ public class SnippetsTask extends DefaultTask {
     private static final String CATCH = /catch:\s*((?:\/[^\/]+\/)|[^ \]]+)/
     private static final String SKIP = /skip:([^\]]+)/
     private static final String SETUP = /setup:([^ \]]+)/
+    private static final String WARNING = /warning:(.+)/
     private static final String TEST_SYNTAX =
-        /(?:$CATCH|$SUBSTITUTION|$SKIP|(continued)|$SETUP) ?/
+        /(?:$CATCH|$SUBSTITUTION|$SKIP|(continued)|$SETUP|$WARNING) ?/
 
     /**
      * Action to take on each snippet. Called with a single parameter, an
@@ -158,6 +159,10 @@ public class SnippetsTask extends DefaultTask {
                                 snippet.setup = it.group(6)
                                 return
                             }
+                            if (it.group(7) != null) {
+                                snippet.warnings.add(it.group(7))
+                                return
+                            }
                             throw new InvalidUserDataException(
                                     "Invalid test marker: $line")
                         }
@@ -230,6 +235,7 @@ public class SnippetsTask extends DefaultTask {
         String language = null
         String catchPart = null
         String setup = null
+        List warnings = new ArrayList()
 
         @Override
         public String toString() {
@@ -254,6 +260,9 @@ public class SnippetsTask extends DefaultTask {
                 if (setup) {
                     result += "[setup:$setup]"
                 }
+                for (String warning in warnings) {
+                    result += "[warning:$warning]"
+                }
             }
             if (testResponse) {
                 result += '// TESTRESPONSE'

docs/README.asciidoc

@@ -28,6 +28,10 @@ are tests even if they don't have `// CONSOLE`.
   * `// TEST[setup:name]`: Run some setup code before running the snippet. This
   is useful for creating and populating indexes used in the snippet. The setup
   code is defined in `docs/build.gradle`.
+  * `// TEST[warning:some warning]`: Expect the response to include a `Warning`
+  header. If the response doesn't include a `Warning` header with the exact
+  text then the test fails. If the response includes `Warning` headers that
+  aren't expected then the test fails.
 * `// TESTRESPONSE`: Matches this snippet against the body of the response of
   the last test. If the response is JSON then order is ignored. With
   `// TEST[continued]` you can make tests that contain multiple command snippets

docs/plugins/mapper-attachments.asciidoc

@@ -196,14 +196,14 @@ PUT /test
         "file" : {
           "type" : "attachment",
           "fields" : {
-            "content" : {"index" : "no"},
-            "title" : {"store" : "yes"},
-            "date" : {"store" : "yes"},
+            "content" : {"index" : true},
+            "title" : {"store" : true},
+            "date" : {"store" : true},
             "author" : {"analyzer" : "my_analyzer"},
-            "keywords" : {"store" : "yes"},
-            "content_type" : {"store" : "yes"},
-            "content_length" : {"store" : "yes"},
-            "language" : {"store" : "yes"}
+            "keywords" : {"store" : true},
+            "content_type" : {"store" : true},
+            "content_length" : {"store" : true},
+            "language" : {"store" : true}
           }
         }
       }

docs/reference/indices/analyze.asciidoc

@@ -127,7 +127,7 @@ experimental[The format of the additional detail information is experimental and
 GET _analyze
 {
   "tokenizer" : "standard",
-  "token_filter" : ["snowball"],
+  "filter" : ["snowball"],
   "text" : "detailed output",
   "explain" : true,
   "attributes" : ["keyword"] <1>

docs/reference/mapping/params/lat-lon.asciidoc

@@ -1,6 +1,9 @@
 [[lat-lon]]
 === `lat_lon`
 
+deprecated[5.0.0, ????????]
+// https://github.com/elastic/elasticsearch/issues/19792
+
 <<geo-queries,Geo-queries>> are usually performed by plugging the value of
 each <<geo-point,`geo_point`>> field into a formula to determine whether it
 falls into the required area or not. Unlike most queries, the inverted index
@@ -10,7 +13,7 @@ Setting `lat_lon` to `true` causes the latitude and longitude values to be
 indexed as numeric fields (called `.lat` and `.lon`). These fields can be used
 by the <<query-dsl-geo-bounding-box-query,`geo_bounding_box`>> and
 <<query-dsl-geo-distance-query,`geo_distance`>> queries instead of
-performing in-memory calculations.
+performing in-memory calculations. So this mapping:
 
 [source,js]
 --------------------------------------------------
@@ -27,8 +30,15 @@ PUT my_index
     }
   }
 }
+--------------------------------------------------
+// TEST[warning:geo_point lat_lon parameter is deprecated and will be removed in the next major release]
+<1> Setting `lat_lon` to true indexes the geo-point in the `location.lat` and `location.lon` fields.
+
+Allows these actions:
 
-PUT my_index/my_type/1
+[source,js]
+--------------------------------------------------
+PUT my_index/my_type/1?refresh
 {
   "location": {
     "lat": 41.12,
@@ -46,18 +56,17 @@ GET my_index/_search
         "lon": -71
       },
       "distance": "50km",
-      "optimize_bbox": "indexed" <2>
+      "optimize_bbox": "indexed" <1>
     }
   }
 }
 --------------------------------------------------
 // CONSOLE
-<1> Setting `lat_lon` to true indexes the geo-point in the `location.lat` and `location.lon` fields.
-<2> The `indexed` option tells the geo-distance query to use the inverted index instead of the in-memory calculation.
+// TEST[continued]
+<1> The `indexed` option tells the geo-distance query to use the inverted index instead of the in-memory calculation.
 
 Whether the in-memory or indexed operation performs better depends both on
 your dataset and on the types of queries that you are running.
 
 NOTE: The `lat_lon` option only makes sense for single-value `geo_point`
 fields. It will not work with arrays of geo-points.
-

docs/reference/query-dsl/function-score-query.asciidoc

@@ -18,7 +18,7 @@ GET /_search
 {
     "query": {
         "function_score": {
-            "query": {},
+            "query": { "match_all": {} },
             "boost": "5",
             "random_score": {}, <1>
             "boost_mode":"multiply"
@@ -40,17 +40,17 @@ GET /_search
 {
     "query": {
         "function_score": {
-          "query": {},
+          "query": { "match_all": {} },
           "boost": "5", <1>
           "functions": [
               {
-                  "filter": {},
+                  "filter": { "match": { "test": "bar" } },
                   "random_score": {}, <2>
                   "weight": 23
               },
               {
-                  "filter": {},
-                  "weight": 42 
+                  "filter": { "match": { "test": "cat" } },
+                  "weight": 42
               }
           ],
           "max_boost": 42,
@@ -170,7 +170,7 @@ you wish to inhibit this, set `"boost_mode": "replace"`
 The `weight` score allows you to multiply the score by the provided
 `weight`. This can sometimes be desired since boost value set on
 specific queries gets normalized, while for this score function it does
-not. The number value is of type float. 
+not. The number value is of type float.
 
 [source,js]
 --------------------------------------------------

docs/reference/query-dsl/indices-query.asciidoc

@@ -1,6 +1,8 @@
 [[query-dsl-indices-query]]
 === Indices Query
 
+deprecated[5.0.0, Search on the '_index' field instead]
+
 The `indices` query is useful in cases where a search is executed across
 multiple indices. It allows to specify a list of index names and an inner
 query that is only executed for indices matching names on that list.
@@ -20,7 +22,8 @@ GET /_search
     }
 }
 --------------------------------------------------
-// CONSOLE 
+// CONSOLE
+// TEST[warning:indices query is deprecated. Instead search on the '_index' field]
 
 You can use the `index` field to provide a single index.
 

docs/reference/query-dsl/mlt-query.asciidoc

@@ -6,7 +6,7 @@ set of documents. In order to do so, MLT selects a set of representative terms
 of these input documents, forms a query using these terms, executes the query
 and returns the results. The user controls the input documents, how the terms
 should be selected and how the query is formed. `more_like_this` can be
-shortened to `mlt` deprecated[5.0.0,use `more_like_this` instead).
+shortened to `mlt` deprecated[5.0.0,use `more_like_this` instead].
 
 The simplest use case consists of asking for documents that are similar to a
 provided piece of text. Here, we are asking for all movies that have some text
@@ -175,7 +175,7 @@ follows a similar syntax to the `per_field_analyzer` parameter of the
 Additionally, to provide documents not necessarily present in the index,
 <<docs-termvectors-artificial-doc,artificial documents>> are also supported.
 
-`unlike`:: 
+`unlike`::
 The `unlike` parameter is used in conjunction with `like` in order not to
 select terms found in a chosen set of documents. In other words, we could ask
 for documents `like: "Apple"`, but `unlike: "cake crumble tree"`. The syntax

docs/reference/query-dsl/parent-id-query.asciidoc

@@ -57,7 +57,7 @@ GET /my_index/_search
 {
   "query": {
     "has_parent": {
-      "type": "blog_post",
+      "parent_type": "blog_post",
         "query": {
           "term": {
             "_id": "1"

docs/reference/query-dsl/percolate-query.asciidoc

@@ -19,7 +19,7 @@ PUT /my-index
         "doctype": {
             "properties": {
                 "message": {
-                    "type": "string"
+                    "type": "keyword"
                 }
             }
         },

docs/reference/query-dsl/prefix-query.asciidoc

@@ -28,7 +28,7 @@ GET /_search
 --------------------------------------------------
 // CONSOLE
 
-Or :
+Or with the `prefix` deprecated[5.0.0, Use `value`] syntax:
 
 [source,js]
 --------------------------------------------------
@@ -39,6 +39,7 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[warning:Deprecated field [prefix] used, expected [value] instead]
 
 This multi term query allows you to control how it gets rewritten using the
 <<query-dsl-multi-term-rewrite,rewrite>>

docs/reference/query-dsl/template-query.asciidoc

@@ -1,6 +1,8 @@
 [[query-dsl-template-query]]
 === Template Query
 
+deprecated[5.0.0, Use the <<search-template>> API]
+
 A query that accepts a query template and a map of key/value pairs to fill in
 template parameters. Templating is based on Mustache. For simple token substitution all you provide
 is a query containing some variable that you want to substitute and the actual
@@ -21,6 +23,7 @@ GET /_search
 }
 ------------------------------------------
 // CONSOLE
+// TEST[warning:[template] query is deprecated, use search template api instead]
 
 The above request is translated into:
 
@@ -54,6 +57,7 @@ GET /_search
 }
 ------------------------------------------
 // CONSOLE
+// TEST[warning:[template] query is deprecated, use search template api instead]
 
 <1> New line characters (`\n`) should be escaped as `\\n` or removed,
     and quotes (`"`) should be escaped as `\\"`.
@@ -80,6 +84,7 @@ GET /_search
 }
 ------------------------------------------
 // CONSOLE
+// TEST[warning:[template] query is deprecated, use search template api instead]
 
 <1> Name of the query template in `config/scripts/`, i.e., `my_template.mustache`.
 
@@ -113,11 +118,10 @@ GET /_search
 ------------------------------------------
 // CONSOLE
 // TEST[continued]
+// TEST[warning:[template] query is deprecated, use search template api instead]
 
 <1> Name of the query template in `config/scripts/`, i.e., `my_template.mustache`.
 
 
 There is also a dedicated `template` endpoint, allows you to template an entire search request.
 Please see <<search-template>> for more details.
-
-

docs/reference/search/request/highlighting.asciidoc

@@ -52,9 +52,9 @@ It tries hard to reflect the query matching logic in terms of understanding word
 
 [WARNING]
 If you want to highlight a lot of fields in a lot of documents with complex queries this highlighter will not be fast.
-In its efforts to accurately reflect query logic it creates a tiny in-memory index and re-runs the original query criteria through 
-Lucene's query execution planner to get access to low-level match information on the current document. 
-This is repeated for every field and every document that needs highlighting. If this presents a performance issue in your system consider using an alternative highlighter. 
+In its efforts to accurately reflect query logic it creates a tiny in-memory index and re-runs the original query criteria through
+Lucene's query execution planner to get access to low-level match information on the current document.
+This is repeated for every field and every document that needs highlighting. If this presents a performance issue in your system consider using an alternative highlighter.
 
 [[postings-highlighter]]
 ==== Postings highlighter
@@ -387,7 +387,7 @@ GET /_search
                 "match_phrase": {
                     "content": {
                         "query": "foo bar",
-                        "phrase_slop": 1
+                        "slop": 1
                     }
                 }
             },
@@ -413,7 +413,7 @@ GET /_search
                             "match_phrase": {
                                 "content": {
                                     "query": "foo bar",
-                                    "phrase_slop": 1,
+                                    "slop": 1,
                                     "boost": 10.0
                                 }
                             }

docs/reference/search/request/source-filtering.asciidoc

@@ -53,15 +53,16 @@ GET /_search
 --------------------------------------------------
 // CONSOLE
 
-Finally, for complete control, you can specify both include and exclude patterns:
+Finally, for complete control, you can specify both `includes` and `excludes`
+patterns:
 
 [source,js]
 --------------------------------------------------
 GET /_search
 {
     "_source": {
-        "include": [ "obj1.*", "obj2.*" ],
-        "exclude": [ "*.description" ]
+        "includes": [ "obj1.*", "obj2.*" ],
+        "excludes": [ "*.description" ]
     },
     "query" : {
         "term" : { "user" : "kimchy" }