Merge branch 'master' into ccr

* master:
  Upgrade to Lucene-7.4-snapshot-6705632810 (elastic#30519)
  add version compatibility from 6.4.0 after backport, see elastic#30319 (elastic#30390)
  Security: Simplify security index listeners (elastic#30466)
  Add proper longitude validation in geo_polygon_query (elastic#30497)
  Remove Discovery.AckListener.onTimeout() (elastic#30514)
  Build: move generated-resources to build (elastic#30366)
  Reindex: Fold "with all deps" project into reindex (elastic#30154)
  Isolate REST client single host tests (elastic#30504)
  Solve Gradle deprecation warnings around shadowJar (elastic#30483)
  SAML: Process only signed data (elastic#30420)
  Remove BWC repository test (elastic#30500)
  Build: Remove xpack specific run task (elastic#30487)
  AwaitsFix IntegTestZipClientYamlTestSuiteIT#indices.split tests
  LLClient: Add setJsonEntity (elastic#30447)
  Expose CommonStatsFlags directly in IndicesStatsRequest. (elastic#30163)
  Silence IndexUpgradeIT test failures. (elastic#30430)
  Bump Gradle heap to 1792m (elastic#30484)
  [docs] add warning for read-write indices in force merge documentation (elastic#28869)
  Avoid deadlocks in cache (elastic#30461)
  Test: remove hardcoded list of unconfigured ciphers (elastic#30367)
  mute SplitIndexIT due to elastic#30416
  Docs: Test examples that recreate lang analyzers  (elastic#29535)
  BulkProcessor to retry based on status code (elastic#29329)
  Add GET Repository High Level REST API (elastic#30362)
  add a comment explaining the need for RetryOnReplicaException on missing mappings
  Add `coordinating_only` node selector (elastic#30313)
  Stop forking groovyc (elastic#30471)
  Avoid setting connection request timeout (elastic#30384)
  Use date format in `date_range` mapping before fallback to default (elastic#29310)
  Watcher: Increase HttpClient parallel sent requests (elastic#30130)

# Conflicts:
#	x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java

benchmarks/build.gradle

@@ -24,22 +24,22 @@ buildscript {
         }
     }
     dependencies {
-        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.2'
+        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
     }
 }
 
 apply plugin: 'elasticsearch.build'
-// build an uberjar with all benchmarks
-apply plugin: 'com.github.johnrengelman.shadow'
-// have the shadow plugin provide the runShadow task
-apply plugin: 'application'
+
+// order of this seciont matters, see: https://github.com/johnrengelman/shadow/issues/336
+apply plugin: 'application' // have the shadow plugin provide the runShadow task
+mainClassName = 'org.openjdk.jmh.Main'
+apply plugin: 'com.github.johnrengelman.shadow' // build an uberjar with all benchmarks
 
 // Not published so no need to assemble
 tasks.remove(assemble)
 build.dependsOn.remove('assemble')
 
 archivesBaseName = 'elasticsearch-benchmarks'
-mainClassName = 'org.openjdk.jmh.Main'
 
 test.enabled = false
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -535,10 +535,15 @@ class BuildPlugin implements Plugin<Project> {
             }
             // also apply release flag to groovy, which is used in build-tools
             project.tasks.withType(GroovyCompile) {
-                final JavaVersion targetCompatibilityVersion = JavaVersion.toVersion(it.targetCompatibility)
-                options.fork = true
-                options.forkOptions.javaHome = new File(project.compilerJavaHome)
-                options.compilerArgs << '--release' << targetCompatibilityVersion.majorVersion
+                final compilerJavaHomeFile = new File(project.compilerJavaHome)
+                // we only fork if the Gradle JDK is not the same as the compiler JDK
+                if (compilerJavaHomeFile.canonicalPath == Jvm.current().javaHome.canonicalPath) {
+                    options.fork = false
+                } else {
+                    options.fork = true
+                    options.forkOptions.javaHome = compilerJavaHomeFile
+                    options.compilerArgs << '--release' << JavaVersion.toVersion(it.targetCompatibility).majorVersion
+                }
             }
         }
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -141,9 +141,11 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
         private static final String SYNTAX = {
             String method = /(?<method>GET|PUT|POST|HEAD|OPTIONS|DELETE)/
             String pathAndQuery = /(?<pathAndQuery>[^\n]+)/
-            String badBody = /GET|PUT|POST|HEAD|OPTIONS|DELETE|#/
+            String badBody = /GET|PUT|POST|HEAD|OPTIONS|DELETE|startyaml|#/
             String body = /(?<body>(?:\n(?!$badBody)[^\n]+)+)/
-            String nonComment = /$method\s+$pathAndQuery$body?/
+            String rawRequest = /(?:$method\s+$pathAndQuery$body?)/
+            String yamlRequest = /(?:startyaml(?s)(?<yaml>.+?)(?-s)endyaml)/
+            String nonComment = /(?:$rawRequest|$yamlRequest)/
             String comment = /(?<comment>#.+)/
             /(?:$comment|$nonComment)\n+/
         }()
@@ -333,6 +335,11 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                     // Comment
                     return
                 }
+                String yamlRequest = matcher.group("yaml");
+                if (yamlRequest != null) {
+                    current.println(yamlRequest)
+                    return
+                }
                 String method = matcher.group("method")
                 String pathAndQuery = matcher.group("pathAndQuery")
                 String body = matcher.group("body")

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.4.0-snapshot-1ed95c097b
+lucene            = 7.4.0-snapshot-6705632810
 
 # optional dependencies
 spatial4j         = 0.7

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -29,6 +29,7 @@
 import org.apache.http.entity.ContentType;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
@@ -644,6 +645,17 @@ static Request indexPutSettings(UpdateSettingsRequest updateSettingsRequest) thr
         return request;
     }
 
+    static Request getRepositories(GetRepositoriesRequest getRepositoriesRequest) {
+        String[] repositories = getRepositoriesRequest.repositories() == null ? Strings.EMPTY_ARRAY : getRepositoriesRequest.repositories();
+        String endpoint = new EndpointBuilder().addPathPartAsIs("_snapshot").addCommaSeparatedPathParts(repositories).build();
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+
+        Params parameters = new Params(request);
+        parameters.withMasterTimeout(getRepositoriesRequest.masterNodeTimeout());
+        parameters.withLocal(getRepositoriesRequest.local());
+        return request;
+    }
+
     static Request putTemplate(PutIndexTemplateRequest putIndexTemplateRequest) throws IOException {
         String endpoint = new EndpointBuilder().addPathPartAsIs("_template").addPathPart(putIndexTemplateRequest.name()).build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -26,6 +26,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
+import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
@@ -189,6 +191,7 @@ public class RestHighLevelClient implements Closeable {
 
     private final IndicesClient indicesClient = new IndicesClient(this);
     private final ClusterClient clusterClient = new ClusterClient(this);
+    private final SnapshotClient snapshotClient = new SnapshotClient(this);
 
     /**
      * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the
@@ -252,6 +255,15 @@ public final ClusterClient cluster() {
         return clusterClient;
     }
 
+    /**
+     * Provides a {@link SnapshotClient} which can be used to access the Snapshot API.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html">Snapshot API on elastic.co</a>
+     */
+    public final SnapshotClient snapshot() {
+        return snapshotClient;
+    }
+
     /**
      * Executes a bulk request using the Bulk API
      *

client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java

@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.Header;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
+import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
+import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
+
+import java.io.IOException;
+
+import static java.util.Collections.emptySet;
+
+/**
+ * A wrapper for the {@link RestHighLevelClient} that provides methods for accessing the Snapshot API.
+ * <p>
+ * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html">Snapshot API on elastic.co</a>
+ */
+public final class SnapshotClient {
+    private final RestHighLevelClient restHighLevelClient;
+
+    SnapshotClient(RestHighLevelClient restHighLevelClient) {
+        this.restHighLevelClient = restHighLevelClient;
+    }
+
+    /**
+     * Gets a list of snapshot repositories. If the list of repositories is empty or it contains a single element "_all", all
+     * registered repositories are returned.
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
+     * API on elastic.co</a>
+     */
+    public GetRepositoriesResponse getRepositories(GetRepositoriesRequest getRepositoriesRequest, Header... headers)
+        throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(getRepositoriesRequest, RequestConverters::getRepositories,
+            GetRepositoriesResponse::fromXContent, emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously gets a list of snapshot repositories. If the list of repositories is empty or it contains a single element "_all", all
+     * registered repositories are returned.
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html"> Snapshot and Restore
+     * API on elastic.co</a>
+     */
+    public void getRepositoriesAsync(GetRepositoriesRequest getRepositoriesRequest,
+                                     ActionListener<GetRepositoriesResponse> listener, Header... headers) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(getRepositoriesRequest, RequestConverters::getRepositories,
+            GetRepositoriesResponse::fromXContent, listener, emptySet(), headers);
+    }
+}