Merge branch 'master' into ccr

* master:
  Stop skipping REST test after backport of #27056
  Fix default value of ignore_unavailable for snapshot REST API (#27056)
  Add composite aggregator (#26800)
  Fix `ShardSplittingQuery` to respect nested documents. (#27398)
  [Docs] Restore section about multi-level parent/child relation in parent-join (#27392)
  Add TcpChannel to unify Transport implementations (#27132)
  Add note on plugin distributions in plugins folder
  Remove implementations of `TransportChannel` (#27388)
  Update Google SDK to version 1.23 (#27381)
  Fix Gradle 4.3.1 compatibility for logging (#27382)
  [Test] Change Elasticsearch startup timeout to 120s in packaging tests
  Docs/windows installer (#27369)

build.gradle

@@ -242,6 +242,7 @@ subprojects {
     "org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
     "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
+    "org.elasticsearch.plugin:aggs-composite-client:${version}": ':modules:aggs-composite',
   ]
   if (indexCompatVersions[-1].snapshot) {
     /* The last and second to last versions can be snapshots. Rather than use

buildSrc/build.gradle

@@ -19,6 +19,8 @@
 
 import java.nio.file.Files
 
+import org.gradle.util.GradleVersion
+
 apply plugin: 'groovy'
 
 group = 'org.elasticsearch.gradle'
@@ -99,9 +101,11 @@ dependencies {
 
 // Gradle 2.14+ removed ProgressLogger(-Factory) classes from the public APIs
 // Use logging dependency instead
+// Gradle 4.3.1 stopped releasing the logging jars to jcenter, just use the last available one
+GradleVersion logVersion = GradleVersion.current() > GradleVersion.version('4.3') ? GradleVersion.version('4.3') : GradleVersion.current()
 
 dependencies {
-  compileOnly "org.gradle:gradle-logging:${GradleVersion.current().getVersion()}"
+  compileOnly "org.gradle:gradle-logging:${logVersion.getVersion()}"
   compile 'ru.vyarus:gradle-animalsniffer-plugin:1.2.0' // Gradle 2.14 requires a version > 1.0.1
 }
 

client/rest-high-level/build.gradle

@@ -39,7 +39,8 @@ dependencies {
   compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
   compile "org.elasticsearch.plugin:parent-join-client:${version}"
   compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
-
+  compile "org.elasticsearch.plugin:aggs-composite-client:${version}"
+
   testCompile "org.elasticsearch.client:test:${version}"
   testCompile "org.elasticsearch.test:framework:${version}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -62,6 +62,7 @@
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.composite.CompositeAggregationBuilder;
 import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.test.ESTestCase;
@@ -647,7 +648,7 @@ public void testDefaultNamedXContents() {
 
     public void testProvidedNamedXContents() {
         List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
-        assertEquals(2, namedXContents.size());
+        assertEquals(3, namedXContents.size());
         Map<Class<?>, Integer> categories = new HashMap<>();
         List<String> names = new ArrayList<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -658,9 +659,10 @@ public void testProvidedNamedXContents() {
             }
         }
         assertEquals(1, categories.size());
-        assertEquals(Integer.valueOf(2), categories.get(Aggregation.class));
+        assertEquals(Integer.valueOf(3), categories.get(Aggregation.class));
         assertTrue(names.contains(ChildrenAggregationBuilder.NAME));
         assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME));
+        assertTrue(names.contains(CompositeAggregationBuilder.NAME));
     }
 
     private static class TrackingActionListener implements ActionListener<Integer> {

client/transport/build.gradle

@@ -32,6 +32,7 @@ dependencies {
   compile "org.elasticsearch.plugin:lang-mustache-client:${version}"
   compile "org.elasticsearch.plugin:percolator-client:${version}"
   compile "org.elasticsearch.plugin:parent-join-client:${version}"
+  compile "org.elasticsearch.plugin:aggs-composite-client:${version}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"

client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.percolator.PercolatorPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.mustache.MustachePlugin;
+import org.elasticsearch.search.aggregations.composite.CompositeAggregationPlugin;
 import org.elasticsearch.transport.Netty4Plugin;
 
 import java.util.Arrays;
@@ -44,6 +45,7 @@
  * {@link PercolatorPlugin},
  * {@link MustachePlugin},
  * {@link ParentJoinPlugin}
+ * {@link CompositeAggregationPlugin}
  * plugins for the client. These plugins are all the required modules for Elasticsearch.
  */
 @SuppressWarnings({"unchecked","varargs"})
@@ -88,7 +90,8 @@ private static void setSystemPropertyIfUnset(final String key, final String valu
                 ReindexPlugin.class,
                 PercolatorPlugin.class,
                 MustachePlugin.class,
-                ParentJoinPlugin.class));
+                ParentJoinPlugin.class,
+                CompositeAggregationPlugin.class));
 
     /**
      * Creates a new transport client with pre-installed plugins.

client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.transport.Netty4Plugin;
+import org.elasticsearch.search.aggregations.composite.CompositeAggregationPlugin;
 import org.junit.Test;
 
 import java.util.Arrays;
@@ -52,7 +53,8 @@ public void testPluginInstalled() {
     @Test
     public void testInstallPluginTwice() {
         for (Class<? extends Plugin> plugin :
-                Arrays.asList(ParentJoinPlugin.class, ReindexPlugin.class, PercolatorPlugin.class, MustachePlugin.class)) {
+                Arrays.asList(ParentJoinPlugin.class, ReindexPlugin.class, PercolatorPlugin.class,
+                    MustachePlugin.class, CompositeAggregationPlugin.class)) {
             try {
                 new PreBuiltTransportClient(Settings.EMPTY, plugin);
                 fail("exception expected");

core/src/main/java/org/elasticsearch/action/ActionListener.java

@@ -24,6 +24,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
 /**
@@ -69,6 +70,42 @@ public void onFailure(Exception e) {
         };
     }
 
+    /**
+     * Creates a listener that listens for a response (or failure) and executes the
+     * corresponding runnable when the response (or failure) is received.
+     *
+     * @param runnable the runnable that will be called in event of success or failure
+     * @param <Response> the type of the response
+     * @return a listener that listens for responses and invokes the runnable when received
+     */
+    static <Response> ActionListener<Response> wrap(Runnable runnable) {
+        return wrap(r -> runnable.run(), e -> runnable.run());
+    }
+
+    /**
+     * Converts a listener to a {@link BiConsumer} for compatibility with the {@link java.util.concurrent.CompletableFuture}
+     * api.
+     *
+     * @param listener that will be wrapped
+     * @param <Response> the type of the response
+     * @return a bi consumer that will complete the wrapped listener
+     */
+    static <Response> BiConsumer<Response, Throwable> toBiConsumer(ActionListener<Response> listener) {
+        return (response, throwable) -> {
+            if (throwable == null) {
+                listener.onResponse(response);
+            } else {
+                if (throwable instanceof Exception) {
+                    listener.onFailure((Exception) throwable);
+                } else if (throwable instanceof Error) {
+                    throw (Error) throwable;
+                } else {
+                    throw new AssertionError("Should have been either Error or Exception", throwable);
+                }
+            }
+        };
+    }
+
     /**
      * Notifies every given listener with the response passed to {@link #onResponse(Object)}. If a listener itself throws an exception
      * the exception is forwarded to {@link #onFailure(Exception)}. If in turn {@link #onFailure(Exception)} fails all remaining

core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java

@@ -380,8 +380,9 @@ public boolean includeGlobalState() {
      * @param source snapshot definition
      * @return this request
      */
+    @SuppressWarnings("unchecked")
     public CreateSnapshotRequest source(Map<String, Object> source) {
-        for (Map.Entry<String, Object> entry : ((Map<String, Object>) source).entrySet()) {
+        for (Map.Entry<String, Object> entry : source.entrySet()) {
             String name = entry.getKey();
             if (name.equals("indices")) {
                 if (entry.getValue() instanceof String) {
@@ -402,7 +403,7 @@ public CreateSnapshotRequest source(Map<String, Object> source) {
                 includeGlobalState = nodeBooleanValue(entry.getValue(), "include_global_state");
             }
         }
-        indicesOptions(IndicesOptions.fromMap((Map<String, Object>) source, IndicesOptions.lenientExpandOpen()));
+        indicesOptions(IndicesOptions.fromMap(source, indicesOptions));
         return this;
     }
 

core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java

@@ -505,6 +505,7 @@ public Settings indexSettings() {
      * @param source restore definition
      * @return this request
      */
+    @SuppressWarnings("unchecked")
     public RestoreSnapshotRequest source(Map<String, Object> source) {
         for (Map.Entry<String, Object> entry : source.entrySet()) {
             String name = entry.getKey();
@@ -558,7 +559,7 @@ public RestoreSnapshotRequest source(Map<String, Object> source) {
                 }
             }
         }
-        indicesOptions(IndicesOptions.fromMap((Map<String, Object>) source, IndicesOptions.lenientExpandOpen()));
+        indicesOptions(IndicesOptions.fromMap(source, indicesOptions));
         return this;
     }
 

core/src/main/java/org/elasticsearch/index/IndexSortConfig.java

@@ -235,7 +235,7 @@ static class FieldSortSpec {
         SortField.Type.FLOAT
     );
 
-    static SortField.Type getSortFieldType(SortField sortField) {
+    public static SortField.Type getSortFieldType(SortField sortField) {
         if (sortField instanceof SortedSetSortField) {
             return SortField.Type.STRING;
         } else if (sortField instanceof SortedNumericSortField) {