Merge remote-tracking branch 'elastic/master' into run-master-service…

…-task-under-system-context

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy

@@ -149,11 +149,11 @@ public class AntFixture extends AntTask implements Fixture {
         }
 
         // the process is started (has a pid) and is bound to a network interface
-        // so now wait undil the waitCondition has been met
+        // so now evaluates if the waitCondition is successful
         // TODO: change this to a loop?
         boolean success
         try {
-            success = waitCondition(this, ant) == false
+            success = waitCondition(this, ant)
         } catch (Exception e) {
             String msg = "Wait condition caught exception for ${name}"
             logger.error(msg, e)

client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java

@@ -129,7 +129,6 @@ public void testClusterHealthYellowClusterLevel() throws IOException {
         createIndex("index2", Settings.EMPTY);
         ClusterHealthRequest request = new ClusterHealthRequest();
         request.timeout("5s");
-        request.level(ClusterHealthRequest.Level.CLUSTER);
         ClusterHealthResponse response = execute(request, highLevelClient().cluster()::health, highLevelClient().cluster()::healthAsync);
 
         assertYellowShards(response);
@@ -170,6 +169,7 @@ public void testClusterHealthYellowSpecificIndex() throws IOException {
         createIndex("index", Settings.EMPTY);
         createIndex("index2", Settings.EMPTY);
         ClusterHealthRequest request = new ClusterHealthRequest("index");
+        request.level(ClusterHealthRequest.Level.SHARDS);
         request.timeout("5s");
         ClusterHealthResponse response = execute(request, highLevelClient().cluster()::health, highLevelClient().cluster()::healthAsync);
 

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java

@@ -1568,7 +1568,7 @@ public void testClusterHealth() {
             healthRequest.level(level);
             expectedParams.put("level", level.name().toLowerCase(Locale.ROOT));
         } else {
-            expectedParams.put("level", "shards");
+            expectedParams.put("level", "cluster");
         }
         if (randomBoolean()) {
             Priority priority = randomFrom(Priority.values());

docs/java-rest/high-level/cluster/health.asciidoc

@@ -67,6 +67,7 @@ include-tagged::{doc-tests}/ClusterClientDocumentationIT.java[health-request-wai
 include-tagged::{doc-tests}/ClusterClientDocumentationIT.java[health-request-level]
 --------------------------------------------------
 <1> The level of detail of the returned health information. Accepts a `ClusterHealthRequest.Level` value.
+Default value is `cluster`.
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------

docs/reference/migration/migrate_7_0/restclient.asciidoc

@@ -10,4 +10,11 @@ header, e.g. `client.index(indexRequest)` becomes
 `client.index(indexRequest, RequestOptions.DEFAULT)`.
 In case you are specifying headers
 e.g. `client.index(indexRequest, new Header("name" "value"))` becomes
-`client.index(indexRequest, RequestOptions.DEFAULT.toBuilder().addHeader("name", "value").build());`
+`client.index(indexRequest, RequestOptions.DEFAULT.toBuilder().addHeader("name", "value").build());`
+
+==== Cluster Health API default to `cluster` level
+
+The Cluster Health API used to default to `shards` level to ease migration
+from transport client that doesn't support the `level` parameter and always
+returns information including indices and shards details. The level default
+value has been aligned with the Elasticsearch default level: `cluster`.

modules/reindex/build.gradle

@@ -121,6 +121,11 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
            baseDir,
            unzip.temporaryDir,
            version == '090'
+      waitCondition = { fixture, ant ->
+        // the fixture writes the ports file when Elasticsearch's HTTP service
+        // is ready, so we can just wait for the file to exist
+        return fixture.portsFile.exists()
+      }
     }
     integTest.dependsOn fixture
     integTestRunner {

modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLFixture.java

@@ -39,6 +39,7 @@
 import java.util.Map;
 import java.util.Objects;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonMap;
@@ -67,7 +68,6 @@ public static void main(String[] args) throws Exception {
             writeFile(workingDirectory, "ports", addressAndPort);
 
             // Exposes the repository over HTTP
-            final String url = "http://" + addressAndPort;
             httpServer.createContext("/", new ResponseHandler(dir(args[1])));
             httpServer.start();
 
@@ -110,7 +110,13 @@ static class ResponseHandler implements HttpHandler {
         @Override
         public void handle(HttpExchange exchange) throws IOException {
             Response response;
-            if ("GET".equalsIgnoreCase(exchange.getRequestMethod())) {
+
+            final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent");
+            if (userAgent != null && userAgent.startsWith("Apache Ant")) {
+                // This is a request made by the AntFixture, just reply "OK"
+                response = new Response(RestStatus.OK, emptyMap(), "text/plain; charset=utf-8", "OK".getBytes(UTF_8));
+
+            } else if ("GET".equalsIgnoreCase(exchange.getRequestMethod())) {
                 String path = exchange.getRequestURI().toString();
                 if (path.length() > 0 && path.charAt(0) == '/') {
                     path = path.substring(1);
@@ -125,13 +131,13 @@ public void handle(HttpExchange exchange) throws IOException {
                         Map<String, String> headers = singletonMap("Content-Length", String.valueOf(content.length));
                         response = new Response(RestStatus.OK, headers, "application/octet-stream", content);
                     } else {
-                        response = new Response(RestStatus.NOT_FOUND, emptyMap(), "text/plain", new byte[0]);
+                        response = new Response(RestStatus.NOT_FOUND, emptyMap(), "text/plain; charset=utf-8", new byte[0]);
                     }
                 } else {
-                    response = new Response(RestStatus.FORBIDDEN, emptyMap(), "text/plain", new byte[0]);
+                    response = new Response(RestStatus.FORBIDDEN, emptyMap(), "text/plain; charset=utf-8", new byte[0]);
                 }
             } else {
-                response = new Response(RestStatus.INTERNAL_SERVER_ERROR, emptyMap(), "text/plain",
+                response = new Response(RestStatus.INTERNAL_SERVER_ERROR, emptyMap(), "text/plain; charset=utf-8",
                     "Unsupported HTTP method".getBytes(StandardCharsets.UTF_8));
             }
             exchange.sendResponseHeaders(response.status.getStatus(), response.body.length);

plugins/examples/rest-handler/src/test/java/org/elasticsearch/example/resthandler/ExampleFixtureIT.java

@@ -23,25 +23,41 @@
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.BufferedReader;
+import java.io.BufferedWriter;
 import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
 import java.net.InetAddress;
 import java.net.Socket;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
-import java.util.Objects;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.Matchers.hasItems;
 
 public class ExampleFixtureIT extends ESTestCase {
 
     public void testExample() throws Exception {
-        final String stringAddress = Objects.requireNonNull(System.getProperty("external.address"));
-        final URL url = new URL("http://" + stringAddress);
+        final String externalAddress = System.getProperty("external.address");
+        assertNotNull("External address must not be null", externalAddress);
 
+        final URL url = new URL("http://" + externalAddress);
         final InetAddress address = InetAddress.getByName(url.getHost());
         try (
             Socket socket = new MockSocket(address, url.getPort());
+            BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8));
             BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))
         ) {
-            assertEquals("TEST", reader.readLine());
+            writer.write("GET / HTTP/1.1\r\n");
+            writer.write("Host: elastic.co\r\n\r\n");
+            writer.flush();
+
+            final List<String> lines = new ArrayList<>();
+            String line;
+            while ((line = reader.readLine()) != null) {
+                lines.add(line);
+            }
+            assertThat(lines, hasItems("HTTP/1.1 200 OK", "TEST"));
         }
     }
 }

plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageFixture.java

@@ -24,6 +24,8 @@
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.mocksocket.MockHttpServer;
+import org.elasticsearch.repositories.azure.AzureStorageTestServer.Response;
+import org.elasticsearch.rest.RestStatus;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -39,6 +41,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Collections.emptyMap;
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
 
@@ -121,7 +125,16 @@ public void handle(HttpExchange exchange) throws IOException {
             ByteArrayOutputStream out = new ByteArrayOutputStream();
             Streams.copy(exchange.getRequestBody(), out);
 
-            final AzureStorageTestServer.Response response = server.handle(method, path, query, headers, out.toByteArray());
+            Response response = null;
+
+            final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent");
+            if (userAgent != null && userAgent.startsWith("Apache Ant")) {
+                // This is a request made by the AntFixture, just reply "OK"
+                response = new Response(RestStatus.OK, emptyMap(), "text/plain; charset=utf-8", "OK".getBytes(UTF_8));
+            } else {
+                // Otherwise simulate a S3 response
+                response = server.handle(method, path, query, headers, out.toByteArray());
+            }
 
             Map<String, List<String>> responseHeaders = exchange.getResponseHeaders();
             responseHeaders.put("Content-Type", singletonList(response.contentType));

plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageFixture.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.core.internal.io.Streams;
 import org.elasticsearch.mocksocket.MockHttpServer;
 import org.elasticsearch.repositories.gcs.GoogleCloudStorageTestServer.Response;
+import org.elasticsearch.rest.RestStatus;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -40,6 +41,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Collections.emptyMap;
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
 
@@ -123,7 +126,16 @@ public void handle(HttpExchange exchange) throws IOException {
             ByteArrayOutputStream out = new ByteArrayOutputStream();
             Streams.copy(exchange.getRequestBody(), out);
 
-            final Response storageResponse = storageServer.handle(method, path, query, headers, out.toByteArray());
+            Response storageResponse = null;
+
+            final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent");
+            if (userAgent != null && userAgent.startsWith("Apache Ant")) {
+                // This is a request made by the AntFixture, just reply "OK"
+                storageResponse = new Response(RestStatus.OK, emptyMap(), "text/plain; charset=utf-8", "OK".getBytes(UTF_8));
+            } else {
+                // Otherwise simulate a S3 response
+                storageResponse = storageServer.handle(method, path, query, headers, out.toByteArray());
+            }
 
             Map<String, List<String>> responseHeaders = exchange.getResponseHeaders();
             responseHeaders.put("Content-Type", singletonList(storageResponse.contentType));

plugins/repository-hdfs/build.gradle

@@ -116,6 +116,11 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture',
     dependsOn project.configurations.hdfsFixture
     executable = new File(project.runtimeJavaHome, 'bin/java')
     env 'CLASSPATH', "${ -> project.configurations.hdfsFixture.asPath }"
+    waitCondition = { fixture, ant ->
+      // the hdfs.MiniHDFS fixture writes the ports file when
+      // it's ready, so we can just wait for the file to exist
+      return fixture.portsFile.exists()
+    }
 
     final List<String> miniHDFSArgs = []
 

plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.mocksocket.MockHttpServer;
 import org.elasticsearch.repositories.s3.AmazonS3TestServer.Response;
+import org.elasticsearch.rest.RestStatus;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -40,6 +41,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Collections.emptyMap;
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
 
@@ -122,7 +125,16 @@ public void handle(HttpExchange exchange) throws IOException {
             ByteArrayOutputStream out = new ByteArrayOutputStream();
             Streams.copy(exchange.getRequestBody(), out);
 
-            final Response storageResponse = storageServer.handle(method, path, query, headers, out.toByteArray());
+            Response storageResponse = null;
+
+            final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent");
+            if (userAgent != null && userAgent.startsWith("Apache Ant")) {
+                // This is a request made by the AntFixture, just reply "OK"
+                storageResponse = new Response(RestStatus.OK, emptyMap(), "text/plain; charset=utf-8", "OK".getBytes(UTF_8));
+            } else {
+                // Otherwise simulate a S3 response
+                storageResponse = storageServer.handle(method, path, query, headers, out.toByteArray());
+            }
 
             Map<String, List<String>> responseHeaders = exchange.getResponseHeaders();
             responseHeaders.put("Content-Type", singletonList(storageResponse.contentType));

server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java

@@ -48,9 +48,9 @@ public class ClusterHealthRequest extends MasterNodeReadRequest<ClusterHealthReq
     private Priority waitForEvents = null;
     /**
      * Only used by the high-level REST Client. Controls the details level of the health information returned.
-     * The default value is 'shards' so it is backward compatible with the transport client behaviour.
+     * The default value is 'cluster'.
      */
-    private Level level = Level.SHARDS;
+    private Level level = Level.CLUSTER;
 
     public ClusterHealthRequest() {
     }
@@ -250,8 +250,7 @@ public Priority waitForEvents() {
 
     /**
      * Set the level of detail for the health information to be returned.
-     * Only used by the high-level REST Client
-     * The default value is 'shards' so it is backward compatible with the transport client behaviour.
+     * Only used by the high-level REST Client.
      */
     public void level(Level level) {
         this.level = Objects.requireNonNull(level, "level must not be null");
@@ -260,7 +259,6 @@ public void level(Level level) {
     /**
      * Get the level of detail for the health information to be returned.
      * Only used by the high-level REST Client.
-     * The default value is 'shards' so it is backward compatible with the transport client behaviour.
      */
     public Level level() {
         return level;

server/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java

@@ -234,11 +234,11 @@ static int prepareResponse(final ClusterHealthRequest request, final ClusterHeal
             ActiveShardCount waitForActiveShards = request.waitForActiveShards();
             assert waitForActiveShards.equals(ActiveShardCount.DEFAULT) == false :
                 "waitForActiveShards must not be DEFAULT on the request object, instead it should be NONE";
-            if (waitForActiveShards.equals(ActiveShardCount.ALL)
-                    && response.getUnassignedShards() == 0
-                    && response.getInitializingShards() == 0) {
-                // if we are waiting for all shards to be active, then the num of unassigned and num of initializing shards must be 0
-                waitForCounter++;
+            if (waitForActiveShards.equals(ActiveShardCount.ALL)) {
+                if (response.getUnassignedShards() == 0 && response.getInitializingShards() == 0) {
+                    // if we are waiting for all shards to be active, then the num of unassigned and num of initializing shards must be 0
+                    waitForCounter++;
+                }
             } else if (waitForActiveShards.enoughShardsActive(response.getActiveShards())) {
                 // there are enough active shards to meet the requirements of the request
                 waitForCounter++;

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java

@@ -82,8 +82,10 @@ public void updateSettings(final UpdateSettingsClusterStateUpdateRequest request
         Settings.Builder settingsForOpenIndices = Settings.builder();
         final Set<String> skippedSettings = new HashSet<>();
 
-        indexScopedSettings.validate(normalizedSettings.filter(s -> Regex.isSimpleMatchPattern(s) == false  /* don't validate wildcards */),
-            false); //don't validate dependencies here we check it below never allow to change the number of shards
+        indexScopedSettings.validate(
+                normalizedSettings.filter(s -> Regex.isSimpleMatchPattern(s) == false), // don't validate wildcards
+                false, // don't validate dependencies here we check it below never allow to change the number of shards
+                true); // validate internal index settings
         for (String key : normalizedSettings.keySet()) {
             Setting setting = indexScopedSettings.get(key);
             boolean isWildcard = setting == null && Regex.isSimpleMatchPattern(key);