Merge branch 'master' into long_sort_optimization

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -57,6 +57,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -880,6 +881,7 @@ private List<File> getInstalledFileSet(Action<? super PatternFilterable> filter)
             // https://docs.gradle.org/nightly/release-notes.html#improved-handling-of-zip-archives-on-classpaths
             .map(zipFile -> project.zipTree(zipFile).matching(filter))
             .flatMap(tree -> tree.getFiles().stream())
+            .sorted(Comparator.comparing(File::getName))
             .collect(Collectors.toList());
     }
 
@@ -901,8 +903,13 @@ private List<File> getInstalledFiles() {
     }
 
     @Classpath
-    private Set<File> getDistributionClasspath() {
-        return project.fileTree(getExtractedDistributionDir()).matching(filter -> filter.include("**/*.jar")).getFiles();
+    private List<File> getDistributionClasspath() {
+        ArrayList<File> files = new ArrayList<>(project.fileTree(getExtractedDistributionDir())
+            .matching(filter -> filter.include("**/*.jar"))
+            .getFiles());
+        files.sort(Comparator.comparing(File::getName));
+
+        return files;
     }
 
     @InputFiles

client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

@@ -192,8 +192,7 @@ public void testIndicesExistsWithTypes() throws IOException {
             = new org.elasticsearch.action.admin.indices.get.GetIndexRequest();
         request.indices(indexName);
 
-        boolean response = execute(request, highLevelClient().indices()::exists, highLevelClient().indices()::existsAsync,
-                expectWarnings(RestGetIndicesAction.TYPES_DEPRECATION_MESSAGE));
+        boolean response = execute(request, highLevelClient().indices()::exists, highLevelClient().indices()::existsAsync);
         assertTrue(response);
     }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -164,6 +164,7 @@
 import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.CoreMatchers.hasItems;
 import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -1406,6 +1407,7 @@ public void testStartDataFrameAnalyticsConfig() throws Exception {
         assertTrue(highLevelClient().indices().exists(new GetIndexRequest(destIndex), RequestOptions.DEFAULT));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/43924")
     public void testStopDataFrameAnalyticsConfig() throws Exception {
         String sourceIndex = "stop-test-source-index";
         String destIndex = "stop-test-dest-index";
@@ -1437,7 +1439,8 @@ public void testStopDataFrameAnalyticsConfig() throws Exception {
             new StartDataFrameAnalyticsRequest(configId),
             machineLearningClient::startDataFrameAnalytics, machineLearningClient::startDataFrameAnalyticsAsync);
         assertTrue(startDataFrameAnalyticsResponse.isAcknowledged());
-        assertThat(getAnalyticsState(configId), equalTo(DataFrameAnalyticsState.STARTED));
+        assertThat(getAnalyticsState(configId), anyOf(equalTo(DataFrameAnalyticsState.STARTED),
+            equalTo(DataFrameAnalyticsState.REINDEXING), equalTo(DataFrameAnalyticsState.ANALYZING)));
 
         StopDataFrameAnalyticsResponse stopDataFrameAnalyticsResponse = execute(
             new StopDataFrameAnalyticsRequest(configId),

docs/plugins/repository-s3.asciidoc

@@ -145,6 +145,20 @@ settings belong in the `elasticsearch.yml` file.
     Whether retries should be throttled (i.e. should back off). Must be `true`
     or `false`. Defaults to `true`.
 
+`path_style_access`::
+
+   Whether to force the use of the path style access pattern. If `true`, the
+   path style access pattern will be used. If `false`, the access pattern will
+   be automatically determined by the AWS Java SDK (See
+   https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#setPathStyleAccessEnabled-java.lang.Boolean-[AWS
+   documentation] for details). Defaults to `false`.
+
+[[repository-s3-path-style-deprecation]]
+NOTE: In versions `7.0`, `7.1`, `7.2` and `7.3` all bucket operations used the
+https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/[now-deprecated]
+path style access pattern. If your deployment requires the path style access
+pattern then you should set this setting to `true` when upgrading.
+
 [float]
 [[repository-s3-compatible-services]]
 ===== S3-compatible services
@@ -381,10 +395,6 @@ bucket, in this example, named "foo".
 The bucket needs to exist to register a repository for snapshots. If you did not
 create the bucket then the repository registration will fail.
 
-Note: Starting in version 7.0, all bucket operations are using the path style
-access pattern. In previous versions the decision to use virtual hosted style or
-path style access was made by the AWS Java SDK.
-
 [[repository-s3-aws-vpc]]
 [float]
 ==== AWS VPC Bandwidth Settings

docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc

@@ -8,12 +8,8 @@
 
 Delete auto-follow patterns.
 
-==== Description
-
-This API deletes a configured collection of
-{stack-ov}/ccr-auto-follow.html[auto-follow patterns].
-
-==== Request
+[[ccr-delete-auto-follow-pattern-request]]
+==== {api-request-title}
 
 //////////////////////////
 
@@ -42,17 +38,28 @@ DELETE /_ccr/auto_follow/<auto_follow_pattern_name>
 // CONSOLE
 // TEST[s/<auto_follow_pattern_name>/my_auto_follow_pattern/]
 
-==== Path Parameters
-`auto_follow_pattern_name` (required)::
-  (string) specifies the auto-follow pattern collection to delete
-
-==== Authorization
+[[ccr-delete-auto-follow-pattern-prereqs]]
+==== {api-prereq-title}
 
-If the {es} {security-features} are enabled, you must have `manage_ccr` cluster
+* If the {es} {security-features} are enabled, you must have `manage_ccr` cluster
 privileges on the cluster that contains the follower index. For more information,
 see {stack-ov}/security-privileges.html[Security privileges].
 
-==== Example
+[[ccr-delete-auto-follow-pattern-desc]]
+==== {api-description-title}
+
+This API deletes a configured collection of
+{stack-ov}/ccr-auto-follow.html[auto-follow patterns].
+
+[[ccr-delete-auto-follow-pattern-path-parms]]
+==== {api-path-parms-title}
+
+`<auto_follow_pattern_name>` (Required)::
+  (string) Specifies the auto-follow pattern collection to delete.
+
+
+[[ccr-delete-auto-follow-pattern-examples]]
+==== {api-examples-title}
 
 This example deletes an auto-follow pattern collection named
 `my_auto_follow_pattern`:

docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc

@@ -8,12 +8,8 @@
 
 Get auto-follow patterns.
 
-==== Description
-
-This API gets configured {stack-ov}/ccr-auto-follow.html[auto-follow patterns].
-This API will return the specified auto-follow pattern collection.
-
-==== Request
+[[ccr-get-auto-follow-pattern-request]]
+==== {api-request-title}
 
 //////////////////////////
 
@@ -56,19 +52,29 @@ GET /_ccr/auto_follow/<auto_follow_pattern_name>
 // CONSOLE
 // TEST[s/<auto_follow_pattern_name>/my_auto_follow_pattern/]
 
-==== Path Parameters
-`auto_follow_pattern_name`::
-  (string) specifies the auto-follow pattern collection that you want to
-  retrieve; if you do not specify a name, the API returns information for all
-  collections
-
-==== Authorization
+[[ccr-get-auto-follow-pattern-prereqs]]
+==== {api-prereq-title}
 
-If the {es} {security-features} are enabled, you must have `manage_ccr` cluster
+* If the {es} {security-features} are enabled, you must have `manage_ccr` cluster
 privileges on the cluster that contains the follower index. For more information,
 see {stack-ov}/security-privileges.html[Security privileges].
 
-==== Example
+[[ccr-get-auto-follow-pattern-desc]]
+==== {api-description-title}
+
+This API gets configured {stack-ov}/ccr-auto-follow.html[auto-follow patterns].
+This API will return the specified auto-follow pattern collection.
+
+[[ccr-get-auto-follow-pattern-path-parms]]
+==== {api-path-parms-title}
+
+`<auto_follow_pattern_name>` (Optional)::
+  (string) Specifies the auto-follow pattern collection that you want to
+  retrieve. If you do not specify a name, the API returns information for all
+  collections.
+
+[[ccr-get-auto-follow-pattern-examples]]
+==== {api-examples-title}
 
 This example retrieves information about an auto-follow pattern collection
 named `my_auto_follow_pattern`:

docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc

@@ -8,15 +8,8 @@
 
 Creates an auto-follow pattern.
 
-==== Description
-
-This API creates a new named collection of
-{stack-ov}/ccr-auto-follow.html[auto-follow patterns] against the remote cluster
-specified in the request body. Newly created indices on the remote cluster
-matching any of the specified patterns will be automatically configured as follower
-indices.
-
-==== Request
+[[ccr-put-auto-follow-pattern-request]]
+==== {api-request-title}
 
 [source,js]
 --------------------------------------------------
@@ -48,35 +41,49 @@ DELETE /_ccr/auto_follow/auto_follow_pattern_name
 
 //////////////////////////
 
-==== Path Parameters
-`auto_follow_pattern_name` (required)::
-  (string) name of the collection of auto-follow patterns
+[[ccr-put-auto-follow-pattern-prereqs]]
+==== {api-prereq-title}
 
-==== Request Body
-`remote_cluster`::
-  (required string) the <<modules-remote-clusters,remote cluster>> containing the
-  leader indices to match against
+* If the {es} {security-features} are enabled, you must have `read` and `monitor`
+index privileges for the leader index patterns. You must also have `manage_ccr`
+cluster privileges on the cluster that contains the follower index. For more
+information, see
+{stack-ov}/security-privileges.html[Security privileges].
 
-`leader_index_patterns`::
-  (array) an array of simple index patterns to match against indices in the
-  remote cluster specified by the `remote_cluster` field
+[[ccr-put-auto-follow-pattern-desc]]
+==== {api-description-title}
 
-`follow_index_pattern`::
-  (string) the name of follower index; the template `{{leader_index}}` can be
-  used to derive the name of the follower index from the name of the leader
-  index
+This API creates a new named collection of
+{stack-ov}/ccr-auto-follow.html[auto-follow patterns] against the remote cluster
+specified in the request body. Newly created indices on the remote cluster
+matching any of the specified patterns will be automatically configured as follower
+indices.
 
-include::../follow-request-body.asciidoc[]
+[[ccr-put-auto-follow-pattern-path-parms]]
+==== {api-path-parms-title}
+`<auto_follow_pattern_name>` (Required)::
+  (string) The name of the collection of auto-follow patterns.
 
-==== Authorization
+[[ccr-put-auto-follow-pattern-request-body]]
+==== {api-request-body-title}
 
-If the {es} {security-features} are enabled, you must have `read` and `monitor`
-index privileges for the leader index patterns. You must also have `manage_ccr`
-cluster privileges on the cluster that contains the follower index. For more
-information, see
-{stack-ov}/security-privileges.html[Security privileges].
+`remote_cluster` (Required)::
+  (string) The <<modules-remote-clusters,remote cluster>> containing the
+  leader indices to match against.
+
+`leader_index_patterns` (Optional)::
+  (array) An array of simple index patterns to match against indices in the
+  remote cluster specified by the `remote_cluster` field.
+
+`follow_index_pattern` (Optional)::
+  (string) The name of follower index. The template `{{leader_index}}` can be
+  used to derive the name of the follower index from the name of the leader
+  index.
+
+include::../follow-request-body.asciidoc[]
 
-==== Example
+[[ccr-put-auto-follow-pattern-examples]]
+==== {api-examples-title}
 
 This example creates an auto-follow pattern named `my_auto_follow_pattern`:
 

docs/reference/ccr/apis/follow-request-body.asciidoc

@@ -1,48 +1,49 @@
 [testenv="platinum"]
 `max_read_request_operation_count`::
-  (integer) the maximum number of operations to pull per read from the remote
-  cluster
+  (integer) The maximum number of operations to pull per read from the remote
+  cluster.
 
 `max_outstanding_read_requests`::
-  (long) the maximum number of outstanding reads requests from the remote
-  cluster
+  (long) The maximum number of outstanding reads requests from the remote
+  cluster.
 
 `max_read_request_size`::
-  (<<byte-units,byte value>>) the maximum size in bytes of per read of a batch
-  of operations pulled from the remote cluster
+  (<<byte-units,byte value>>) The maximum size in bytes of per read of a batch
+  of operations pulled from the remote cluster.
 
 `max_write_request_operation_count`::
-  (integer) the maximum number of operations per bulk write request executed on
-  the follower
+  (integer) The maximum number of operations per bulk write request executed on
+  the follower.
 
 `max_write_request_size`::
-  (<<byte-units,byte value>>) the maximum total bytes of operations per bulk write request
-  executed on the follower
+  (<<byte-units,byte value>>) The maximum total bytes of operations per bulk write request
+  executed on the follower.
 
 `max_outstanding_write_requests`::
-  (integer) the maximum number of outstanding write requests on the follower
+  (integer) The maximum number of outstanding write requests on the follower.
 
 `max_write_buffer_count`::
-  (integer) the maximum number of operations that can be queued for writing;
-  when this limit is reached, reads from the remote cluster will be deferred
-  until the number of queued operations goes below the limit
+  (integer) The maximum number of operations that can be queued for writing.
+  When this limit is reached, reads from the remote cluster will be deferred
+  until the number of queued operations goes below the limit.
 
 `max_write_buffer_size`::
-  (<<byte-units,byte value>>) the maximum total bytes of operations that can be queued for
-  writing; when this limit is reached, reads from the remote cluster will be
-  deferred until the total bytes of queued operations goes below the limit
+  (<<byte-units,byte value>>) The maximum total bytes of operations that can be
+  queued for
+  writing. When this limit is reached, reads from the remote cluster will be
+  deferred until the total bytes of queued operations goes below the limit.
 
 `max_retry_delay`::
-  (<<time-units,time value>>) the maximum time to wait before retrying an
-  operation that failed exceptionally; an exponential backoff strategy is
-  employed when retrying
+  (<<time-units,time value>>) The maximum time to wait before retrying an
+  operation that failed exceptionally. An exponential backoff strategy is
+  employed when retrying.
 
 `read_poll_timeout`::
-  (<<time-units,time value>>) the maximum time to wait for new operations on the
-  remote cluster when the follower index is synchronized with the leader index;
-  when the timeout has elapsed, the poll for operations will return to the
-  follower so that it can update some statistics, and then the follower will
-  immediately attempt to read from the leader again
+  (<<time-units,time value>>) The maximum time to wait for new operations on the
+  remote cluster when the follower index is synchronized with the leader index.
+  When the timeout has elapsed, the poll for operations will return to the
+  follower so that it can update some statistics. Then the follower will
+  immediately attempt to read from the leader again.
 
 ===== Default values