Merge remote-tracking branch 'origin/main' into profiler-plugin

.github/ISSUE_TEMPLATE/bug.yml

@@ -4,22 +4,27 @@ labels: [">bug", "needs:triage"]
 body:
   - type: markdown
     attributes:
-      value: |
+      value: >
         Github is reserved for bug reports and feature requests; it is
         not the place for general questions. If you have a question or
         an unconfirmed bug, please visit the [forums](https://discuss.elastic.co/c/elasticsearch).
-        Please also check your OS is [supported](https://www.elastic.co/support/matrix#show_os).
-        If it is not, the issue is likely to be closed.
+        Please also check your OS is [supported](https://www.elastic.co/support/matrix#show_os),
+        and that the version of Elasticsearch has not passed [end-of-life](https://www.elastic.co/support/eol).
+        If you are using an unsupported OS or an unsupported version then the issue is likely to be closed.
+
 
         For security vulnerabilities please only send reports to [email protected].
         See https://www.elastic.co/community/security for more information.
 
+
         Please fill in the following details to help us reproduce the bug:
   - type: input
     id: es_version
     attributes:
       label: Elasticsearch Version
-      description: The version of Elasticsearch you are running, found with `bin/elasticsearch --version`
+      description: >-
+        The version of Elasticsearch you are running, found with `bin/elasticsearch --version`.
+        Ensure this version has not [passed end-of-life](https://www.elastic.co/support/eol).
     validations:
       required: true
   - type: input

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/PrecommitTaskPlugin.java

@@ -31,10 +31,10 @@ public void apply(Project project) {
                         "lifecycle-base",
                         p -> project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(precommit))
                 );
-        project.getPluginManager().withPlugin("java", p -> {
+        project.getPluginManager().withPlugin("java-base", p -> {
             // run compilation as part of precommit
             project.getExtensions().getByType(JavaPluginExtension.class).getSourceSets().all(sourceSet ->
-                    precommit.configure(t -> t.shouldRunAfter(sourceSet.getClassesTaskName()))
+                    precommit.configure(t -> t.dependsOn(sourceSet.getClassesTaskName()))
             );
             // make sure tests run after all precommit tasks
             project.getTasks().withType(Test.class).configureEach(t -> t.mustRunAfter(precommit));

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -38,8 +38,8 @@ public class ElasticsearchJavaBasePlugin implements Plugin<Project> {
     public void apply(Project project) {
         // make sure the global build info plugin is applied to the root project
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
-        // common repositories setup
         project.getPluginManager().apply(JavaBasePlugin.class);
+        // common repositories setup
         project.getPluginManager().apply(RepositoriesSetupPlugin.class);
         project.getPluginManager().apply(ElasticsearchTestBasePlugin.class);
         project.getPluginManager().apply(PrecommitTaskPlugin.class);

build.gradle

@@ -137,9 +137,9 @@ tasks.register("verifyVersions") {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
+boolean bwc_tests_enabled = true
 // place a PR link here when committing bwc changes:
-String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/91823"
+String bwc_tests_disabled_issue = ""
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

docs/changelog/91256.yaml

@@ -0,0 +1,6 @@
+pr: 91256
+summary: Prevalidate node removal API (pt. 2)
+area: Allocation
+type: enhancement
+issues:
+ - 87776

docs/changelog/91508.yaml

@@ -0,0 +1,5 @@
+pr: 91508
+summary: "Add kibana.stats.elasticsearch_client stats to the monitoring index templates."
+area: Monitoring
+type: enhancement
+issues: []

docs/changelog/91701.yaml

@@ -0,0 +1,5 @@
+pr: 91701
+summary: "[Fleet] Added logs-elastic_agent* read privileges to `kibana_system`"
+area: Authorization
+type: enhancement
+issues: []

docs/changelog/91704.yaml

@@ -0,0 +1,7 @@
+pr: 91704
+summary: '`DoPrivileged` in `ElasticsearchEncaughtExceptionHandler` and check modify
+  thread'
+area: Infra/Core
+type: bug
+issues:
+ - 91650

docs/changelog/91772.yaml

@@ -0,0 +1,5 @@
+pr: 91772
+summary: Add `trace.id` to request trace logs
+area: Infra/Core
+type: bug
+issues: [88174]

docs/changelog/91780.yaml

@@ -0,0 +1,5 @@
+pr: 91780
+summary: Check stable plugin version at install and load time
+area: Infra/Plugins
+type: enhancement
+issues: []

docs/changelog/91802.yaml

@@ -0,0 +1,5 @@
+pr: 91802
+summary: Fix potential leak in `RemoteRecoveryHandler`
+area: Recovery
+type: bug
+issues: []

docs/changelog/91851.yaml

@@ -0,0 +1,7 @@
+pr: 91851
+summary: Simplify and optimize deduplication of `RepositoryData` for a non-caching
+  repository instance
+area: Snapshot/Restore
+type: bug
+issues:
+ - 89952

docs/changelog/91869.yaml

@@ -0,0 +1,5 @@
+pr: 91869
+summary: Load stable plugins as synthetic modules
+area: Infra/Plugins
+type: enhancement
+issues: []

docs/changelog/91873.yaml

@@ -0,0 +1,5 @@
+pr: 91873
+summary: Record timestamp on API key invalidation
+area: Security
+type: enhancement
+issues: []

docs/changelog/91895.yaml

@@ -0,0 +1,6 @@
+pr: 91895
+summary: Skip remote clusters when performing up front privileges validation for datafeeds
+area: Machine Learning
+type: bug
+issues:
+ - 87832

docs/changelog/91917.yaml

@@ -0,0 +1,6 @@
+pr: 91917
+summary: ML stats failures should not stop the usage API working
+area: Machine Learning
+type: bug
+issues:
+ - 91893

docs/changelog/91924.yaml

@@ -0,0 +1,5 @@
+pr: 91924
+summary: Guard against input sequences that are too long for Question Answering models
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/91946.yaml

@@ -0,0 +1,5 @@
+pr: 91946
+summary: Support SAN/dnsName for restricted trust
+area: TLS
+type: enhancement
+issues: []

docs/changelog/91951.yaml

@@ -0,0 +1,5 @@
+pr: 91951
+summary: Set default `cluster.routing.allocation.balance.disk_usage`
+area: Allocation
+type: enhancement
+issues: []

docs/changelog/91981.yaml

@@ -0,0 +1,5 @@
+pr: 91981
+summary: Handle any exception thrown while generating source for an `IngestDocument`
+area: Ingest Node
+type: bug
+issues: []

docs/java-rest/low-level/usage.asciidoc

@@ -152,7 +152,7 @@ A `RestClient` instance can be built through the corresponding
 `RestClientBuilder` class, created via `RestClient#builder(HttpHost...)`
 static method. The only required argument is one or more hosts that the
 client will communicate with, provided as instances of
-https://hc.apache.org/httpcomponents-core-ga/httpcore/apidocs/org/apache/http/HttpHost.html[HttpHost]
+https://hc.apache.org/httpcomponents-core-5.2.x/current/httpcore5/apidocs/org/apache/hc/core5/http/HttpHost.html[HttpHost]
  as follows:
 
 ["source","java",subs="attributes,callouts,macros"]

docs/plugins/plugin-script.asciidoc

@@ -188,6 +188,7 @@ purge the configuration files while removing a plugin, use `-p` or `--purge`.
 This can option can be used after a plugin is removed to remove any lingering
 configuration files.
 
+[discrete]
 [[removing-multiple-plugins]]
 === Removing multiple plugins
 

docs/reference/cluster/prevalidate-node-removal.asciidoc

@@ -21,7 +21,9 @@ Prevalidate node removal.
 [[prevalidate-node-removal-api-desc]]
 ==== {api-description-title}
 
-This API checks whether attempting to remove the specified node(s) from the cluster is likely to succeed or not. For a cluster with no unassigned shards, removal of any node is considered safe which means the removal of the nodes is likely to succeed. In case the cluster has a <<cluster-health,`red` cluster health status>>, it verifies that the removal of the node(s) would not risk removing the last remaining copy of an unassigned shard.
+This API checks whether attempting to remove the specified node(s) from the cluster is likely to succeed or not. For a cluster with no unassigned shards, removal of any node is considered safe which means the removal of the nodes is likely to succeed.
+
+In case the cluster has a <<cluster-health,`red` cluster health status>>, it verifies that the removal of the node(s) would not risk removing the last remaining copy of an unassigned shard. If there are red indices in the cluster, the API checks whether the red indices are <<searchable-snapshots, Searchable Snapshot>> indices, and if not, it sends a request to each of nodes specified in the API call to verify whether the nodes might contain local shard copies of the red indices that are not Searchable Snapshot indices. This request is processed on each receiving node, by checking whether the node has a shard directory for any of the red index shards.
 
 The response includes the overall safety of the removal of the specified nodes, and a detailed response for each node. The node-specific part of the response also includes more details on why removal of that node might not succeed.
 
@@ -32,7 +34,7 @@ Note that if the prevalidation result for a set of nodes returns `true` (i.e. it
 [[prevalidate-node-removal-api-query-params]]
 ==== {api-query-parms-title}
 
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms]
 
 `names`::
 (Optional, string) Comma-separated list of node names.
@@ -54,11 +56,50 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
 
 `nodes`::
     (object) Prevalidation result for the removal of each of the provided nodes.
++
+.Properties of `nodes`
+[%collapsible%open]
+====
+`<node>`::
+    (object) Contains information about the removal prevalidation of a specific node.
++
+.Properties of `<node>`
+[%collapsible%open]
+=======
+`id`::
+    (string) node ID
+`name`::
+    (string) node name
+`external_id`::
+    (string) node external ID
+`result`::
+    (object) Contains removal prevalidation result of the node.
++
+.Properties of `result`
+[%collapsible%open]
+========
+`is_safe`::
+    (boolean) Whether the removal of the node is considered safe or not.
+`reason`::
+    (string) A string that specifies the reason why the prevalidation result is considered safe or not. It can be one of the following values:
++
+--
+    * `no_problems`: The prevalidation did not find any issues that could prevent the node from being safely removed.
+    * `no_red_shards_except_searchable_snapshots`: The node can be safely removed as all red indices are searchable snapshot indices and therefore removing a node does not risk removing the last copy of that index from the cluster.
+    * `no_red_shards_on_node`: The node does not contain any copies of the red non-searchable-snapshot index shards.
+    * `red_shards_on_node`: The node might contain shard copies of some non-searchable-snapshot red indices. The list of the shards that might be on the node are specified in the `message` field.
+    * `unable_to_verify_red_shards`: Contacting the node failed or timed out. More details is provided in the `message` field.
+--
+`message`::
+    (Optional, string) Detailed information about the removal prevalidation result.
+========
+=======
+====
 
 [[prevalidate-node-removal-api-example]]
 ==== {api-examples-title}
 
-This example validates whether it is safe to remove the nodes `node1` and `node2`. The response indicates that it is safe to remove `node1`, but it might not be safe to remove `node2`. Therefore, the overall prevalidation of the removal of the two nodes returns `false`.
+This example validates whether it is safe to remove the nodes `node1` and `node2`. The response indicates that it is safe to remove `node1`, but it might not be safe to remove `node2` as it might contain copies of the specified red shards. Therefore, the overall prevalidation of the removal of the two nodes returns `false`.
 
 [source,console]
 --------------------------------------------------
@@ -72,14 +113,15 @@ The API returns the following response:
 --------------------------------------------------
 {
   "is_safe": false,
-  "message": "cluster health is RED",
+  "message": "removal of the following nodes might not be safe: [node2-id]",
   "nodes": [
     {
       "id": "node1-id",
       "name" : "node1",
       "external_id" : "node1-externalId",
       "result" : {
         "is_safe": true,
+        "reason": "no_red_shards_on_node",
         "message": ""
       }
     },
@@ -89,7 +131,8 @@ The API returns the following response:
       "external_id" : "node2-externalId",
       "result" : {
         "is_safe": false,
-        "message": "node may contain a copy of a red index shard"
+        "reason": "red_shards_on_node",
+        "message": "node contains copies of the following red shards: [[indexName][0]]"
       }
     }
   ]

docs/reference/commands/reconfigure-node.asciidoc

@@ -28,7 +28,7 @@ cluster where security features are already enabled and configured. Before
 starting your new node, run the 
 <<create-enrollment-token,`elasticsearch-create-enrollment-token`>> tool
 with the `-s node` option to generate an enrollment token on any node in your
-existing cluster. On your new node, run the the
+existing cluster. On your new node, run the
 `elasticsearch-reconfigure-node` tool and pass the enrollment token as a
 parameter.
 

docs/reference/features/apis/reset-features-api.asciidoc

@@ -6,7 +6,7 @@
 
 experimental::[]
 
-Clears all of the the state information stored in system indices by {es} features, including the security and machine learning indices.
+Clears all of the state information stored in system indices by {es} features, including the security and machine learning indices.
 
 WARNING: Intended for development and testing use only. Do not reset features on a production cluster.
 

docs/reference/how-to/search-speed.asciidoc

@@ -10,7 +10,7 @@ goes to the filesystem cache so that Elasticsearch can keep hot regions of the
 index in physical memory.
 
 [discrete]
-tag::readahead[]
+// tag::readahead[]
 === Avoid page cache thrashing by using modest readahead values on Linux
 
 Search can cause a lot of randomized read I/O. When the underlying block
@@ -35,7 +35,7 @@ as a transient setting). We recommend a value of `128KiB` for readahead.
 WARNING: `blockdev` expects values in 512 byte sectors whereas `lsblk` reports
 values in `KiB`. As an example, to temporarily set readahead to `128KiB`
 for `/dev/nvme0n1`, specify `blockdev --setra 256 /dev/nvme0n1`.
-end::readahead[]
+// end::readahead[]
 
 [discrete]
 === Use faster hardware
@@ -358,7 +358,7 @@ PUT index
 }
 --------------------------------------------------
 
-tag::warm-fs-cache[]
+// tag::warm-fs-cache[]
 [discrete]
 === Warm up the filesystem cache
 
@@ -372,7 +372,7 @@ depending on the file extension using the
 WARNING: Loading data into the filesystem cache eagerly on too many indices or
 too many files will make search _slower_ if the filesystem cache is not large
 enough to hold all the data. Use with caution.
-end::warm-fs-cache[]
+// end::warm-fs-cache[]
 
 [discrete]
 === Use index sorting to speed up conjunctions
@@ -424,6 +424,7 @@ be able to cope with `max_failures` node failures at once at most, then the
 right number of replicas for you is
 `max(max_failures, ceil(num_nodes / num_primaries) - 1)`.
 
+[discrete]
 === Tune your queries with the Search Profiler
 
 The {ref}/search-profile.html[Profile API] provides detailed information about
@@ -438,6 +439,7 @@ Because the Profile API itself adds significant overhead to the query,
 this information is best used to understand the relative cost of the various
 query components. It does not provide a reliable measure of actual processing time.
 
+[discrete]
 [[faster-phrase-queries]]
 === Faster phrase queries with `index_phrases`
 
@@ -446,6 +448,7 @@ indexes 2-shingles and is automatically leveraged by query parsers to run phrase
 queries that don't have a slop. If your use-case involves running lots of phrase
 queries, this can speed up queries significantly.
 
+[discrete]
 [[faster-prefix-queries]]
 === Faster prefix queries with `index_prefixes`
 
@@ -454,6 +457,7 @@ indexes prefixes of all terms and is automatically leveraged by query parsers to
 run prefix queries. If your use-case involves running lots of prefix queries,
 this can speed up queries significantly.
 
+[discrete]
 [[faster-filtering-with-constant-keyword]]
 === Use `constant_keyword` to speed up filtering