Merge branch 'main' into heap_attack_2

.ci/dockerOnLinuxExclusions

@@ -6,13 +6,15 @@
 # excluded.
 debian-8
 opensuse-leap-15.1
+opensuse-leap-15.5
 ol-7.7
 sles-12.3 # older version used in Vagrant image
 sles-12.5
 sles-15.1
 sles-15.2
 sles-15.3
 sles-15.4
+sles-15.5
 
 # These OSes are deprecated and filtered starting with 8.0.0, but need to be excluded
 # for PR checks

build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt

@@ -62,9 +62,6 @@ org.apache.lucene.index.NoMergePolicy#INSTANCE @ explicit use of NoMergePolicy r
 org.apache.lucene.search.TimeLimitingCollector#getGlobalTimerThread()
 org.apache.lucene.search.TimeLimitingCollector#getGlobalCounter()
 
-@defaultMessage use @org.elasticsearch.common.lucene.RegExp instead to avoid StackOverflowError
-org.apache.lucene.util.automaton.RegExp
-
 @defaultMessage Don't interrupt threads use FutureUtils#cancel(Future<T>) instead
 java.util.concurrent.Future#cancel(boolean)
 

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.11.0
-lucene            = 9.7.0
+lucene            = 9.8.0
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 21+35@fd2272bbf8e04c3dbaee13770090416c

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.7.0
-:lucene_version_path:   9_7_0
+:lucene_version:        9.8.0
+:lucene_version_path:   9_8_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/100106.yaml

@@ -0,0 +1,5 @@
+pr: 100106
+summary: Validate enrich index before completing policy execution
+area: Ingest Node
+type: bug
+issues: []

docs/changelog/100138.yaml

@@ -0,0 +1,5 @@
+pr: 100138
+summary: Upgrade main to Lucene 9.8.0
+area: Search
+type: upgrade
+issues: []

docs/changelog/100143.yaml

@@ -0,0 +1,5 @@
+pr: 100143
+summary: Preserve order of inference results when calling the _infer API with multiple inputs on a model deployment with more than one allocation the output results order was not guaranteed to match the input order. The fix ensures the output order matches the input order.
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/100154.yaml

@@ -0,0 +1,5 @@
+pr: 100154
+summary: Log warnings for jobs unassigned for a long time
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/97729.yaml

@@ -0,0 +1,5 @@
+pr: 97729
+summary: Allow parsing on non-string routing fields
+area: Aggregations
+type: bug
+issues: []

docs/changelog/99527.yaml

@@ -0,0 +1,5 @@
+pr: 99445
+summary: Add new max_inner_product vector similarity function 
+area: Vector Search
+type: enhancement
+issues: []

docs/changelog/99532.yaml

@@ -0,0 +1,5 @@
+pr: 99532
+summary: Adds `nested` support for indexed `dense_vector` fields
+area: Vector Search
+type: enhancement
+issues: []

docs/community-clients/index.asciidoc

@@ -25,6 +25,7 @@ a number of clients that have been contributed by the community for various lang
 * <<haskell>>
 * <<java>>
 * <<javascript>>
+* <<julia>>
 * <<kotlin>>
 * <<lua>>
 * <<dotnet>>
@@ -121,6 +122,12 @@ client].
 See the {client}/javascript-api/current/index.html[official Elasticsearch
 JavaScript client].
 
+[[julia]]
+== Julia
+
+* https://github.com/OpenSesame/ElasticsearchClient.jl[ElasticsearchClient.jl]:
+Elasticsearch client inspired by the {client}/ruby-api/current/index.html[official Elasticsearch Ruby client].
+
 [[kotlin]]
 == Kotlin
 

docs/reference/data-streams/downsampling-manual.asciidoc

@@ -360,19 +360,16 @@ This returns:
           "index_name": ".ds-my-data-stream-2023.07.26-000001", <1>
           "index_uuid": "ltOJGmqgTVm4T-Buoe7Acg",
           "prefer_ilm": true,
-          "managed_by": "Data stream lifecycle"
+          "managed_by": "Unmanaged"
         }
       ],
       "generation": 1,
       "status": "GREEN",
-      "next_generation_managed_by": "Data stream lifecycle",
+      "next_generation_managed_by": "Unmanaged",
       "prefer_ilm": true,
       "template": "my-data-stream-template",
       "hidden": false,
       "system": false,
-      "lifecycle": {
-        "enabled": true
-      },
       "allow_custom_routing": false,
       "replicated": false,
       "time_series": {

docs/reference/mapping/types/dense-vector.asciidoc

@@ -111,9 +111,6 @@ PUT my-index-2
 efficient kNN search. Like most kNN algorithms, HNSW is an approximate method
 that sacrifices result accuracy for improved speed.
 
-Dense vector fields cannot be indexed if they are within
-<<nested, `nested`>> mappings.
-
 [role="child_attributes"]
 [[dense-vector-params]]
 ==== Parameters for dense vector fields
@@ -159,7 +156,7 @@ distance) between the vectors. The document `_score` is computed as
 `1 / (1 + l2_norm(query, vector)^2)`.
 
 `dot_product`:::
-Computes the dot product of two vectors. This option provides an optimized way
+Computes the dot product of two unit vectors. This option provides an optimized way
 to perform cosine similarity. The constraints and computed score are defined
 by `element_type`.
 +
@@ -181,6 +178,14 @@ original vectors and cannot normalize them in advance. The document `_score`
 is computed as `(1 + cosine(query, vector)) / 2`. The `cosine` similarity does
 not allow vectors with zero magnitude, since cosine is not defined in this
 case.
+
+`max_inner_product`:::
+Computes the maximum inner product of two vectors. This is similar to `dot_product`,
+but doesn't require vectors to be normalized. This means that each vector's magnitude
+can significantly effect the score. The document `_score` is adjusted to prevent negative
+values. For `max_inner_product` values `< 0`, the `_score` is
+`1 / (1 + -1 * max_inner_product(query, vector))`. For non-negative `max_inner_product` results
+the `_score` is calculated `max_inner_product(query, vector) + 1`.
 ====
 
 NOTE: Although they are conceptually related, the `similarity` parameter is