Merge branch 'main' into dsl-downsampling-docs

docs/changelog/102435.yaml

@@ -0,0 +1,5 @@
+pr: 102435
+summary: S3 first byte latency metric
+area: Search
+type: enhancement
+issues: []

docs/changelog/103024.yaml

@@ -0,0 +1,6 @@
+pr: 103024
+summary: Fix template simulate setting application ordering
+area: Indices APIs
+type: bug
+issues:
+ - 103008

docs/reference/release-notes.asciidoc

@@ -8,6 +8,7 @@ This section summarizes the changes in each release.
 
 * <<release-notes-8.13.0>>
 * <<release-notes-8.12.0>>
+* <<release-notes-8.11.2>>
 * <<release-notes-8.11.1>>
 * <<release-notes-8.11.0>>
 * <<release-notes-8.10.4>>
@@ -58,6 +59,7 @@ This section summarizes the changes in each release.
 
 include::release-notes/8.13.0.asciidoc[]
 include::release-notes/8.12.0.asciidoc[]
+include::release-notes/8.11.2.asciidoc[]
 include::release-notes/8.11.1.asciidoc[]
 include::release-notes/8.11.0.asciidoc[]
 include::release-notes/8.10.4.asciidoc[]

docs/reference/release-notes/8.11.2.asciidoc

@@ -0,0 +1,83 @@
+[[release-notes-8.11.2]]
+== {es} version 8.11.2
+
+Also see <<breaking-changes-8.11,Breaking changes in 8.11>>.
+
+[[known-issues-8.11.2]]
+[float]
+=== Known issues
+include::8.10.3.asciidoc[tag=no-preventive-gc-issue]
+
+[[bug-8.11.2]]
+[float]
+=== Bug fixes
+
+Allocation::
+* Improve failure handling in `ContinuousComputation` {es-pull}102281[#102281]
+
+Application::
+* Default `run_ml_inference` should be true {es-pull}102151[#102151]
+* [Query Rules] Fix bug where combining the same metadata with text/numeric values leads to error {es-pull}102891[#102891] (issue: {es-issue}102827[#102827])
+
+Cluster Coordination::
+* Synchronize Coordinator#onClusterStateApplied {es-pull}100986[#100986] (issue: {es-issue}99023[#99023])
+
+Data streams::
+* [Usage API] Count all the data streams that have lifecycle {es-pull}102259[#102259]
+
+ES|QL::
+* ES|QL: Fix drop of renamed grouping {es-pull}102282[#102282] (issue: {es-issue}102121[#102121])
+* ES|QL: Fix layout management for Project {es-pull}102399[#102399] (issue: {es-issue}102120[#102120])
+* Fix DISSECT with empty patterns {es-pull}102580[#102580] (issue: {es-issue}102577[#102577])
+* Fix leaking blocks in TopN {es-pull}102715[#102715] (issue: {es-issue}102646[#102646])
+* Fix leaking blocks in `BlockUtils` {es-pull}102716[#102716]
+* Fix memory tracking in TopN.Row {es-pull}102831[#102831] (issues: {es-issue}100640[#100640], {es-issue}102784[#102784], {es-issue}102790[#102790], {es-issue}102683[#102683])
+
+ILM+SLM::
+* [ILM] Fix downsample to skip already downsampled indices {es-pull}102250[#102250] (issue: {es-issue}102249[#102249])
+
+Infra/Circuit Breakers::
+* Add more logging to the real memory circuit breaker and lower minimum interval {es-pull}102396[#102396]
+
+Ingest Node::
+* Better processor stat merge {es-pull}102821[#102821]
+
+Machine Learning::
+* Ensure datafeed previews with no start or end time don't search the cold or frozen tiers {es-pull}102492[#102492]
+* Recreate the Elasticsearch private temporary directory if it doesn't exist when an ML job is opened {es-pull}102599[#102599]
+
+Mapping::
+* Fix dense_vector cluster stats indexed_vector_dim_min/max values {es-pull}102467[#102467] (issue: {es-issue}102416[#102416])
+
+Search::
+* Allow mismatched sort-by field types if there are no docs to sort {es-pull}102779[#102779]
+
+Security::
+* Fix double-completion in `SecurityUsageTransportAction` {es-pull}102114[#102114] (issue: {es-issue}102111[#102111])
+
+Snapshot/Restore::
+* Set region for the STS client via privileged calls in AWS SDK {es-pull}102230[#102230] (issue: {es-issue}102173[#102173])
+* Simplify `BlobStoreRepository` idle check {es-pull}102057[#102057] (issue: {es-issue}101948[#101948])
+
+Transform::
+* Ensure transform updates only modify the expected transform task {es-pull}102934[#102934] (issue: {es-issue}102933[#102933])
+* Exclude stack traces from transform audit messages and health {es-pull}102240[#102240]
+
+[[enhancement-8.11.2]]
+[float]
+=== Enhancements
+
+Machine Learning::
+* Add inference counts by model to the machine learning usage stats {es-pull}101915[#101915]
+
+Security::
+* Upgrade xmlsec to 2.3.4 {es-pull}102220[#102220]
+
+[[upgrade-8.11.2]]
+[float]
+=== Upgrades
+
+Snapshot/Restore::
+* Upgrade reactor netty http version {es-pull}102311[#102311]
+
+

docs/reference/security/authentication/saml-guide.asciidoc

@@ -328,24 +328,15 @@ groups:: _(Recommended)_
 +
 [NOTE]
 ====
-Some IdPs are configured to send the `groups` list as a comma-separated string,
-but {es} can't parse this string into an array of groups. To map this SAML
-attribute to the `attributes.groups` setting in the {es} realm, a cluster
-security administrator can use a wildcard when
-<<saml-role-mapping,configuring role mappings>>. While flexible, wildcards are
-less accurate and can match on unwanted patterns. Instead, a cluster security
-administrator can use a regular expression to create a role mapping rule that
-matches only a single group. For example, the following regular expression
-matches only on the `elasticsearch-admins` group:
-
-[source,sh]
-----
-/^(.*,)?elasticsearch-admins(,.*)?$/
-----
-
-These regular expressions are based on Lucene’s
-{ref}/regexp-syntax.html[regular expression syntax], and can match more complex
-patterns. All regular expressions must start and end with a forward slash.
+Some IdPs are configured to send the `groups` list as a single value, comma-separated
+string. To map this SAML attribute to the `attributes.groups` setting in the {es}
+realm, you can configure a string delimiter using the `attribute_delimiters.group`
+setting.
+
+For example, splitting the SAML attribute value
+`engineering,elasticsearch-admins,employees` on a delimiter value of `,` will
+result in `engineering`, `elasticsearch-admins`, and `employees` as the list of
+groups for the user.
 ====
 
 name:: _(Optional)_ The user's full name.

docs/reference/settings/security-settings.asciidoc

@@ -1222,7 +1222,7 @@ _Distinguished Name_.
 `attribute_patterns.principal` {ess-icon}::
 (<<static-cluster-setting,Static>>)
 A Java regular expression that is matched against the SAML attribute specified
-by `attributes.pattern` before it is applied to the user's _principal_ property.
+by `attributes.principal` before it is applied to the user's _principal_ property.
 The attribute value must match the pattern and the value of the first
 _capturing group_ is used as the principal. For example, `^([^@]+)@example\\.com$`
 matches email addresses from the "example.com" domain and uses the local-part as
@@ -1257,13 +1257,13 @@ As per `attribute_patterns.principal`, but for the _dn_ property.
 `attribute_delimiters.groups` {ess-icon}::
 (<<static-cluster-setting,Static>>)
 A plain string that is used as a delimiter to split a single-valued SAML
-attribute specified by attributes.groups before it is applied to the user's
-groups property. For example, splitting the SAML attribute value
-engineering,elasticsearch-admins,employees on a delimiter value of , will
-result in engineering, elasticsearch-admins, and employees as the list of
+attribute specified by `attributes.groups` before it is applied to the user's
+_groups_ property. For example, splitting the SAML attribute value
+`engineering,elasticsearch-admins,employees` on a delimiter value of `,` will
+result in `engineering`, `elasticsearch-admins`, and `employees` as the list of
 groups for the user. The delimiter will always be split on, regardless of
 escaping in the input string. This setting does not support multi-valued SAML
-attributes. It cannot be used together with the attribute_patterns setting.
+attributes. It cannot be used together with the `attribute_patterns` setting.
 You can only configure this setting for the groups attribute.
 // end::saml-attributes-delimiters-groups-tag[]
 

modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadAnalyzerTests.java

@@ -11,9 +11,9 @@
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction.AnalyzeToken;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction.Response;
-import org.elasticsearch.action.admin.indices.analyze.ReloadAnalyzerAction;
 import org.elasticsearch.action.admin.indices.analyze.ReloadAnalyzersRequest;
 import org.elasticsearch.action.admin.indices.analyze.ReloadAnalyzersResponse;
+import org.elasticsearch.action.admin.indices.analyze.TransportReloadAnalyzersAction;
 import org.elasticsearch.index.mapper.MapperException;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
@@ -121,7 +121,7 @@ private void updateSynonyms(Path synonymsFile, boolean preview) throws IOExcepti
             out.println("foo, baz, buzz");
         }
         ReloadAnalyzersResponse reloadResponse = client().execute(
-            ReloadAnalyzerAction.INSTANCE,
+            TransportReloadAnalyzersAction.TYPE,
             new ReloadAnalyzersRequest(null, preview, INDEX_NAME)
         ).actionGet();
         assertNoFailures(reloadResponse);
@@ -183,7 +183,7 @@ public void testSynonymsInMultiplexerUpdateable() throws FileNotFoundException,
             out.println("foo, baz, buzz");
         }
         ReloadAnalyzersResponse reloadResponse = client().execute(
-            ReloadAnalyzerAction.INSTANCE,
+            TransportReloadAnalyzersAction.TYPE,
             new ReloadAnalyzersRequest(null, false, INDEX_NAME)
         ).actionGet();
         assertNoFailures(reloadResponse);
@@ -303,7 +303,7 @@ public void testKeywordMarkerUpdateable() throws IOException {
         }
 
         ReloadAnalyzersResponse reloadResponse = client().execute(
-            ReloadAnalyzerAction.INSTANCE,
+            TransportReloadAnalyzersAction.TYPE,
             new ReloadAnalyzersRequest(null, false, INDEX_NAME)
         ).actionGet();
         assertNoFailures(reloadResponse);

modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java

@@ -10,9 +10,9 @@
 
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction.AnalyzeToken;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction.Response;
-import org.elasticsearch.action.admin.indices.analyze.ReloadAnalyzerAction;
 import org.elasticsearch.action.admin.indices.analyze.ReloadAnalyzersRequest;
 import org.elasticsearch.action.admin.indices.analyze.ReloadAnalyzersResponse;
+import org.elasticsearch.action.admin.indices.analyze.TransportReloadAnalyzersAction;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
@@ -91,7 +91,7 @@ private void testSynonymsUpdate(boolean preview) throws FileNotFoundException, I
                 out.println("foo, baz, " + testTerm);
             }
             ReloadAnalyzersResponse reloadResponse = client().execute(
-                ReloadAnalyzerAction.INSTANCE,
+                TransportReloadAnalyzersAction.TYPE,
                 new ReloadAnalyzersRequest(null, preview, "test")
             ).actionGet();
             assertNoFailures(reloadResponse);

modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java

@@ -21,9 +21,9 @@
 import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
-import org.elasticsearch.action.index.IndexAction;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.index.TransportIndexAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlocks;
@@ -221,7 +221,7 @@ public void testIndexChunks() throws IOException {
 
         AtomicInteger chunkIndex = new AtomicInteger();
 
-        client.addHandler(IndexAction.INSTANCE, (IndexRequest request, ActionListener<DocWriteResponse> listener) -> {
+        client.addHandler(TransportIndexAction.TYPE, (IndexRequest request, ActionListener<DocWriteResponse> listener) -> {
             int chunk = chunkIndex.getAndIncrement();
             assertEquals(OpType.CREATE, request.opType());
             assertThat(request.id(), Matchers.startsWith("test_" + (chunk + 15) + "_"));

modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java

@@ -21,8 +21,8 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.recovery.RecoverySettings;
+import org.elasticsearch.repositories.RepositoriesMetrics;
 import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository;
-import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
 import java.util.Locale;
@@ -109,7 +109,7 @@ public AzureRepository(
             recoverySettings,
             buildBasePath(metadata),
             buildLocation(metadata),
-            MeterRegistry.NOOP
+            RepositoriesMetrics.NOOP
         );
         this.chunkSize = Repository.CHUNK_SIZE_SETTING.get(metadata.settings());
         this.storageService = storageService;

modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
+import org.elasticsearch.repositories.RepositoriesMetrics;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ScalingExecutorBuilder;
@@ -62,7 +63,8 @@ public Map<String, Repository.Factory> getRepositories(
         NamedXContentRegistry namedXContentRegistry,
         ClusterService clusterService,
         BigArrays bigArrays,
-        RecoverySettings recoverySettings
+        RecoverySettings recoverySettings,
+        RepositoriesMetrics repositoriesMetrics
     ) {
         return Collections.singletonMap(AzureRepository.TYPE, metadata -> {
             AzureStorageService storageService = azureStoreService.get();

modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java

@@ -42,6 +42,7 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.RepositoriesMetrics;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
@@ -256,7 +257,8 @@ public Map<String, Repository.Factory> getRepositories(
             NamedXContentRegistry registry,
             ClusterService clusterService,
             BigArrays bigArrays,
-            RecoverySettings recoverySettings
+            RecoverySettings recoverySettings,
+            RepositoriesMetrics repositoriesMetrics
         ) {
             return Collections.singletonMap(
                 GoogleCloudStorageRepository.TYPE,

modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
+import org.elasticsearch.repositories.RepositoriesMetrics;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
@@ -48,7 +49,8 @@ public Map<String, Repository.Factory> getRepositories(
         NamedXContentRegistry namedXContentRegistry,
         ClusterService clusterService,
         BigArrays bigArrays,
-        RecoverySettings recoverySettings
+        RecoverySettings recoverySettings,
+        RepositoriesMetrics repositoriesMetrics
     ) {
         return Collections.singletonMap(
             GoogleCloudStorageRepository.TYPE,

modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java

@@ -19,9 +19,9 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.recovery.RecoverySettings;
+import org.elasticsearch.repositories.RepositoriesMetrics;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository;
-import org.elasticsearch.telemetry.metric.MeterRegistry;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
 import java.util.Map;
@@ -78,7 +78,7 @@ class GoogleCloudStorageRepository extends MeteredBlobStoreRepository {
             recoverySettings,
             buildBasePath(metadata),
             buildLocation(metadata),
-            MeterRegistry.NOOP
+            RepositoriesMetrics.NOOP
         );
         this.storageService = storageService;