Merge branch 'main' into ingest-geoip-boundary-refactor-wip

docs/changelog/113237.yaml

@@ -0,0 +1,5 @@
+pr: 113237
+summary: Retry throttled snapshot deletions
+area: Snapshot/Restore
+type: bug
+issues: []

docs/changelog/114750.yaml

@@ -0,0 +1,5 @@
+pr: 114750
+summary: Create an ml node inference endpoint referencing an existing model
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/114836.yaml

@@ -0,0 +1,6 @@
+pr: 114836
+summary: Support multi-valued fields in compute engine for ST_DISTANCE
+area: ES|QL
+type: enhancement
+issues:
+ - 112910

docs/changelog/114848.yaml

@@ -0,0 +1,5 @@
+pr: 114848
+summary: "ESQL: Fix grammar changes around per agg filtering"
+area: ES|QL
+type: bug
+issues: []

docs/reference/data-streams/promote-data-stream-api.asciidoc

@@ -18,6 +18,10 @@ available, the data stream in the local cluster can be promoted
 to a regular data stream, which allows these data streams to
 be rolled over in the local cluster.
 
+NOTE: When promoting a data stream, ensure the local cluster has a data stream enabled index template that matches the data stream.
+If this is missing, the data stream will not be able to roll over until a matching index template is created.
+This will affect the lifecycle management of the data stream and interfere with the data stream size and retention.
+
 [source,console]
 ----
 POST /_data_stream/_promote/my-data-stream

docs/reference/snapshot-restore/index.asciidoc

@@ -48,6 +48,9 @@ Snapshots don't contain or back up:
 * Node configuration files
 * <<security-files,Security configuration files>>
 
+NOTE: When restoring a data stream, if the target cluster does not have an index template that matches the data stream, the data stream will not be able to roll over until a matching index template is created.
+This will affect the lifecycle management of the data stream and interfere with the data stream size and retention.
+
 [discrete]
 [[feature-state]]
 === Feature states

docs/reference/snapshot-restore/repository-s3.asciidoc

@@ -329,6 +329,20 @@ include::repository-shared-settings.asciidoc[]
     `1000` which is the maximum number supported by the https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html[AWS
     ListMultipartUploads API]. If set to `0`, {es} will not attempt to clean up dangling multipart uploads.
 
+`throttled_delete_retry.delay_increment`::
+
+    (<<time-units,time value>>) This value is used as the delay before the first retry and the amount the delay is incremented by on each subsequent retry. Default is 50ms, minimum is 0ms.
+
+`throttled_delete_retry.maximum_delay`::
+
+    (<<time-units,time value>>) This is the upper bound on how long the delays between retries will grow to. Default is 5s, minimum is 0ms.
+
+`throttled_delete_retry.maximum_number_of_retries`::
+
+    (integer) Sets the number times to retry a throttled snapshot deletion. Defaults to `10`, minimum value is `0` which
+    will disable retries altogether. Note that if retries are enabled in the Azure client, each of these retries
+    comprises that many client-level retries.
+
 NOTE: The option of defining client settings in the repository settings as
 documented below is considered deprecated, and will be removed in a future
 version.

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequestBuilder;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
@@ -45,6 +46,7 @@
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
 import org.elasticsearch.snapshots.RestoreInfo;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInProgressException;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotRestoreException;
@@ -62,7 +64,9 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoWarningHeaderOnResponse;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertWarningHeaderOnResponse;
 import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -80,6 +84,8 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
     private static final Map<String, Integer> DOCUMENT_SOURCE = Collections.singletonMap("@timestamp", 123);
     public static final String REPO = "repo";
     public static final String SNAPSHOT = "snap";
+    public static final String TEMPLATE_1_ID = "t1";
+    public static final String TEMPLATE_2_ID = "t2";
     private Client client;
 
     private String dsBackingIndexName;
@@ -103,8 +109,8 @@ public void setup() throws Exception {
         Path location = randomRepoPath();
         createRepository(REPO, "fs", location);
 
-        DataStreamIT.putComposableIndexTemplate("t1", List.of("ds", "other-ds"));
-        DataStreamIT.putComposableIndexTemplate("t2", """
+        DataStreamIT.putComposableIndexTemplate(TEMPLATE_1_ID, List.of("ds", "other-ds"));
+        DataStreamIT.putComposableIndexTemplate(TEMPLATE_2_ID, """
             {
                 "properties": {
                   "@timestamp": {
@@ -1335,4 +1341,149 @@ public void testRestoreDataStreamAliasWithConflictingIndicesAlias() throws Excep
         );
         assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (my-alias)"));
     }
+
+    public void testWarningHeaderOnRestoreWithoutTemplates() throws Exception {
+        String datastreamName = "ds";
+
+        CreateSnapshotResponse createSnapshotResponse = client.admin()
+            .cluster()
+            .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
+            .setWaitForCompletion(true)
+            .setIndices(datastreamName)
+            .setIncludeGlobalState(false)
+            .get();
+
+        RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
+        SnapshotId snapshotId = createSnapshotResponse.getSnapshotInfo().snapshotId();
+        assertEquals(RestStatus.OK, status);
+
+        assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
+
+        assertAcked(
+            client.execute(
+                DeleteDataStreamAction.INSTANCE,
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, datastreamName, "other-ds")
+            )
+        );
+
+        assertAcked(
+            client.execute(
+                TransportDeleteComposableIndexTemplateAction.TYPE,
+                new TransportDeleteComposableIndexTemplateAction.Request(TEMPLATE_1_ID)
+            ).get()
+        );
+
+        RestoreSnapshotRequestBuilder request = client.admin()
+            .cluster()
+            .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
+            .setWaitForCompletion(true)
+            .setIndices(datastreamName);
+
+        assertWarningHeaderOnResponse(
+            client,
+            request,
+            "Snapshot ["
+                + snapshotId
+                + "] contains data stream ["
+                + datastreamName
+                + "] but custer does not have a matching index "
+                + "template. This will cause rollover to fail until a matching index template is created"
+        );
+
+    }
+
+    public void testWarningHeaderAbsentOnRestoreWithTemplates() throws Exception {
+        String datastreamName = "ds";
+
+        CreateSnapshotResponse createSnapshotResponse = client.admin()
+            .cluster()
+            .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
+            .setWaitForCompletion(true)
+            .setIndices(datastreamName)
+            .setIncludeGlobalState(false)
+            .get();
+
+        RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
+        SnapshotId snapshotId = createSnapshotResponse.getSnapshotInfo().snapshotId();
+        assertEquals(RestStatus.OK, status);
+
+        assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
+
+        assertAcked(
+            client.execute(
+                DeleteDataStreamAction.INSTANCE,
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, datastreamName, "other-ds", "with-fs")
+            )
+        );
+
+        RestoreSnapshotRequestBuilder request = client.admin()
+            .cluster()
+            .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
+            .setWaitForCompletion(true)
+            .setIndices(datastreamName);
+
+        assertNoWarningHeaderOnResponse(
+            client,
+            request,
+            "but custer does not have a matching index template. This will cause rollover to fail until a matching index "
+                + "template is created"
+        );
+
+    }
+
+    /**
+     * This test is muted as it's awaiting the same fix as {@link #testPartialRestoreSnapshotThatIncludesDataStreamWithGlobalState()}
+     */
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107515")
+    public void testWarningHeaderOnRestoreTemplateFromSnapshot() throws Exception {
+        String datastreamName = "ds";
+
+        CreateSnapshotResponse createSnapshotResponse = client.admin()
+            .cluster()
+            .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
+            .setWaitForCompletion(true)
+            .setIndices(datastreamName)
+            .setIncludeGlobalState(true)
+            .get();
+
+        RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
+        SnapshotId snapshotId = createSnapshotResponse.getSnapshotInfo().snapshotId();
+        assertEquals(RestStatus.OK, status);
+
+        assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
+
+        assertAcked(
+            client.execute(
+                DeleteDataStreamAction.INSTANCE,
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, datastreamName, "other-ds")
+            )
+        );
+
+        assertAcked(
+            client.execute(
+                TransportDeleteComposableIndexTemplateAction.TYPE,
+                new TransportDeleteComposableIndexTemplateAction.Request(TEMPLATE_1_ID)
+            ).get()
+        );
+
+        RestoreSnapshotRequestBuilder request = client.admin()
+            .cluster()
+            .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
+            .setWaitForCompletion(true)
+            .setRestoreGlobalState(true)
+            .setIndices(datastreamName);
+
+        assertNoWarningHeaderOnResponse(
+            client,
+            request,
+            "Snapshot ["
+                + snapshotId
+                + "] contains data stream ["
+                + datastreamName
+                + "] but custer does not have a matching index "
+                + "template. This will cause rollover to fail until a matching index template is created"
+        );
+
+    }
+
 }