Merge branch 'main' into fix/unblock-after-update

.buildkite/scripts/get-latest-test-mutes.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-if [[ ! "${BUILDKITE_PULL_REQUEST:-}" || "${BUILDKITE_AGENT_META_DATA_PROVIDER:-}" == "k8s" ]]; then
+if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false"  || "${BUILDKITE_AGENT_META_DATA_PROVIDER:-}" == "k8s" ]]; then
   exit 0
 fi
 

benchmarks/build.gradle

@@ -1,4 +1,5 @@
 import org.elasticsearch.gradle.internal.test.TestUtil
+import org.elasticsearch.gradle.OS
 
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
@@ -77,7 +78,7 @@ tasks.register("copyPainless", Copy) {
 }
 
 tasks.named("run").configure {
-  executable = "${buildParams.runtimeJavaHome.get()}/bin/java"
+  executable = "${buildParams.runtimeJavaHome.get()}/bin/java" + (OS.current() == OS.WINDOWS ? '.exe' : '')
   args << "-Dplugins.dir=${buildDir}/plugins" << "-Dtests.index=${buildDir}/index"
   dependsOn "copyExpression", "copyPainless", configurations.nativeLib
   systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../libs/native/libraries/build/platform/").toString())

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -22,7 +22,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:eef54b3a414aa53b98f0f8df2633aed83c3ba6230722769282925442968f0364",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:dd66beec64a7f9b19c6c35a1195153b2b630a55e16ec71949ed5187c5947eea1",
         "-wolfi",
         "apk"
     ),

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java

@@ -122,7 +122,7 @@ public void apply(Project project) {
         composeExtension.getRemoveContainers().set(true);
         composeExtension.getCaptureContainersOutput()
             .set(EnumSet.of(LogLevel.INFO, LogLevel.DEBUG).contains(project.getGradle().getStartParameter().getLogLevel()));
-        composeExtension.getUseDockerComposeV2().set(false);
+        composeExtension.getUseDockerComposeV2().set(true);
         composeExtension.getExecutable().set(this.providerFactory.provider(() -> {
             String composePath = dockerSupport.get().getDockerAvailability().dockerComposePath();
             LOGGER.debug("Docker Compose path: {}", composePath);

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java

@@ -26,7 +26,7 @@ public interface TestClustersAware extends Task {
     Collection<ElasticsearchCluster> getClusters();
 
     @ServiceReference(REGISTRY_SERVICE_NAME)
-    Property<TestClustersRegistry> getRegistery();
+    Property<TestClustersRegistry> getRegistry();
 
     @ServiceReference(TEST_CLUSTER_TASKS_SERVICE)
     Property<TestClustersPlugin.TaskEventsService> getTasksService();
@@ -47,6 +47,14 @@ default void useCluster(ElasticsearchCluster cluster) {
         getClusters().add(cluster);
     }
 
+    default Provider<TestClusterInfo> getClusterInfo(String clusterName) {
+        return getProject().getProviders().of(TestClusterValueSource.class, source -> {
+            source.getParameters().getService().set(getRegistry());
+            source.getParameters().getClusterName().set(clusterName);
+            source.getParameters().getPath().set(getProject().getIsolated().getPath());
+        });
+    }
+
     default void useCluster(Provider<ElasticsearchCluster> cluster) {
         useCluster(cluster.get());
     }

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java

@@ -249,7 +249,7 @@ private void configureStartClustersHook(Gradle gradle) {
                     .forEach(awareTask -> {
                         awareTask.doFirst(task -> {
                             awareTask.beforeStart();
-                            awareTask.getClusters().forEach(awareTask.getRegistery().get()::maybeStartCluster);
+                            awareTask.getClusters().forEach(awareTask.getRegistry().get()::maybeStartCluster);
                         });
                     });
             });

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersRegistry.java

@@ -109,6 +109,23 @@ public void restart(String path, String clusterName) {
         cluster.restart();
     }
 
+    public void nextNodeToNextVersion(Provider<ElasticsearchCluster> cluster) {
+        nextNodeToNextVersion(cluster.get());
+    }
+
+    public void nextNodeToNextVersion(ElasticsearchCluster cluster) {
+        nextNodeToNextVersion(cluster.getPath(), cluster.getName());
+    }
+
+    public void nextNodeToNextVersion(String path, String clusterName) {
+        ElasticsearchCluster cluster = runningClusters.stream()
+            .filter(c -> c.getPath().equals(path))
+            .filter(c -> c.getName().equals(clusterName))
+            .findFirst()
+            .orElseThrow();
+        cluster.nextNodeToNextVersion();
+    }
+
     public void storeProcess(String id, Process esProcess) {
         nodeProcesses.put(id, esProcess);
     }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java

@@ -187,12 +187,20 @@ static String agentCommandLineOption(Path agentJar, Path tmpPropertiesFile) {
     static void extractSecureSettings(SecureSettings secrets, Map<String, String> propertiesMap) {
         final Set<String> settingNames = secrets.getSettingNames();
         for (String key : List.of("api_key", "secret_token")) {
-            String prefix = "telemetry.";
-            if (settingNames.contains(prefix + key)) {
-                try (SecureString token = secrets.getString(prefix + key)) {
-                    propertiesMap.put(key, token.toString());
+            for (String prefix : List.of("telemetry.", "tracing.apm.")) {
+                if (settingNames.contains(prefix + key)) {
+                    if (propertiesMap.containsKey(key)) {
+                        throw new IllegalStateException(
+                            Strings.format("Duplicate telemetry setting: [telemetry.%s] and [tracing.apm.%s]", key, key)
+                        );
+                    }
+
+                    try (SecureString token = secrets.getString(prefix + key)) {
+                        propertiesMap.put(key, token.toString());
+                    }
                 }
             }
+
         }
     }
 
@@ -219,12 +227,44 @@ private static Map<String, String> extractDynamicSettings(Map<String, String> pr
     static Map<String, String> extractApmSettings(Settings settings) throws UserException {
         final Map<String, String> propertiesMap = new HashMap<>();
 
+        // tracing.apm.agent. is deprecated by telemetry.agent.
         final String telemetryAgentPrefix = "telemetry.agent.";
+        final String deprecatedTelemetryAgentPrefix = "tracing.apm.agent.";
 
         final Settings telemetryAgentSettings = settings.getByPrefix(telemetryAgentPrefix);
         telemetryAgentSettings.keySet().forEach(key -> propertiesMap.put(key, String.valueOf(telemetryAgentSettings.get(key))));
 
+        final Settings apmAgentSettings = settings.getByPrefix(deprecatedTelemetryAgentPrefix);
+        for (String key : apmAgentSettings.keySet()) {
+            if (propertiesMap.containsKey(key)) {
+                throw new IllegalStateException(
+                    Strings.format(
+                        "Duplicate telemetry setting: [%s%s] and [%s%s]",
+                        telemetryAgentPrefix,
+                        key,
+                        deprecatedTelemetryAgentPrefix,
+                        key
+                    )
+                );
+            }
+            propertiesMap.put(key, String.valueOf(apmAgentSettings.get(key)));
+        }
+
         StringJoiner globalLabels = extractGlobalLabels(telemetryAgentPrefix, propertiesMap, settings);
+        if (globalLabels.length() == 0) {
+            globalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
+        } else {
+            StringJoiner tracingGlobalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
+            if (tracingGlobalLabels.length() != 0) {
+                throw new IllegalArgumentException(
+                    "Cannot have global labels with tracing.agent prefix ["
+                        + globalLabels
+                        + "] and telemetry.apm.agent prefix ["
+                        + tracingGlobalLabels
+                        + "]"
+                );
+            }
+        }
         if (globalLabels.length() > 0) {
             propertiesMap.put("global_labels", globalLabels.toString());
         }
@@ -234,7 +274,7 @@ static Map<String, String> extractApmSettings(Settings settings) throws UserExce
             if (propertiesMap.containsKey(key)) {
                 throw new UserException(
                     ExitCodes.CONFIG,
-                    "Do not set a value for [telemetry.agent." + key + "], as this is configured automatically by Elasticsearch"
+                    "Do not set a value for [tracing.apm.agent." + key + "], as this is configured automatically by Elasticsearch"
                 );
             }
         }

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java

@@ -25,15 +25,18 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 import static org.elasticsearch.test.MapMatcher.matchesMap;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
@@ -79,63 +82,109 @@ public void testFileDeleteWorks() throws IOException {
     }
 
     public void testExtractSecureSettings() {
-        MockSecureSettings secureSettings = new MockSecureSettings();
-        secureSettings.setString("telemetry.secret_token", "token");
-        secureSettings.setString("telemetry.api_key", "key");
+        MockSecureSettings duplicateSecureSettings = new MockSecureSettings();
 
-        Map<String, String> propertiesMap = new HashMap<>();
-        APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap);
+        for (String prefix : List.of("telemetry.", "tracing.apm.")) {
+            MockSecureSettings secureSettings = new MockSecureSettings();
+            secureSettings.setString(prefix + "secret_token", "token");
+            secureSettings.setString(prefix + "api_key", "key");
 
-        assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key")));
-    }
+            duplicateSecureSettings.setString(prefix + "api_key", "secret");
 
-    public void testExtractSettings() throws UserException {
-        Settings defaults = Settings.builder()
-            .put("telemetry.agent.server_url", "https://myurl:443")
-            .put("telemetry.agent.service_node_name", "instance-0000000001")
-            .build();
+            Map<String, String> propertiesMap = new HashMap<>();
+            APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap);
+
+            assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key")));
+        }
 
-        var name = "APM Tracing";
-        var deploy = "123";
-        var org = "456";
-        var extracted = APMJvmOptions.extractApmSettings(
-            Settings.builder()
-                .put(defaults)
-                .put("telemetry.agent.global_labels.deployment_name", name)
-                .put("telemetry.agent.global_labels.deployment_id", deploy)
-                .put("telemetry.agent.global_labels.organization_id", org)
-                .build()
+        Exception exception = expectThrows(
+            IllegalStateException.class,
+            () -> APMJvmOptions.extractSecureSettings(duplicateSecureSettings, new HashMap<>())
         );
+        assertThat(exception.getMessage(), containsString("Duplicate telemetry setting"));
+        assertThat(exception.getMessage(), containsString("telemetry.api_key"));
+        assertThat(exception.getMessage(), containsString("tracing.apm.api_key"));
 
-        assertThat(
-            extracted,
-            allOf(
-                hasEntry("server_url", "https://myurl:443"),
-                hasEntry("service_node_name", "instance-0000000001"),
-                hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one
-                not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys
+    }
+
+    public void testExtractSettings() throws UserException {
+        Function<String, Settings.Builder> buildSettings = (prefix) -> Settings.builder()
+            .put(prefix + "server_url", "https://myurl:443")
+            .put(prefix + "service_node_name", "instance-0000000001");
+
+        for (String prefix : List.of("tracing.apm.agent.", "telemetry.agent.")) {
+            var name = "APM Tracing";
+            var deploy = "123";
+            var org = "456";
+            var extracted = APMJvmOptions.extractApmSettings(
+                buildSettings.apply(prefix)
+                    .put(prefix + "global_labels.deployment_name", name)
+                    .put(prefix + "global_labels.deployment_id", deploy)
+                    .put(prefix + "global_labels.organization_id", org)
+                    .build()
+            );
+
+            assertThat(
+                extracted,
+                allOf(
+                    hasEntry("server_url", "https://myurl:443"),
+                    hasEntry("service_node_name", "instance-0000000001"),
+                    hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one
+                    not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys
+                )
+            );
+
+            List<String> labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
+            assertThat(labels, hasSize(3));
+            assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy));
+
+            // test replacing with underscores and skipping empty
+            name = "APM=Tracing";
+            deploy = "";
+            org = ",456";
+            extracted = APMJvmOptions.extractApmSettings(
+                buildSettings.apply(prefix)
+                    .put(prefix + "global_labels.deployment_name", name)
+                    .put(prefix + "global_labels.deployment_id", deploy)
+                    .put(prefix + "global_labels.organization_id", org)
+                    .build()
+            );
+            labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
+            assertThat(labels, hasSize(2));
+            assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456"));
+        }
+
+        IllegalStateException err = expectThrows(
+            IllegalStateException.class,
+            () -> APMJvmOptions.extractApmSettings(
+                Settings.builder()
+                    .put("tracing.apm.agent.server_url", "https://myurl:443")
+                    .put("telemetry.agent.server_url", "https://myurl-2:443")
+                    .build()
             )
         );
+        assertThat(err.getMessage(), is("Duplicate telemetry setting: [telemetry.agent.server_url] and [tracing.apm.agent.server_url]"));
+    }
+
+    public void testNoMixedLabels() {
+        String telemetryAgent = "telemetry.agent.";
+        String tracingAgent = "tracing.apm.agent.";
+        Settings settings = Settings.builder()
+            .put("tracing.apm.enabled", true)
+            .put(telemetryAgent + "server_url", "https://myurl:443")
+            .put(telemetryAgent + "service_node_name", "instance-0000000001")
+            .put(tracingAgent + "global_labels.deployment_id", "123")
+            .put(telemetryAgent + "global_labels.organization_id", "456")
+            .build();
 
-        List<String> labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
-        assertThat(labels, hasSize(3));
-        assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy));
-
-        // test replacing with underscores and skipping empty
-        name = "APM=Tracing";
-        deploy = "";
-        org = ",456";
-        extracted = APMJvmOptions.extractApmSettings(
-            Settings.builder()
-                .put(defaults)
-                .put("telemetry.agent.global_labels.deployment_name", name)
-                .put("telemetry.agent.global_labels.deployment_id", deploy)
-                .put("telemetry.agent.global_labels.organization_id", org)
-                .build()
+        IllegalArgumentException err = assertThrows(IllegalArgumentException.class, () -> APMJvmOptions.extractApmSettings(settings));
+        assertThat(
+            err.getMessage(),
+            is(
+                "Cannot have global labels with tracing.agent prefix [organization_id=456] and"
+                    + " telemetry.apm.agent prefix [deployment_id=123]"
+            )
         );
-        labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
-        assertThat(labels, hasSize(2));
-        assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456"));
     }
 
     private Path makeFakeAgentJar() throws IOException {

docs/changelog/118188.yaml

@@ -0,0 +1,5 @@
+pr: 118188
+summary: Check for early termination in Driver
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/119575.yaml

@@ -0,0 +1,6 @@
+pr: 119575
+summary: Fix realtime get of nested fields with synthetic source
+area: Mapping
+type: bug
+issues:
+ - 119553

docs/changelog/119679.yaml

@@ -0,0 +1,5 @@
+pr: 119679
+summary: Support mTLS for the Elastic Inference Service integration inside the inference API
+area: Machine Learning
+type: feature
+issues: []

docs/changelog/120020.yaml

@@ -0,0 +1,5 @@
+pr: 120020
+summary: Resume Driver on cancelled or early finished
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/120055.yaml

@@ -0,0 +1,5 @@
+pr: 120055
+summary: Optimize loading mappings when determining synthetic source usage and whether host.name can be sorted on.
+area: Logs
+type: enhancement
+issues: []