Merge remote-tracking branch 'origin/docs-esql-more-functions-example…

…s' into docs-esql-more-functions-examples

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.17", "8.12.1", "8.13.0"]
+            BWC_VERSION: ["7.17.18", "8.12.1", "8.13.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1105,6 +1105,22 @@ steps:
         env:
           BWC_VERSION: 7.17.17
 
+      - label: "{{matrix.image}} / 7.17.18 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.18
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.18
+
       - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -672,6 +672,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 7.17.17
+      - label: 7.17.18 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.18#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.18
       - label: 8.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest
         timeout_in_minutes: 300

.buildkite/scripts/fixture-deploy.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "$DOCKER_REGISTRY_PASSWORD" | docker login -u "$DOCKER_REGISTRY_USERNAME" --password-stdin docker.elastic.co
+unset DOCKER_REGISTRY_USERNAME DOCKER_REGISTRY_PASSWORD
+
+docker buildx create --use
+.ci/scripts/run-gradle.sh deployFixtureDockerImages

.ci/bwcVersions

@@ -66,6 +66,7 @@ BWC_VERSION:
   - "7.17.15"
   - "7.17.16"
   - "7.17.17"
+  - "7.17.18"
   - "8.0.0"
   - "8.0.1"
   - "8.1.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,4 @@
 BWC_VERSION:
-  - "7.17.17"
+  - "7.17.18"
   - "8.12.1"
   - "8.13.0"

TRACING.md

@@ -24,7 +24,7 @@ In your `elasticsearch.yml` add the following configuration:
 
 ```
 tracing.apm.enabled: true
-tracing.apm.agent.server_url: https://<your-apm-server>:443
+telemetry.agent.server_url: https://<your-apm-server>:443
 ```
 
 When using a secret token to authenticate with the APM server, you must add it to the Elasticsearch keystore under `tracing.apm.secret_token`. For example, execute:
@@ -34,7 +34,7 @@ When using a secret token to authenticate with the APM server, you must add it t
 then enter the token when prompted. If you are using API keys, change the keystore key name to `tracing.apm.api_key`.
 
 All APM settings live under `tracing.apm`. All settings related to the Java agent
-go under `tracing.apm.agent`. Anything you set under there will be propagated to
+go under `telemetry.agent`. Anything you set under there will be propagated to
 the agent.
 
 For agent settings that can be changed dynamically, you can use the cluster
@@ -43,7 +43,7 @@ settings REST API. For example, to change the sampling rate:
     curl -XPUT \
       -H "Content-type: application/json" \
       -u "$USERNAME:$PASSWORD" \
-      -d '{ "persistent": { "tracing.apm.agent.transaction_sample_rate": "0.75" } }' \
+      -d '{ "persistent": { "telemetry.agent.transaction_sample_rate": "0.75" } }' \
       https://localhost:9200/_cluster/settings
 
 

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java

@@ -189,6 +189,11 @@ public String indexName() {
                     return "benchmark";
                 }
 
+                @Override
+                public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
+                    return MappedFieldType.FieldExtractPreference.NONE;
+                }
+
                 @Override
                 public SearchLookup lookup() {
                     throw new UnsupportedOperationException();

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java

@@ -20,8 +20,10 @@
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.provider.MapProperty;
 import org.gradle.api.provider.Property;
+import org.gradle.api.provider.SetProperty;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputDirectory;
+import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.OutputFile;
 import org.gradle.api.tasks.PathSensitive;
 import org.gradle.api.tasks.PathSensitivity;
@@ -36,14 +38,15 @@
 import java.nio.file.Files;
 import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 
 /**
  * This task wraps up the details of building a Docker image, including adding a pull
  * mechanism that can retry, and emitting the image SHA as a task output.
  */
-public class DockerBuildTask extends DefaultTask {
+public abstract class DockerBuildTask extends DefaultTask {
     private static final Logger LOGGER = Logging.getLogger(DockerBuildTask.class);
 
     private final WorkerExecutor workerExecutor;
@@ -55,15 +58,13 @@ public class DockerBuildTask extends DefaultTask {
     private boolean noCache = true;
     private String[] baseImages;
     private MapProperty<String, String> buildArgs;
-    private Property<String> platform;
 
     @Inject
     public DockerBuildTask(WorkerExecutor workerExecutor, ObjectFactory objectFactory, ProjectLayout projectLayout) {
         this.workerExecutor = workerExecutor;
         this.markerFile = objectFactory.fileProperty();
         this.dockerContext = objectFactory.directoryProperty();
         this.buildArgs = objectFactory.mapProperty(String.class, String.class);
-        this.platform = objectFactory.property(String.class).convention(Architecture.current().dockerPlatform);
         this.markerFile.set(projectLayout.getBuildDirectory().file("markers/" + this.getName() + ".marker"));
     }
 
@@ -75,9 +76,10 @@ public void build() {
             params.getTags().set(Arrays.asList(tags));
             params.getPull().set(pull);
             params.getNoCache().set(noCache);
+            params.getPush().set(getPush().getOrElse(false));
             params.getBaseImages().set(Arrays.asList(baseImages));
             params.getBuildArgs().set(buildArgs);
-            params.getPlatform().set(platform);
+            params.getPlatforms().set(getPlatforms());
         });
     }
 
@@ -129,10 +131,16 @@ public MapProperty<String, String> getBuildArgs() {
     }
 
     @Input
-    public Property<String> getPlatform() {
-        return platform;
+    public abstract SetProperty<String> getPlatforms();
+
+    public void setPlatform(String platform) {
+        getPlatforms().set(Arrays.asList(platform));
     }
 
+    @Input
+    @Optional
+    public abstract Property<Boolean> getPush();
+
     @OutputFile
     public RegularFileProperty getMarkerFile() {
         return markerFile;
@@ -181,7 +189,7 @@ public void execute() {
             }
 
             final List<String> tags = parameters.getTags().get();
-            final boolean isCrossPlatform = parameters.getPlatform().get().equals(Architecture.current().dockerPlatform) == false;
+            final boolean isCrossPlatform = isCrossPlatform();
 
             LoggedExec.exec(execOperations, spec -> {
                 spec.executable("docker");
@@ -193,7 +201,7 @@ public void execute() {
                 spec.args("build", parameters.getDockerContext().get().getAsFile().getAbsolutePath());
 
                 if (isCrossPlatform) {
-                    spec.args("--platform", parameters.getPlatform().get());
+                    spec.args("--platform", parameters.getPlatforms().get().stream().collect(Collectors.joining(",")));
                 }
 
                 if (parameters.getNoCache().get()) {
@@ -203,18 +211,34 @@ public void execute() {
                 tags.forEach(tag -> spec.args("--tag", tag));
 
                 parameters.getBuildArgs().get().forEach((k, v) -> spec.args("--build-arg", k + "=" + v));
+
+                if (parameters.getPush().getOrElse(false)) {
+                    spec.args("--push");
+                }
             });
 
             // Fetch the Docker image's hash, and write it to desk as the task's output. Doing this allows us
             // to do proper up-to-date checks in Gradle.
             try {
+                // multi-platform image builds do not end up in local registry, so we need to pull the just build image
+                // first to get the checksum and also serves as a test for the image being pushed correctly
+                if (parameters.getPlatforms().get().size() > 1 && parameters.getPush().getOrElse(false)) {
+                    pullBaseImage(tags.get(0));
+                }
                 final String checksum = getImageChecksum(tags.get(0));
                 Files.writeString(parameters.getMarkerFile().getAsFile().get().toPath(), checksum + "\n");
             } catch (IOException e) {
                 throw new RuntimeException("Failed to write marker file", e);
             }
         }
 
+        private boolean isCrossPlatform() {
+            return getParameters().getPlatforms()
+                .get()
+                .stream()
+                .anyMatch(any -> any.equals(Architecture.current().dockerPlatform) == false);
+        }
+
         private String getImageChecksum(String imageTag) {
             final ByteArrayOutputStream stdout = new ByteArrayOutputStream();
 
@@ -243,6 +267,8 @@ interface Parameters extends WorkParameters {
 
         MapProperty<String, String> getBuildArgs();
 
-        Property<String> getPlatform();
+        SetProperty<String> getPlatforms();
+
+        Property<Boolean> getPush();
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java

@@ -28,6 +28,7 @@
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -187,7 +188,7 @@ public void checkDependencies() {
         }
         File licensesDirAsFile = licensesDir.get().getAsFile();
         if (dependencies.isEmpty()) {
-            if (licensesDirAsFile.exists()) {
+            if (licensesDirAsFile.exists() && allIgnored() == false) {
                 throw new GradleException("Licenses dir " + licensesDirAsFile + " exists, but there are no dependencies");
             }
             return; // no dependencies to check
@@ -227,6 +228,10 @@ public void checkDependencies() {
         sources.forEach((item, exists) -> failIfAnyMissing(item, exists, "sources"));
     }
 
+    private boolean allIgnored() {
+        return Arrays.asList(getLicensesDir().listFiles()).stream().map(f -> f.getName()).allMatch(ignoreFiles::contains);
+    }
+
     // This is just a marker output folder to allow this task being up-to-date.
     // The check logic is exception driven so a failed tasks will not be defined
     // by this output but when successful we can safely mark the task as up-to-date.

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/UpdateVersionsTask.java

@@ -8,13 +8,17 @@
 
 package org.elasticsearch.gradle.internal.release;
 
+import com.github.javaparser.GeneratedJavaParserConstants;
 import com.github.javaparser.StaticJavaParser;
 import com.github.javaparser.ast.CompilationUnit;
 import com.github.javaparser.ast.NodeList;
 import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration;
 import com.github.javaparser.ast.body.FieldDeclaration;
 import com.github.javaparser.ast.body.VariableDeclarator;
 import com.github.javaparser.ast.expr.NameExpr;
+import com.github.javaparser.ast.observer.ObservableProperty;
+import com.github.javaparser.printer.ConcreteSyntaxModel;
+import com.github.javaparser.printer.concretesyntaxmodel.CsmElement;
 import com.github.javaparser.printer.lexicalpreservation.LexicalPreservingPrinter;
 import com.google.common.annotations.VisibleForTesting;
 
@@ -27,6 +31,7 @@
 import org.gradle.initialization.layout.BuildLayout;
 
 import java.io.IOException;
+import java.lang.reflect.Field;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
@@ -42,7 +47,84 @@
 import javax.annotation.Nullable;
 import javax.inject.Inject;
 
+import static com.github.javaparser.ast.observer.ObservableProperty.TYPE_PARAMETERS;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmConditional.Condition.FLAG;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.block;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.child;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.comma;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.comment;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.conditional;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.list;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.newline;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.none;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.sequence;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.space;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.string;
+import static com.github.javaparser.printer.concretesyntaxmodel.CsmElement.token;
+
 public class UpdateVersionsTask extends DefaultTask {
+
+    static {
+        replaceDefaultJavaParserClassCsm();
+    }
+
+    /*
+     * The default JavaParser CSM which it uses to format any new declarations added to a class
+     * inserts two newlines after each declaration. Our version classes only have one newline.
+     * In order to get javaparser lexical printer to use our format, we have to completely replace
+     * the statically declared CSM pattern using hacky reflection
+     * to access the static map where these are stored, and insert a replacement that is identical
+     * apart from only one newline at the end of each member declaration, rather than two.
+     */
+    private static void replaceDefaultJavaParserClassCsm() {
+        try {
+            Field classCsms = ConcreteSyntaxModel.class.getDeclaredField("concreteSyntaxModelByClass");
+            classCsms.setAccessible(true);
+            @SuppressWarnings({ "unchecked", "rawtypes" })
+            Map<Class, CsmElement> csms = (Map) classCsms.get(null);
+
+            // copied from the static initializer in ConcreteSyntaxModel
+            csms.put(
+                ClassOrInterfaceDeclaration.class,
+                sequence(
+                    comment(),
+                    list(ObservableProperty.ANNOTATIONS, newline(), none(), newline()),
+                    list(ObservableProperty.MODIFIERS, space(), none(), space()),
+                    conditional(
+                        ObservableProperty.INTERFACE,
+                        FLAG,
+                        token(GeneratedJavaParserConstants.INTERFACE),
+                        token(GeneratedJavaParserConstants.CLASS)
+                    ),
+                    space(),
+                    child(ObservableProperty.NAME),
+                    list(
+                        TYPE_PARAMETERS,
+                        sequence(comma(), space()),
+                        string(GeneratedJavaParserConstants.LT),
+                        string(GeneratedJavaParserConstants.GT)
+                    ),
+                    list(
+                        ObservableProperty.EXTENDED_TYPES,
+                        sequence(string(GeneratedJavaParserConstants.COMMA), space()),
+                        sequence(space(), token(GeneratedJavaParserConstants.EXTENDS), space()),
+                        none()
+                    ),
+                    list(
+                        ObservableProperty.IMPLEMENTED_TYPES,
+                        sequence(string(GeneratedJavaParserConstants.COMMA), space()),
+                        sequence(space(), token(GeneratedJavaParserConstants.IMPLEMENTS), space()),
+                        none()
+                    ),
+                    space(),
+                    block(sequence(newline(), list(ObservableProperty.MEMBERS, sequence(newline()/*, newline()*/), newline(), newline())))
+                )
+            );
+        } catch (ReflectiveOperationException e) {
+            throw new AssertionError(e);
+        }
+    }
+
     private static final Logger LOGGER = Logging.getLogger(UpdateVersionsTask.class);
 
     static final String SERVER_MODULE_PATH = "server/src/main/java/";