Merge remote-tracking branch 'upstream/main' into ecs-mappings-date-f…

…ields
eyalkoren · Sep 9, 2024 · 220b7bc · 220b7bc
2 parents 1f01aca + a43ffd5
commit 220b7bc
Show file tree

Hide file tree

Showing 567 changed files with 13,477 additions and 4,614 deletions.
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -62,7 +62,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"]
+            BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -594,8 +594,8 @@ steps:
         env:
           BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.1 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.1
+      - label: "{{matrix.image}} / 8.15.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.2
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -609,7 +609,7 @@ steps:
           buildDirectory: /dev/shm/bk
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.15.1
+          BWC_VERSION: 8.15.2
 
       - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -662,8 +662,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.1 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.1#bwcTest
+      - label: 8.15.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.2#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -673,7 +673,7 @@ steps:
           preemptible: true
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.15.1
+          BWC_VERSION: 8.15.2
         retry:
           automatic:
             - exit_status: "-1"
@@ -771,7 +771,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"]
+            BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -821,7 +821,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"]
+            BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -32,5 +32,5 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.4"
   - "8.14.3"
-  - "8.15.1"
+  - "8.15.2"
   - "8.16.0"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
@@ -1,4 +1,4 @@
 BWC_VERSION:
   - "7.17.24"
-  - "8.15.1"
+  - "8.15.2"
   - "8.16.0"
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -70,3 +70,7 @@ server/src/main/java/org/elasticsearch/threadpool @elastic/es-core-infra
 # Security
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege @elastic/es-security
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java @elastic/es-security
+
+# Analytical engine
+x-pack/plugin/esql @elastic/es-analytical-engine
+x-pack/plugin/esql-core @elastic/es-analytical-engine
diff --git a/.github/workflows/sync-main-to-jdk-branch.yml b/.github/workflows/sync-main-to-jdk-branch.yml
@@ -0,0 +1,20 @@
+# Daily update of JDK update branch with changes from main
+name: "Merge main to openjdk23-bundle branch"
+on:
+  schedule:
+    - cron: '30 17 * * *'
+  workflow_dispatch: {}
+
+jobs:
+  merge-branch:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@master
+
+      - name: merge
+        uses: devmasx/[email protected]
+        with:
+          type: 'now'
+          target_branch: openjdk23-bundle
+          github_token: ${{ secrets.ELASTICSEARCHMACHINE_TOKEN }}
diff --git a/...org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy b/...org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy
@@ -55,8 +55,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest", '--stacktrace').build()
 
         then:
-        // we set the task to be skipped if there are no matching tests in the compatibility test sourceSet
-        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
+        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
         result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
@@ -165,7 +164,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         then:
         result.task(':check').outcome == TaskOutcome.UP_TO_DATE
         result.task(':checkRestCompat').outcome == TaskOutcome.UP_TO_DATE
-        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
+        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
         result.task(transformTask).outcome == TaskOutcome.NO_SOURCE

diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle
@@ -41,7 +41,7 @@ develocity {
       if (BuildParams.inFipsJvm) {
         tag 'FIPS'
       }
-      println "onCI = $onCI"
+
       if (onCI) { //Buildkite-specific build scan metadata
         String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL')
         def branch = System.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') ?: System.getenv('BUILDKITE_BRANCH')

diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
@@ -167,7 +167,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
             vmParameters = [
               '-ea',
               '-Djava.security.manager=allow',
-              '-Djava.locale.providers=SPI,COMPAT',
+              '-Djava.locale.providers=SPI,CLDR',
               '-Des.nativelibs.path="' + testLibraryPath + '"',
               // TODO: only open these for mockito when it is modularized
               '--add-opens=java.base/java.security.cert=ALL-UNNAMED',

diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
@@ -12,27 +12,40 @@
  * This class models the different Docker base images that are used to build Docker distributions of Elasticsearch.
  */
 public enum DockerBase {
-    DEFAULT("ubuntu:20.04", ""),
+    DEFAULT("ubuntu:20.04", "", "apt-get"),
 
     // "latest" here is intentional, since the image name specifies "8"
-    UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi8"),
+    UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi8", "microdnf"),
 
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
-    IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank"),
+    IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),
 
     // Base image with extras for Cloud
-    CLOUD("ubuntu:20.04", "-cloud"),
+    CLOUD("ubuntu:20.04", "-cloud", "apt-get"),
 
     // Based on CLOUD above, with more extras. We don't set a base image because
     // we programmatically extend from the Cloud image.
-    CLOUD_ESS(null, "-cloud-ess");
+    CLOUD_ESS(null, "-cloud-ess", "apt-get"),
+
+    // Chainguard based wolfi image with latest jdk
+    WOLFI(
+        "docker.elastic.co/wolfi/chainguard-base:latest@sha256:c16d3ad6cebf387e8dd2ad769f54320c4819fbbaa21e729fad087c7ae223b4d0",
+        "wolfi",
+        "apk"
+    );
 
     private final String image;
     private final String suffix;
+    private final String packageManager;
 
     DockerBase(String image, String suffix) {
+        this(image, suffix, "apt-get");
+    }
+
+    DockerBase(String image, String suffix, String packageManager) {
         this.image = image;
         this.suffix = suffix;
+        this.packageManager = packageManager;
     }
 
     public String getImage() {
@@ -42,4 +55,8 @@ public String getImage() {
     public String getSuffix() {
         return suffix;
     }
+
+    public String getPackageManager() {
+        return packageManager;
+    }
 }
diff --git a/...internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/...internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java
@@ -92,7 +92,7 @@ public void execute(Task t) {
                     mkdirs(test.getWorkingDir().toPath().resolve("temp").toFile());
 
                     // TODO remove once jvm.options are added to test system properties
-                    test.systemProperty("java.locale.providers", "SPI,COMPAT");
+                    test.systemProperty("java.locale.providers", "SPI,CLDR");
                 }
             });
             test.getJvmArgumentProviders().add(nonInputProperties);

diff --git a/...l/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java b/...l/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java
@@ -177,6 +177,9 @@ private static String distributionProjectName(ElasticsearchDistribution distribu
         if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_CLOUD_ESS) {
             return projectName + "cloud-ess-docker" + archString + "-export";
         }
+        if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI) {
+            return projectName + "wolfi-docker" + archString + "-export";
+        }
         return projectName + distribution.getType().getName();
     }
 

diff --git a/.../elasticsearch/gradle/internal/distribution/DockerWolfiElasticsearchDistributionType.java b/.../elasticsearch/gradle/internal/distribution/DockerWolfiElasticsearchDistributionType.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.distribution;
+
+import org.elasticsearch.gradle.ElasticsearchDistributionType;
+
+public class DockerWolfiElasticsearchDistributionType implements ElasticsearchDistributionType {
+
+    DockerWolfiElasticsearchDistributionType() {}
+
+    @Override
+    public String getName() {
+        return "dockerWolfi";
+    }
+
+    @Override
+    public boolean isDocker() {
+        return true;
+    }
+}
diff --git a/...rg/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java b/...rg/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java
@@ -20,6 +20,7 @@ public class InternalElasticsearchDistributionTypes {
     public static ElasticsearchDistributionType DOCKER_IRONBANK = new DockerIronBankElasticsearchDistributionType();
     public static ElasticsearchDistributionType DOCKER_CLOUD = new DockerCloudElasticsearchDistributionType();
     public static ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType();
+    public static ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType();
 
     public static List<ElasticsearchDistributionType> ALL_INTERNAL = List.of(
         DEB,
@@ -28,6 +29,7 @@ public class InternalElasticsearchDistributionTypes {
         DOCKER_UBI,
         DOCKER_IRONBANK,
         DOCKER_CLOUD,
-        DOCKER_CLOUD_ESS
+        DOCKER_CLOUD_ESS,
+        DOCKER_WOLFI
     );
 }
diff --git a/...tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java b/...tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java
@@ -52,6 +52,7 @@
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_CLOUD_ESS;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_IRONBANK;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_UBI;
+import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM;
 
 /**
@@ -93,6 +94,7 @@ public void apply(Project project) {
 
         for (ElasticsearchDistribution distribution : testDistributions) {
             String taskname = destructiveDistroTestTaskName(distribution);
+            ElasticsearchDistributionType type = distribution.getType();
             TaskProvider<Test> destructiveTask = configureTestTask(project, taskname, distribution, t -> {
                 t.onlyIf(
                     "Docker is not available",
@@ -106,12 +108,13 @@ public void apply(Project project) {
             if (distribution.getPlatform() == Platform.WINDOWS) {
                 windowsTestTasks.add(destructiveTask);
             } else {
-                linuxTestTasks.computeIfAbsent(distribution.getType(), k -> new ArrayList<>()).add(destructiveTask);
+                linuxTestTasks.computeIfAbsent(type, k -> new ArrayList<>()).add(destructiveTask);
             }
             destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask));
-            lifecycleTasks.get(distribution.getType()).configure(t -> t.dependsOn(destructiveTask));
+            TaskProvider<?> lifecycleTask = lifecycleTasks.get(type);
+            lifecycleTask.configure(t -> t.dependsOn(destructiveTask));
 
-            if ((distribution.getType() == DEB || distribution.getType() == RPM) && distribution.getBundledJdk()) {
+            if ((type == DEB || type == RPM) && distribution.getBundledJdk()) {
                 for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) {
                     final ElasticsearchDistribution bwcDistro;
                     if (version.equals(Version.fromString(distribution.getVersion()))) {
@@ -121,7 +124,7 @@ public void apply(Project project) {
                         bwcDistro = createDistro(
                             allDistributions,
                             distribution.getArchitecture(),
-                            distribution.getType(),
+                            type,
                             distribution.getPlatform(),
                             distribution.getBundledJdk(),
                             version.toString()
@@ -147,6 +150,7 @@ private static Map<ElasticsearchDistributionType, TaskProvider<?>> lifecycleTask
         lifecyleTasks.put(DOCKER_IRONBANK, project.getTasks().register(taskPrefix + ".docker-ironbank"));
         lifecyleTasks.put(DOCKER_CLOUD, project.getTasks().register(taskPrefix + ".docker-cloud"));
         lifecyleTasks.put(DOCKER_CLOUD_ESS, project.getTasks().register(taskPrefix + ".docker-cloud-ess"));
+        lifecyleTasks.put(DOCKER_WOLFI, project.getTasks().register(taskPrefix + ".docker-wolfi"));
         lifecyleTasks.put(ARCHIVE, project.getTasks().register(taskPrefix + ".archives"));
         lifecyleTasks.put(DEB, project.getTasks().register(taskPrefix + ".packages"));
         lifecyleTasks.put(RPM, lifecyleTasks.get(DEB));

diff --git a/...asticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java b/...asticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java
@@ -35,6 +35,7 @@
 import org.gradle.api.tasks.Sync;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.testing.Test;
+import org.gradle.language.jvm.tasks.ProcessResources;
 
 import java.io.File;
 import java.nio.file.Path;
@@ -213,6 +214,17 @@ public void apply(Project project) {
             .named(RestResourcesPlugin.COPY_YAML_TESTS_TASK)
             .flatMap(CopyRestTestsTask::getOutputResourceDir);
 
+        // ensure we include other non rest spec related test resources
+        project.getTasks()
+            .withType(ProcessResources.class)
+            .named(yamlCompatTestSourceSet.getProcessResourcesTaskName())
+            .configure(processResources -> {
+                processResources.from(
+                    sourceSets.getByName(YamlRestTestPlugin.YAML_REST_TEST).getResources(),
+                    spec -> { spec.exclude("rest-api-spec/**"); }
+                );
+            });
+
         // setup the test task
         TaskProvider<? extends Test> yamlRestCompatTestTask = registerTestTask(project, yamlCompatTestSourceSet);
         yamlRestCompatTestTask.configure(testTask -> {
@@ -221,7 +233,7 @@ public void apply(Project project) {
             testTask.setTestClassesDirs(
                 yamlTestSourceSet.getOutput().getClassesDirs().plus(yamlCompatTestSourceSet.getOutput().getClassesDirs())
             );
-            testTask.onlyIf("Compatibility tests are available", t -> yamlCompatTestSourceSet.getAllSource().isEmpty() == false);
+            testTask.onlyIf("Compatibility tests are available", t -> yamlCompatTestSourceSet.getOutput().isEmpty() == false);
             testTask.setClasspath(
                 yamlCompatTestSourceSet.getRuntimeClasspath()
                     // remove the "normal" api and tests

diff --git a/distribution/docker/README.md b/distribution/docker/README.md
@@ -6,11 +6,13 @@ the [DockerBase] enum.
    * Default - this is what most people use, and is based on Ubuntu
    * UBI - the same as the default image, but based upon [RedHat's UBI
      images][ubi], specifically their minimal flavour.
+   * Wolfi - the same as the default image, but based upon [Wolfi](https://github.com/wolfi-dev)
    * Iron Bank - this is the US Department of Defence's repository of digitally
      signed, binary container images including both Free and Open-Source
      software (FOSS) and Commercial off-the-shelf (COTS). In practice, this is
      another UBI build, this time on the regular UBI image, with extra
      hardening. See below for more details.
+
    * Cloud - this is mostly the same as the default image, with some notable differences:
       * `filebeat` and `metricbeat` are included
       * `wget` is included