Merge branch 'main' into threadpool-merge-scheduler-sort-all-merges

albertzaharovits · Jan 23, 2025 · 68db209 · 68db209
2 parents 60a71b8 + 1484f78
commit 68db209
Show file tree

Hide file tree

Showing 634 changed files with 8,649 additions and 6,780 deletions.
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.16.3", "8.17.1", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.4", "8.17.2", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -287,8 +287,8 @@ steps:
         env:
           BWC_VERSION: 8.15.5
 
-      - label: "{{matrix.image}} / 8.16.3 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.3
+      - label: "{{matrix.image}} / 8.16.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -301,10 +301,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.16.3
+          BWC_VERSION: 8.16.4
 
-      - label: "{{matrix.image}} / 8.17.1 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.1
+      - label: "{{matrix.image}} / 8.17.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.2
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -317,7 +317,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.17.1
+          BWC_VERSION: 8.17.2
 
       - label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -306,8 +306,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.16.3 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.3#bwcTest
+      - label: 8.16.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -316,7 +316,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.16.3
+          BWC_VERSION: 8.16.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -325,8 +325,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.17.1 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.1#bwcTest
+      - label: 8.17.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.2#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -335,7 +335,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.17.1
+          BWC_VERSION: 8.17.2
         retry:
           automatic:
             - exit_status: "-1"
@@ -448,7 +448,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.16.3", "8.17.1", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.4", "8.17.2", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -490,7 +490,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.16.3", "8.17.1", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.4", "8.17.2", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/pull-request/release-tests.yml b/.buildkite/pipelines/pull-request/release-tests.yml
@@ -1,11 +1,22 @@
 config:
   allow-labels: test-release
 steps:
-  - label: release-tests
-    command: .buildkite/scripts/release-tests.sh
-    timeout_in_minutes: 300
-    agents:
-      provider: gcp
-      image: family/elasticsearch-ubuntu-2004
-      diskSizeGb: 350
-      machineType: custom-32-98304
+  - group: release-tests
+    steps:
+      - label: "{{matrix.CHECK_TASK}} / release-tests"
+        key: "packaging-tests-unix"
+        command: .buildkite/scripts/release-tests.sh {{matrix.CHECK_TASK}}
+        timeout_in_minutes: 120
+        matrix:
+          setup:
+            CHECK_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - checkPart4
+              - checkPart5
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          diskSizeGb: 350
+          machineType: custom-32-98304
diff --git a/.buildkite/scripts/release-tests.sh b/.buildkite/scripts/release-tests.sh
@@ -20,4 +20,4 @@ curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/m
 curl --fail -o "${ML_IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/${ES_VERSION}/ml-cpp-${ES_VERSION}.zip" https://artifacts-snapshot.elastic.co/ml-cpp/${ES_VERSION}-SNAPSHOT/downloads/ml-cpp/ml-cpp-${ES_VERSION}-SNAPSHOT.zip
 
 .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false -Dbuild.ml_cpp.repo=file://${ML_IVY_REPO} \
-  -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef assemble functionalTests
+  -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef ${@:-functionalTests}
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -15,7 +15,7 @@ BWC_VERSION:
   - "8.13.4"
   - "8.14.3"
   - "8.15.5"
-  - "8.16.3"
-  - "8.17.1"
+  - "8.16.4"
+  - "8.17.2"
   - "8.18.0"
   - "9.0.0"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
@@ -1,5 +1,5 @@
 BWC_VERSION:
-  - "8.16.3"
-  - "8.17.1"
+  - "8.16.4"
+  - "8.17.2"
   - "8.18.0"
   - "9.0.0"
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java
@@ -60,6 +60,7 @@
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -77,7 +78,7 @@ public class ScriptScoreBenchmark {
     private final PluginsService pluginsService = new PluginsService(
         Settings.EMPTY,
         null,
-        PluginsLoader.createPluginsLoader(null, Path.of(System.getProperty("plugins.dir")))
+        PluginsLoader.createPluginsLoader(Set.of(), PluginsLoader.loadPluginsBundles(Path.of(System.getProperty("plugins.dir"))))
     );
     private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class).toList());
 

diff --git a/build.gradle b/build.gradle
@@ -293,22 +293,57 @@ allprojects {
     }
   }
 
+  ext.withReleaseBuild = { Closure config ->
+    if(buildParams.snapshotBuild == false) {
+      config.call()
+    }
+  }
+
   plugins.withId('lifecycle-base') {
     if (project.path.startsWith(":x-pack:")) {
       if (project.path.contains("security") || project.path.contains(":ml")) {
-        tasks.register('checkPart4') { dependsOn 'check' }
+        tasks.register('checkPart4') {
+          dependsOn 'check'
+          withReleaseBuild {
+            dependsOn 'assemble'
+          }
+        }
       } else if (project.path == ":x-pack:plugin" || project.path.contains("ql") || project.path.contains("smoke-test")) {
-        tasks.register('checkPart3') { dependsOn 'check' }
+        tasks.register('checkPart3') {
+          dependsOn 'check'
+          withReleaseBuild {
+            dependsOn 'assemble'
+          }
+        }
       } else if (project.path.contains("multi-node")) {
-        tasks.register('checkPart5') { dependsOn 'check' }
+        tasks.register('checkPart5') {
+          dependsOn 'check'
+          withReleaseBuild {
+            dependsOn 'assemble'
+          }
+        }
       } else {
-        tasks.register('checkPart2') { dependsOn 'check' }
+        tasks.register('checkPart2') {
+          dependsOn 'check'
+          withReleaseBuild {
+            dependsOn 'assemble'
+          }
+        }
       }
     } else {
-      tasks.register('checkPart1') { dependsOn 'check' }
+      tasks.register('checkPart1') {
+        dependsOn 'check'
+        withReleaseBuild {
+          dependsOn 'assemble'
+        }
+      }
+    }
+    tasks.register('functionalTests') {
+      dependsOn 'check'
+      withReleaseBuild {
+        dependsOn 'assemble'
+      }
     }
-
-    tasks.register('functionalTests') { dependsOn 'check' }
   }
 
   /*

diff --git a/docs/changelog/117469.yaml b/docs/changelog/117469.yaml
diff --git a/docs/changelog/117840.yaml b/docs/changelog/117840.yaml
diff --git a/docs/changelog/117851.yaml b/docs/changelog/117851.yaml
diff --git a/docs/changelog/118454.yaml b/docs/changelog/118454.yaml
diff --git a/docs/changelog/118516.yaml b/docs/changelog/118516.yaml
diff --git a/docs/changelog/118603.yaml b/docs/changelog/118603.yaml
diff --git a/docs/changelog/118757.yaml b/docs/changelog/118757.yaml
diff --git a/docs/changelog/118816.yaml b/docs/changelog/118816.yaml
diff --git a/docs/changelog/118837.yaml b/docs/changelog/118837.yaml
diff --git a/docs/changelog/118844.yaml b/docs/changelog/118844.yaml
diff --git a/docs/changelog/118919.yaml b/docs/changelog/118919.yaml
diff --git a/docs/changelog/118921.yaml b/docs/changelog/118921.yaml
diff --git a/docs/changelog/118954.yaml b/docs/changelog/118954.yaml
diff --git a/docs/changelog/118958.yaml b/docs/changelog/118958.yaml
diff --git a/docs/changelog/118999.yaml b/docs/changelog/118999.yaml
diff --git a/docs/changelog/119131.yaml b/docs/changelog/119131.yaml
diff --git a/docs/changelog/119134.yaml b/docs/changelog/119134.yaml
diff --git a/docs/changelog/119233.yaml b/docs/changelog/119233.yaml
diff --git a/docs/changelog/119449.yaml b/docs/changelog/119449.yaml
diff --git a/docs/changelog/119495.yaml b/docs/changelog/119495.yaml
diff --git a/docs/changelog/119516.yaml b/docs/changelog/119516.yaml
diff --git a/docs/changelog/119580.yaml b/docs/changelog/119580.yaml
@@ -0,0 +1,5 @@
+pr: 119580
+summary: Do not serialize `EsIndex` in plan
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/119637.yaml b/docs/changelog/119637.yaml
diff --git a/docs/changelog/119743.yaml b/docs/changelog/119743.yaml
@@ -0,0 +1,5 @@
+pr: 119743
+summary: POC mark read-only
+area: Engine
+type: enhancement
+issues: []
diff --git a/docs/changelog/119750.yaml b/docs/changelog/119750.yaml