Merge branch '8.x' into backport/8.x/pr-120144

muted-tests.yml

@@ -460,3 +460,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
   method: test {date.IN operator with null in list, finds match}
   issue: https://github.com/elastic/elasticsearch/issues/120156
+- class: org.elasticsearch.search.basic.SearchWithRandomIOExceptionsIT
+  method: testRandomDirectoryIOExceptions
+  issue: https://github.com/elastic/elasticsearch/issues/118733

server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java

@@ -54,6 +54,7 @@ public class TestSystemIndexDescriptor extends SystemIndexDescriptor {
             "version",
             "stack",
             null,
+            null,
             Type.INTERNAL_MANAGED,
             List.of(),
             List.of(),
@@ -75,6 +76,7 @@ public class TestSystemIndexDescriptor extends SystemIndexDescriptor {
             "version",
             "stack",
             Version.fromString(Build.current().minWireCompatVersion()),
+            null,
             Type.INTERNAL_MANAGED,
             List.of(),
             List.of(),

server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java

@@ -14,6 +14,7 @@
 import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
 import org.elasticsearch.action.admin.indices.create.AutoCreateAction;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -149,6 +150,24 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable<Sy
     /** For internally-managed indices, specifies the origin to use when creating or updating the index */
     private final String origin;
 
+    /**
+     * An optional reindexing script to use when migrating an index created
+     * before {@link TransportGetFeatureUpgradeStatusAction#NO_UPGRADE_REQUIRED_INDEX_VERSION}.
+     * This script can be used to modify documents before they are added to the new index.
+     * For example, it can be used to remove deprecated fields from the index.
+     * <br>
+     * Note: the script usually should only exist in  the versions supporting migration to the next major release -
+     * specifically, the last (two) minors of the current major.
+     * It should be created once the last minor branch has diverged from the next major branch (main).
+     * This ensures the script is available only in the versions where it is needed
+     * and avoids removing and maintaining it in the next major branch.
+     * For example: In order to migrate an index created in v7 when upgrading to v9,
+     * the script should be in the v8 minors supporting upgrade to v9 - 8.18 and 8.19.
+     * <br>
+     * See: <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#reindex-scripts">Reindex scripts</a>
+     */
+    private final String migrationScript;
+
     /** The minimum cluster node version required for this descriptor */
     private final Version minimumNodeVersion;
 
@@ -210,6 +229,7 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable<Sy
      *               system indices must do so.
      * @param minimumNodeVersion the minimum cluster node version required for this descriptor
      * @param type The {@link Type} of system index
+     * @param migrationScript The script to apply when migrating this system index, or null
      * @param allowedElasticProductOrigins A list of allowed origin values that should be allowed access in the case of external system
      *                                     indices
      * @param priorSystemIndexDescriptors A list of system index descriptors that describe the same index in a way that is compatible with
@@ -227,6 +247,7 @@ protected SystemIndexDescriptor(
         String mappingsNodeVersionMetaKey,
         String origin,
         @Deprecated Version minimumNodeVersion,
+        String migrationScript,
         Type type,
         List<String> allowedElasticProductOrigins,
         List<SystemIndexDescriptor> priorSystemIndexDescriptors,
@@ -365,6 +386,7 @@ protected SystemIndexDescriptor(
 
         this.description = description;
         this.mappings = mappings;
+        this.migrationScript = migrationScript;
 
         settings = Objects.isNull(settings) ? Settings.EMPTY : settings;
 
@@ -644,6 +666,10 @@ public ExecutorNames getThreadPoolNames() {
         return this.executorNames;
     }
 
+    public String getMigrationScript() {
+        return migrationScript;
+    }
+
     public static Builder builder() {
         return new Builder();
     }
@@ -748,6 +774,7 @@ public static class Builder {
         private String versionMetaKey = null;
         private String origin = null;
         private Version minimumNodeVersion = Version.CURRENT.minimumCompatibilityVersion();
+        private String migrationScript;
         private Type type = Type.INTERNAL_MANAGED;
         private List<String> allowedElasticProductOrigins = List.of();
         private List<SystemIndexDescriptor> priorSystemIndexDescriptors = List.of();
@@ -820,6 +847,11 @@ public Builder setMinimumNodeVersion(Version version) {
             return this;
         }
 
+        public Builder setMigrationScript(String migrationScript) {
+            this.migrationScript = migrationScript;
+            return this;
+        }
+
         public Builder setType(Type type) {
             this.type = type;
             return this;
@@ -866,6 +898,7 @@ public SystemIndexDescriptor build() {
                 versionMetaKey,
                 origin,
                 minimumNodeVersion,
+                migrationScript,
                 type,
                 allowedElasticProductOrigins,
                 priorSystemIndexDescriptors,

server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationInfo.java

@@ -44,6 +44,7 @@ class SystemIndexMigrationInfo implements Comparable<SystemIndexMigrationInfo> {
     private final Settings settings;
     private final String mapping;
     private final String origin;
+    private final String migrationScript;
     private final SystemIndices.Feature owningFeature;
     private final boolean allowsTemplates;
 
@@ -57,6 +58,7 @@ private SystemIndexMigrationInfo(
         Settings settings,
         String mapping,
         String origin,
+        String migrationScript,
         SystemIndices.Feature owningFeature,
         boolean allowsTemplates
     ) {
@@ -65,6 +67,7 @@ private SystemIndexMigrationInfo(
         this.settings = settings;
         this.mapping = mapping;
         this.origin = origin;
+        this.migrationScript = migrationScript;
         this.owningFeature = owningFeature;
         this.allowsTemplates = allowsTemplates;
     }
@@ -118,6 +121,10 @@ String getOrigin() {
         return origin;
     }
 
+    String getMigrationScript() {
+        return migrationScript;
+    }
+
     /**
      * By default, system indices should not be affected by user defined templates, so this
      * method should return false in almost all cases. At the moment certain Kibana indices use
@@ -217,6 +224,7 @@ static SystemIndexMigrationInfo build(
             settings,
             mapping,
             descriptor.getOrigin(),
+            descriptor.getMigrationScript(),
             feature,
             descriptor.allowsTemplates()
         );

server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrator.java

@@ -44,6 +44,7 @@
 import org.elasticsearch.index.reindex.ReindexRequest;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.persistent.AllocatedPersistentTask;
+import org.elasticsearch.script.Script;
 import org.elasticsearch.tasks.TaskId;
 
 import java.util.LinkedList;
@@ -556,6 +557,10 @@ private void reindex(SystemIndexMigrationInfo migrationInfo, ActionListener<Bulk
         reindexRequest.setSourceIndices(migrationInfo.getCurrentIndexName());
         reindexRequest.setDestIndex(migrationInfo.getNextIndexName());
         reindexRequest.setRefresh(true);
+        String migrationScript = migrationInfo.getMigrationScript();
+        if (Strings.isNullOrEmpty(migrationScript) == false) {
+            reindexRequest.setScript(Script.parse(migrationScript));
+        }
         migrationInfo.createClient(baseClient).execute(ReindexAction.INSTANCE, reindexRequest, listener);
     }