Reuse previous indices lookup in IndexMetadataUpdater#applyChanges(...)

The IndexMetadataUpdater#applyChanges(...) method builds a new metadata
instance, but only primary term or insync allocations may be updated.
No new indices, aliases or data streams are added, so re-building indices
lookup is not necessary.

In clusters with many indices the cost of building indices lookup is
non-neglectable and should be avoided in this case.

Closes elastic#78980

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -1020,7 +1020,11 @@ public static Builder builder() {
     }
 
     public static Builder builder(Metadata metadata) {
-        return new Builder(metadata);
+        return builder(metadata, false);
+    }
+
+    public static Builder builder(Metadata metadata, boolean reuseIndicesLookup) {
+        return new Builder(metadata, reuseIndicesLookup);
     }
 
     public static class Builder {
@@ -1037,16 +1041,18 @@ public static class Builder {
         private final ImmutableOpenMap.Builder<String, IndexMetadata> indices;
         private final ImmutableOpenMap.Builder<String, IndexTemplateMetadata> templates;
         private final ImmutableOpenMap.Builder<String, Custom> customs;
+        private final SortedMap<String, IndexAbstraction> previousIndicesLookup;
 
         public Builder() {
             clusterUUID = UNKNOWN_CLUSTER_UUID;
             indices = ImmutableOpenMap.builder();
             templates = ImmutableOpenMap.builder();
             customs = ImmutableOpenMap.builder();
             indexGraveyard(IndexGraveyard.builder().build()); // create new empty index graveyard to initialize
+            previousIndicesLookup = null;
         }
 
-        public Builder(Metadata metadata) {
+        Builder(Metadata metadata, boolean reuseIndicesLookup) {
             this.clusterUUID = metadata.clusterUUID;
             this.clusterUUIDCommitted = metadata.clusterUUIDCommitted;
             this.coordinationMetadata = metadata.coordinationMetadata;
@@ -1057,6 +1063,11 @@ public Builder(Metadata metadata) {
             this.indices = ImmutableOpenMap.builder(metadata.indices);
             this.templates = ImmutableOpenMap.builder(metadata.templates);
             this.customs = ImmutableOpenMap.builder(metadata.customs);
+            if (reuseIndicesLookup) {
+                previousIndicesLookup = metadata.getIndicesLookup();
+            } else {
+                previousIndicesLookup = null;
+            }
         }
 
         public Builder put(IndexMetadata.Builder indexMetadataBuilder) {
@@ -1539,8 +1550,14 @@ public Metadata build(boolean builtIndicesLookupEagerly) {
 
             SortedMap<String, IndexAbstraction> indicesLookup;
             if (builtIndicesLookupEagerly) {
-                indicesLookup = Collections.unmodifiableSortedMap(buildIndicesLookup(dataStreamMetadata, indices));
+                if (previousIndicesLookup != null) {
+                    assert previousIndicesLookup.equals(buildIndicesLookup(dataStreamMetadata, indices));
+                    indicesLookup = previousIndicesLookup;
+                } else {
+                    indicesLookup = Collections.unmodifiableSortedMap(buildIndicesLookup(dataStreamMetadata, indices));
+                }
             } else {
+                assert previousIndicesLookup != null : "previous indicesLookup provided, but builtIndicesLookupEagerly=false";
                 indicesLookup = null;
             }
 

server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexMetadataUpdater.java

@@ -114,7 +114,7 @@ public Metadata applyChanges(Metadata oldMetadata, RoutingTable newRoutingTable)
 
             if (indexMetadataBuilder != null) {
                 if (metadataBuilder == null) {
-                    metadataBuilder = Metadata.builder(oldMetadata);
+                    metadataBuilder = Metadata.builder(oldMetadata, true);
                 }
                 metadataBuilder.put(indexMetadataBuilder);
             }

server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java

@@ -354,7 +354,7 @@ private static ClusterState createNonInitializedState(final int numNodes, final
     private static ClusterState nextState(final ClusterState previousState, List<TestCustomMetadata> customMetadataList) {
         final ClusterState.Builder builder = ClusterState.builder(previousState);
         builder.stateUUID(UUIDs.randomBase64UUID());
-        Metadata.Builder metadataBuilder = new Metadata.Builder(previousState.metadata());
+        Metadata.Builder metadataBuilder = Metadata.builder(previousState.metadata());
         for (ObjectObjectCursor<String, Metadata.Custom> customMetadata : previousState.metadata().customs()) {
             if (customMetadata.value instanceof TestCustomMetadata) {
                 metadataBuilder.removeCustom(customMetadata.key);

server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java

@@ -334,7 +334,7 @@ private ClusterState createRecoveryStateAndInitializeAllocations(
         final TestSnapshotsInfoService snapshotsInfoService
         ) {
         DiscoveryNode node1 = newNode("node1");
-        Metadata.Builder metadataBuilder = new Metadata.Builder(metadata);
+        Metadata.Builder metadataBuilder = Metadata.builder(metadata);
         RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
         Snapshot snapshot = new Snapshot("repo", new SnapshotId("snap", "randomId"));
         Set<String> snapshotIndices = new HashSet<>();

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigAutoUpdaterTests.java

@@ -179,7 +179,7 @@ public void testIsAbleToRun() {
         final ClusterState clusterState = csBuilder.build();
         assertThat(updater.isAbleToRun(clusterState), is(true));
 
-        metadata = new Metadata.Builder(clusterState.metadata());
+        metadata = Metadata.builder(clusterState.metadata());
         routingTable = new RoutingTable.Builder(clusterState.routingTable());
         if (randomBoolean()) {
             routingTable.remove(MlConfigIndex.indexName());

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/task/AbstractJobPersistentTasksExecutorTests.java

@@ -55,7 +55,7 @@ public void testVerifyIndicesPrimaryShardsAreActive() {
             MlMetaIndex.indexName(),
             MlConfigIndex.indexName()).size());
 
-        metadata = new Metadata.Builder(cs.metadata());
+        metadata = Metadata.builder(cs.metadata());
         routingTable = new RoutingTable.Builder(cs.routingTable());
         String indexToRemove = randomFrom(resolver.concreteIndexNames(cs, IndicesOptions.lenientExpandOpen(),
             ".ml-anomalies-shared",

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java

@@ -83,7 +83,7 @@ public static RoutingTable buildIndexRoutingTable(Index index) {
      */
     public static Metadata addAliasToMetadata(Metadata metadata, String indexName) {
         AliasMetadata aliasMetadata = AliasMetadata.newAliasMetadataBuilder(SECURITY_MAIN_ALIAS).build();
-        Metadata.Builder metadataBuilder = new Metadata.Builder(metadata);
+        Metadata.Builder metadataBuilder = Metadata.builder(metadata);
         IndexMetadata indexMetadata = metadata.index(indexName);
         metadataBuilder.put(IndexMetadata.builder(indexMetadata).putAlias(aliasMetadata));
         return metadataBuilder.build();

x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java

@@ -194,7 +194,7 @@ public void testVerifyIndicesPrimaryShardsAreActive() {
             TransformPersistentTasksExecutor.verifyIndicesPrimaryShardsAreActive(cs, TestIndexNameExpressionResolver.newInstance()).size()
         );
 
-        metadata = new Metadata.Builder(cs.metadata());
+        metadata = Metadata.builder(cs.metadata());
         routingTable = new RoutingTable.Builder(cs.routingTable());
         String indexToRemove = TransformInternalIndexConstants.LATEST_INDEX_NAME;
         if (randomBoolean()) {