Allow indices lookup to be built lazily

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -189,7 +189,7 @@ public interface NonRestorableCustom extends Custom {
     private final String[] allClosedIndices;
     private final String[] visibleClosedIndices;
 
-    private final SortedMap<String, IndexAbstraction> indicesLookup;
+    private volatile SortedMap<String, IndexAbstraction> indicesLookup;
 
     Metadata(String clusterUUID, boolean clusterUUIDCommitted, long version, CoordinationMetadata coordinationMetadata,
              Settings transientSettings, Settings persistentSettings, DiffableStringMap hashesOfConsistentSettings,
@@ -304,6 +304,13 @@ public boolean equalsAliases(Metadata other) {
     }
 
     public SortedMap<String, IndexAbstraction> getIndicesLookup() {
+        if (indicesLookup != null) {
+            return indicesLookup;
+        }
+        synchronized (this) {
+            DataStreamMetadata dataStreamMetadata = custom(DataStreamMetadata.TYPE);
+            indicesLookup = Collections.unmodifiableSortedMap(Builder.buildIndicesLookup(dataStreamMetadata, indices));
+        }
         return indicesLookup;
     }
 
@@ -1431,6 +1438,10 @@ public Builder generateClusterUuidIfNeeded() {
         }
 
         public Metadata build() {
+            return build(true);
+        }
+
+        public Metadata build(boolean builtIndicesLookupEagerly) {
             // TODO: We should move these datastructures to IndexNameExpressionResolver, this will give the following benefits:
             // 1) The datastructures will be rebuilt only when needed. Now during serializing we rebuild these datastructures
             //    while these datastructures aren't even used.
@@ -1514,9 +1525,15 @@ public Metadata build() {
                     "were found [" + Strings.collectionToCommaDelimitedString(duplicates) + "]");
             }
 
-            SortedMap<String, IndexAbstraction> indicesLookup = Collections.unmodifiableSortedMap(buildIndicesLookup());
+            ImmutableOpenMap<String, IndexMetadata> indices = this.indices.build();
+
+            SortedMap<String, IndexAbstraction> indicesLookup;
+            if (builtIndicesLookupEagerly) {
+                indicesLookup = Collections.unmodifiableSortedMap(buildIndicesLookup(dataStreamMetadata, indices));
+            } else {
+                indicesLookup = null;
+            }
 
-            validateDataStreams(indicesLookup, (DataStreamMetadata) customs.get(DataStreamMetadata.TYPE));
 
             // build all concrete indices arrays:
             // TODO: I think we can remove these arrays. it isn't worth the effort, for operations on all indices.
@@ -1530,14 +1547,14 @@ public Metadata build() {
             String[] visibleClosedIndicesArray = visibleClosedIndices.toArray(Strings.EMPTY_ARRAY);
 
             return new Metadata(clusterUUID, clusterUUIDCommitted, version, coordinationMetadata, transientSettings, persistentSettings,
-                hashesOfConsistentSettings, indices.build(), templates.build(), customs.build(), allIndicesArray, visibleIndicesArray,
+                hashesOfConsistentSettings, indices, templates.build(), customs.build(), allIndicesArray, visibleIndicesArray,
                 allOpenIndicesArray, visibleOpenIndicesArray, allClosedIndicesArray, visibleClosedIndicesArray, indicesLookup);
         }
 
-        private SortedMap<String, IndexAbstraction> buildIndicesLookup() {
+        static SortedMap<String, IndexAbstraction> buildIndicesLookup(DataStreamMetadata dataStreamMetadata,
+                                                                      ImmutableOpenMap<String, IndexMetadata> indices) {
             SortedMap<String, IndexAbstraction> indicesLookup = new TreeMap<>();
             Map<String, DataStream> indexToDataStreamLookup = new HashMap<>();
-            DataStreamMetadata dataStreamMetadata = (DataStreamMetadata) this.customs.get(DataStreamMetadata.TYPE);
             // If there are no indices, then skip data streams. This happens only when metadata is read from disk
             if (dataStreamMetadata != null && indices.size() > 0) {
                 Map<String, List<String>> dataStreamToAliasLookup = new HashMap<>();
@@ -1579,9 +1596,7 @@ private SortedMap<String, IndexAbstraction> buildIndicesLookup() {
             }
 
             Map<String, List<IndexMetadata>> aliasToIndices = new HashMap<>();
-            for (var cursor : indices.values()) {
-                IndexMetadata indexMetadata = cursor.value;
-
+            for (var indexMetadata : indices.values()) {
                 IndexAbstraction.Index index;
                 DataStream parent = indexToDataStreamLookup.get(indexMetadata.getIndex().getName());
                 if (parent != null) {
@@ -1600,12 +1615,12 @@ private SortedMap<String, IndexAbstraction> buildIndicesLookup() {
 
                 for (var aliasCursor : indexMetadata.getAliases()) {
                     AliasMetadata aliasMetadata = aliasCursor.value;
-                    aliasToIndices.compute(aliasMetadata.getAlias(), (aliasName, indices) -> {
-                       if (indices == null) {
-                           indices = new ArrayList<>();
+                    aliasToIndices.compute(aliasMetadata.getAlias(), (aliasName, val) -> {
+                       if (val == null) {
+                           val = new ArrayList<>();
                        }
-                       indices.add(indexMetadata);
-                       return indices;
+                       val.add(indexMetadata);
+                       return val;
                     });
                 }
             }
@@ -1616,6 +1631,7 @@ private SortedMap<String, IndexAbstraction> buildIndicesLookup() {
                 assert existing == null : "duplicate for " + entry.getKey();
             }
 
+            validateDataStreams(indicesLookup, dataStreamMetadata);
             return indicesLookup;
         }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopyExecutionStateStep.java

@@ -89,7 +89,7 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
             .put(IndexMetadata.builder(targetIndexMetadata)
                 .putCustom(ILM_CUSTOM_METADATA_KEY, relevantTargetCustomData.build().asMap()));
 
-        return ClusterState.builder(clusterState).metadata(newMetadata).build();
+        return ClusterState.builder(clusterState).metadata(newMetadata.build(false)).build();
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CopySettingsStep.java

@@ -87,7 +87,7 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
             .put(IndexMetadata.builder(targetIndexMetadata)
                 .settingsVersion(targetIndexMetadata.getSettingsVersion() + 1)
                 .settings(settings));
-        return ClusterState.builder(clusterState).metadata(newMetaData).build();
+        return ClusterState.builder(clusterState).metadata(newMetaData.build(false)).build();
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateSnapshotNameStep.java

@@ -93,7 +93,8 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
         return ClusterState.builder(clusterState)
             .metadata(Metadata.builder(clusterState.getMetadata())
                 .put(IndexMetadata.builder(indexMetaData)
-                    .putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap())))
+                    .putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap()))
+                .build(false))
             .build();
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/GenerateUniqueIndexNameStep.java

@@ -90,7 +90,7 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
 
         IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexMetadata);
         indexMetadataBuilder.putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap());
-        newClusterStateBuilder.metadata(Metadata.builder(clusterState.getMetadata()).put(indexMetadataBuilder));
+        newClusterStateBuilder.metadata(Metadata.builder(clusterState.getMetadata()).put(indexMetadataBuilder).build(false));
         return newClusterStateBuilder.build();
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/InitializePolicyContextStep.java

@@ -68,7 +68,7 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
         indexMetadataBuilder.putCustom(ILM_CUSTOM_METADATA_KEY, newCustomData.build().asMap());
 
         newClusterStateBuilder.metadata(
-            Metadata.builder(clusterState.getMetadata()).put(indexMetadataBuilder)
+            Metadata.builder(clusterState.getMetadata()).put(indexMetadataBuilder).build(false)
         );
         return newClusterStateBuilder.build();
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PhaseCacheManagement.java

@@ -51,7 +51,7 @@ public static ClusterState refreshPhaseDefinition(final ClusterState state, fina
         final IndexMetadata idxMeta = state.metadata().index(index);
         Metadata.Builder metadataBuilder = Metadata.builder(state.metadata());
         refreshPhaseDefinition(metadataBuilder, idxMeta, updatedPolicy);
-        return ClusterState.builder(state).metadata(metadataBuilder).build();
+        return ClusterState.builder(state).metadata(metadataBuilder.build(false)).build();
     }
 
     /**
@@ -109,7 +109,7 @@ public static ClusterState updateIndicesForPolicy(final ClusterState state, fina
                                                       final LifecyclePolicyMetadata newPolicy, XPackLicenseState licenseState) {
         Metadata.Builder mb = Metadata.builder(state.metadata());
         if (updateIndicesForPolicy(mb, state, xContentRegistry, client, oldPolicy, newPolicy, licenseState)) {
-            return ClusterState.builder(state).metadata(mb).build();
+            return ClusterState.builder(state).metadata(mb.build(false)).build();
         }
         return state;
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStep.java

@@ -97,7 +97,7 @@ public ClusterState performAction(Index index, ClusterState clusterState) {
 
         Metadata.Builder newMetaData = Metadata.builder(clusterState.getMetadata())
             .put(dataStream.getDataStream().replaceBackingIndex(index, targetIndexMetadata.getIndex()));
-        return ClusterState.builder(clusterState).metadata(newMetaData).build();
+        return ClusterState.builder(clusterState).metadata(newMetaData.build(false)).build();
     }
 
     @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UpdateRolloverLifecycleDateStep.java

@@ -70,7 +70,7 @@ public ClusterState performAction(Index index, ClusterState currentState) {
         IndexMetadata.Builder newIndexMetadata = IndexMetadata.builder(indexMetadata);
         newIndexMetadata.putCustom(ILM_CUSTOM_METADATA_KEY, newLifecycleState.build().asMap());
         return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata())
-            .put(newIndexMetadata)).build();
+            .put(newIndexMetadata).build(false)).build();
     }
 
     private static String getRolloverTarget(Index index, ClusterState currentState) {

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleRunner.java

@@ -67,6 +67,8 @@ class IndexLifecycleRunner {
                 builder.failure(task, e);
             }
         }
+        // Trigger indices lookup creation and related validation
+        state.metadata().getIndicesLookup();
         return builder.build(state);
     };
 

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleTransition.java

@@ -344,7 +344,8 @@ public static ClusterState.Builder newClusterStateWithLifecycleState(Index index
         ClusterState.Builder newClusterStateBuilder = ClusterState.builder(clusterState);
         newClusterStateBuilder.metadata(Metadata.builder(clusterState.getMetadata())
             .put(IndexMetadata.builder(clusterState.getMetadata().index(index))
-                .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.asMap())));
+                .putCustom(ILM_CUSTOM_METADATA_KEY, lifecycleState.asMap()))
+            .build(false));
         return newClusterStateBuilder;
     }