GetMappingsView API - index pattern/alias/datastream support (#245)

Signed-off-by: Petar Dzepina <[email protected]>
(cherry picked from commit c062cf4)

src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java

@@ -121,7 +121,7 @@ public Collection<Object> createComponents(Client client,
                                                Supplier<RepositoriesService> repositoriesServiceSupplier) {
         detectorIndices = new DetectorIndices(client.admin(), clusterService, threadPool);
         ruleTopicIndices = new RuleTopicIndices(client, clusterService);
-        mapperService = new MapperService(client.admin().indices());
+        mapperService = new MapperService(client.admin().indices(), clusterService);
         ruleIndices = new RuleIndices(client, clusterService, threadPool);
         return List.of(detectorIndices, ruleTopicIndices, ruleIndices, mapperService);
     }

src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java

@@ -12,13 +12,17 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.OpenSearchStatusException;
 import org.opensearch.action.ActionListener;
+import org.opensearch.action.admin.indices.get.GetIndexRequest;
+import org.opensearch.action.admin.indices.get.GetIndexResponse;
 import org.opensearch.action.admin.indices.mapping.get.GetMappingsRequest;
 import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.opensearch.action.support.GroupedActionListener;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.IndicesAdminClient;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.MappingMetadata;
+import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.collect.ImmutableOpenMap;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.rest.RestStatus;
@@ -31,6 +35,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 import org.opensearch.securityanalytics.action.GetMappingsViewResponse;
+import org.opensearch.securityanalytics.util.IndexUtils;
 import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
 
 
@@ -40,13 +45,15 @@
 public class MapperService {
 
     private static final Logger log = LogManager.getLogger(MapperService.class);
+    private ClusterService clusterService;
 
     IndicesAdminClient indicesClient;
 
     public MapperService() {}
 
-    public MapperService(IndicesAdminClient indicesClient) {
+    public MapperService(IndicesAdminClient indicesClient, ClusterService clusterService) {
         this.indicesClient = indicesClient;
+        this.clusterService = clusterService;
     }
 
     void setIndicesAdminClient(IndicesAdminClient client) {
@@ -93,7 +100,8 @@ public void onResponse(Collection<AcknowledgedResponse> response) {
             public void onFailure(Exception e) {
                 actionListener.onFailure(
                     new SecurityAnalyticsException(
-                        "Failed applying mappings to index", RestStatus.INTERNAL_SERVER_ERROR, e)
+                        "Failed applying mappings to index", RestStatus.INTERNAL_SERVER_ERROR, e
+                    )
                 );
             }
         }, numOfIndices);
@@ -280,7 +288,34 @@ public void getMappingsViewAction(
             String mapperTopic,
             ActionListener<GetMappingsViewResponse> actionListener
     ) {
-        GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(indexName);
+        try {
+            // We are returning mappings view for only 1 index: writeIndex or latest from the pattern
+            resolveConcreteIndex(indexName, new ActionListener<>() {
+                @Override
+                public void onResponse(String concreteIndex) {
+                    doGetMappingsView(mapperTopic, actionListener, concreteIndex);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    actionListener.onFailure(e);
+                }
+            });
+
+
+        } catch (IOException e) {
+            throw SecurityAnalyticsException.wrap(e);
+        }
+    }
+
+    /**
+     * Constructs Mappings View of index
+     * @param mapperTopic Mapper Topic describing set of alias mappings
+     * @param actionListener Action Listener
+     * @param concreteIndex Concrete Index name for which we're computing Mappings View
+     */
+    private void doGetMappingsView(String mapperTopic, ActionListener<GetMappingsViewResponse> actionListener, String concreteIndex) {
+        GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(concreteIndex);
         indicesClient.getMappings(getMappingsRequest, new ActionListener<>() {
             @Override
             public void onResponse(GetMappingsResponse getMappingsResponse) {
@@ -333,4 +368,45 @@ public void onFailure(Exception e) {
             }
         });
     }
+
+    /**
+     * Given index name, resolves it to single concrete index, depending on what initial <code>indexName</code> is.
+     * In case of Datastream or Alias, WriteIndex would be returned. In case of index pattern, newest index by creation date would be returned.
+     * @param indexName Datastream, Alias, index patter or concrete index
+     * @param actionListener Action Listener
+     * @throws IOException
+     */
+    private void resolveConcreteIndex(String indexName, ActionListener<String> actionListener) throws IOException {
+
+        indicesClient.getIndex((new GetIndexRequest()).indices(indexName), new ActionListener<>() {
+            @Override
+            public void onResponse(GetIndexResponse getIndexResponse) {
+                String[] indices = getIndexResponse.indices();
+                if (indices.length == 0) {
+                    actionListener.onFailure(
+                            SecurityAnalyticsException.wrap(
+                                    new IllegalArgumentException("Invalid index name: [" + indexName + "]")
+                            )
+                    );
+                } else if (indices.length == 1) {
+                    actionListener.onResponse(indices[0]);
+                } else if (indices.length > 1) {
+                    String writeIndex = IndexUtils.getWriteIndex(indexName, MapperService.this.clusterService.state());
+                    if (writeIndex != null) {
+                        actionListener.onResponse(writeIndex);
+                    } else {
+                        actionListener.onResponse(
+                            IndexUtils.getNewestIndexByCreationDate(indices, MapperService.this.clusterService.state())
+                        );
+                    }
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                actionListener.onFailure(e);
+            }
+        });
+
+    }
 }

src/main/java/org/opensearch/securityanalytics/transport/TransportGetMappingsViewAction.java

@@ -47,17 +47,6 @@ public TransportGetMappingsViewAction(
     @Override
     protected void doExecute(Task task, GetMappingsViewRequest request, ActionListener<GetMappingsViewResponse> actionListener) {
         this.threadPool.getThreadContext().stashContext();
-        IndexMetadata index = clusterService.state().metadata().index(request.getIndexName());
-        if (index == null) {
-            actionListener.onFailure(
-                    SecurityAnalyticsException.wrap(
-                            new OpenSearchStatusException(
-                                    "Could not find index [" + request.getIndexName() + "]", RestStatus.NOT_FOUND
-                            )
-                    )
-            );
-            return;
-        }
-        mapperService.getMappingsViewAction(request.getIndexName(), request.getRuleTopic(), actionListener);
+        this.mapperService.getMappingsViewAction(request.getIndexName(), request.getRuleTopic(), actionListener);
     }
 }

src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java

@@ -4,11 +4,13 @@
  */
 package org.opensearch.securityanalytics.util;
 
+import java.util.SortedMap;
 import org.opensearch.action.ActionListener;
 import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.IndicesAdminClient;
 import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.IndexAbstraction;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
 import org.opensearch.common.xcontent.NamedXContentRegistry;
@@ -105,4 +107,39 @@ public static void updateIndexMapping(
             }
         }
     }
+
+    public static boolean isDataStream(String name, ClusterState clusterState) {
+        return clusterState.getMetadata().dataStreams().containsKey(name);
+    }
+    public static boolean isAlias(String indexName, ClusterState clusterState) {
+        return clusterState.getMetadata().hasAlias(indexName);
+    }
+    public static String getWriteIndex(String indexName, ClusterState clusterState) {
+        if(isAlias(indexName, clusterState) || isDataStream(indexName, clusterState)) {
+            IndexMetadata metadata = clusterState.getMetadata()
+                    .getIndicesLookup()
+                    .get(indexName).getWriteIndex();
+            if (metadata != null) {
+                return metadata.getIndex().getName();
+            }
+        }
+        return null;
+    }
+
+    public static String getNewestIndexByCreationDate(String[] concreteIndices, ClusterState clusterState) {
+        final SortedMap<String, IndexAbstraction> lookup = clusterState.getMetadata().getIndicesLookup();
+        long maxCreationDate = Long.MIN_VALUE;
+        String newestIndex = null;
+        for (String indexName : concreteIndices) {
+            IndexAbstraction index = lookup.get(indexName);
+            IndexMetadata indexMetadata = clusterState.getMetadata().index(indexName);
+            if(index != null && index.getType() == IndexAbstraction.Type.CONCRETE_INDEX) {
+                if (indexMetadata.getCreationDate() > maxCreationDate) {
+                    maxCreationDate = indexMetadata.getCreationDate();
+                    newestIndex = indexName;
+                }
+            }
+        }
+        return newestIndex;
+    }
 }

src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java

@@ -1342,4 +1342,9 @@ private Map<String, Object> getIndexSettingsAPI(String index) throws IOException
         Map<String, Object> respMap = asMap(resp);
         return respMap;
     }
+
+    protected void doRollover(String datastreamName) throws IOException {
+        Response response = makeRequest(client(), "POST", datastreamName + "/_rollover", Collections.emptyMap(), null);
+        assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
+    }
 }