Skip to content

Commit

Permalink
GetIndexMappings index pattern support (opensearch-project#265)
Browse files Browse the repository at this point in the history 
Signed-off-by: Petar Dzepina <[email protected]>
  • Loading branch information
@petardz @eirsep
petardz authored and eirsep committed Apr 3, 2023
1 parent ca56e77 commit 493f60c
Show file tree
Hide file tree
Showing 6 changed files with 414 additions and 130 deletions.
52 changes: 25 additions & 27 deletions src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,27 @@ public void onFailure(Exception e) {
}

public void getMappingAction(String indexName, ActionListener<GetIndexMappingsResponse> actionListener) {
try {
// We are returning mappings view for only 1 index: writeIndex or latest from the pattern
resolveConcreteIndex(indexName, new ActionListener<>() {
@Override
public void onResponse(String concreteIndex) {
doGetMappingAction(concreteIndex, actionListener);
}

@Override
public void onFailure(Exception e) {
actionListener.onFailure(e);
}
});


} catch (IOException e) {
throw SecurityAnalyticsException.wrap(e);
}
}

public void doGetMappingAction(String indexName, ActionListener<GetIndexMappingsResponse> actionListener) {
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(indexName);
indicesClient.getMappings(getMappingsRequest, new ActionListener<>() {
@Override
Expand Down Expand Up @@ -287,36 +308,13 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {

// Traverse mappings and do copy with excluded type=alias properties
MappingsTraverser mappingsTraverser = new MappingsTraverser(mappingMetadata);
// Resulting properties after filtering
Map<String, Object> filteredProperties = new HashMap<>();

mappingsTraverser.addListener(new MappingsTraverser.MappingsTraverserListener() {
@Override
public void onLeafVisited(MappingsTraverser.Node node) {
// Skip everything except aliases we found
if (appliedAliases.contains(node.currentPath) == false) {
return;
}
MappingsTraverser.Node n = node;
while (n.parent != null) {
n = n.parent;
}
if (n == null) {
n = node;
}
filteredProperties.put(n.getNodeName(), n.getProperties());
}
// Resulting mapping after filtering
Map<String, Object> filteredMapping = mappingsTraverser.traverseAndCopyWithFilter(appliedAliases);


@Override
public void onError(String error) {
throw new IllegalArgumentException("");
}
});
mappingsTraverser.traverse();
// Construct filtered mappings and return them as result
ImmutableOpenMap.Builder<String, MappingMetadata> outIndexMappings = ImmutableOpenMap.builder();
Map<String, Object> outRootProperties = Map.of(PROPERTIES, filteredProperties);
Map<String, Object> root = Map.of(org.opensearch.index.mapper.MapperService.SINGLE_MAPPING_NAME, outRootProperties);
Map<String, Object> root = Map.of(org.opensearch.index.mapper.MapperService.SINGLE_MAPPING_NAME, filteredMapping);
MappingMetadata outMappingMetadata = new MappingMetadata(org.opensearch.index.mapper.MapperService.SINGLE_MAPPING_NAME, root);
outIndexMappings.put(indexName, outMappingMetadata);

Expand Down
80 changes: 77 additions & 3 deletions src/main/java/org/opensearch/securityanalytics/mapper/MappingsTraverser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@

package org.opensearch.securityanalytics.mapper;

import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.ListIterator;
import org.apache.commons.lang3.tuple.Pair;
import org.opensearch.cluster.metadata.MappingMetadata;
import org.opensearch.common.xcontent.DeprecationHandler;
Expand Down Expand Up @@ -154,7 +157,7 @@ public void traverse() {
try {

Map<String, Object> rootProperties = (Map<String, Object>) this.mappingsMap.get(PROPERTIES);
rootProperties.forEach((k, v) -> nodeStack.push(new Node(Map.of(k, v), "")));
rootProperties.forEach((k, v) -> nodeStack.push(new Node(Map.of(k, v), null, rootProperties, "", "")));

while (nodeStack.size() > 0) {
Node node = nodeStack.pop();
Expand Down Expand Up @@ -182,7 +185,7 @@ public void traverse() {
node.currentPath.length() > 0 ?
node.currentPath + "." + currentNodeName :
currentNodeName;
nodeStack.push(new Node(Map.of(k, v), node, currentPath));
nodeStack.push(new Node(Map.of(k, v), node, children, currentNodeName, currentPath));
});
}
}
Expand Down Expand Up @@ -211,6 +214,70 @@ private boolean shouldSkipNode(Map<String, Object> properties) {
return false;
}

public Map<String, Object> traverseAndCopyWithFilter(List<String> nodePathsToCopy) {

Map<String, Object> outRoot = new LinkedHashMap<>(Map.of(PROPERTIES, new LinkedHashMap()));
this.addListener(new MappingsTraverserListener() {
@Override
public void onLeafVisited(Node node) {
if (nodePathsToCopy.contains(node.currentPath) == false) {
return;
}
// Collect all nodes from root to this leaf.
List<Node> nodes = new ArrayList<>();
Node n = node;
nodes.add(n);
while (n.parent != null) {
n = n.parent;
nodes.add(n);
}
// Iterate from root node up to this leaf and copy node in each iteration to "out" tree
ListIterator<Node> nodesIterator = nodes.listIterator(nodes.size());
Map<String, Object> outNode = outRoot;
while (nodesIterator.hasPrevious()) {
Node currentNode = nodesIterator.previous();

appendNode(currentNode, outNode, !nodesIterator.hasPrevious());
// Move to next output node
outNode = (Map<String, Object>) ((Map<?, ?>) outNode.get(PROPERTIES)).get(currentNode.getNodeName());
}
}

@Override
public void onError(String error) {
throw new IllegalArgumentException("");
}
});
traverse();
return outRoot;
}

/**
* Appends src node to dst node's properties
* @param srcNode source node
* @param dstNode destination node where source node is appended
* @param isSourceLeaf flag which indicated if source node is leaf
*/
private void appendNode(Node srcNode, Map<String, Object> dstNode, boolean isSourceLeaf) {
Map<String, Object> existingProps = (Map<String, Object>) ((Map) dstNode.get(PROPERTIES)).get(srcNode.getNodeName());
if (existingProps == null) {
Map<String, Object> srcNodeProps = srcNode.getProperties();
Map<String, Object> newProps = isSourceLeaf ?
srcNodeProps :
new LinkedHashMap();
// In case of type="nested" node, we need to copy that type field too, beside properties
if (srcNodeProps.containsKey(TYPE) && srcNodeProps.get(TYPE).equals(NESTED)) {
((Map) dstNode.get(PROPERTIES)).put(srcNode.getNodeName(), new LinkedHashMap(Map.of(PROPERTIES, newProps, TYPE, NESTED)));
} else {
// Append src node to dst node's properties
((Map) dstNode.get(PROPERTIES)).put(
srcNode.getNodeName(),
isSourceLeaf ? newProps : new LinkedHashMap(Map.of(PROPERTIES, newProps))
);
}
}
}

/**
* Traverses index mappings tree and copies it into 1-level tree with flatten nodes. (level1.level2.level3) Listeners are notified when leaves are visited,
* just like during {@link #traverse()} call.
Expand Down Expand Up @@ -254,20 +321,27 @@ private void notifyLeafVisited(Node node) {
);
}

public Map<String, Object> getMappingsMap() {
return mappingsMap;
}

static class Node {
Map<String, Object> node;
Node parent;
Map<String, Object> properties;
Map<String, Object> parentProperties;
String parentKey;
String currentPath;
String name;

public Node(Map<String, Object> node, String currentPath) {
this.node = node;
this.currentPath = currentPath;
}
public Node(Map<String, Object> node, Node parent, String currentPath) {
public Node(Map<String, Object> node, Node parent, Map<String, Object> parentProperties, String parentKey, String currentPath) {
this.node = node;
this.parent = parent;
this.parentProperties = parentProperties;
this.currentPath = currentPath;
}
/**
Expand Down
12 changes: 1 addition & 11 deletions ...main/java/org/opensearch/securityanalytics/transport/TransportGetIndexMappingsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,17 +45,7 @@ public TransportGetIndexMappingsAction(
@Override
protected void doExecute(Task task, GetIndexMappingsRequest request, ActionListener<GetIndexMappingsResponse> actionListener) {
this.threadPool.getThreadContext().stashContext();
IndexMetadata index = clusterService.state().metadata().index(request.getIndexName());
if (index == null) {
actionListener.onFailure(
SecurityAnalyticsException.wrap(
new OpenSearchStatusException(
"Could not find index [" + request.getIndexName() + "]", RestStatus.NOT_FOUND
)
)
);
return;
}

mapperService.getMappingAction(request.getIndexName(), actionListener);
}
}
25 changes: 23 additions & 2 deletions src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
import java.util.stream.Collectors;

import static org.opensearch.action.admin.indices.create.CreateIndexRequest.MAPPINGS;
import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.MAPPER_BASE_URI;
import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule;
import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule;
import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping;
Expand Down Expand Up @@ -1405,7 +1406,7 @@ protected void createComposableIndexTemplate(String templateName, List<String> i
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
}

protected Map<String, Object> getIndexMappingsFlat(String indexName) throws IOException {
protected Map<String, Object> getIndexMappingsAPIFlat(String indexName) throws IOException {
Request request = new Request("GET", indexName + "/_mapping");
Response response = client().performRequest(request);
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
Expand All @@ -1416,8 +1417,28 @@ protected Map<String, Object> getIndexMappingsFlat(String indexName) throws IOEx
return (Map<String, Object>) flatMappings.get("properties");
}

protected Map<String, Object> getIndexMappingsAPI(String indexName) throws IOException {
Request request = new Request("GET", indexName + "/_mapping");
Response response = client().performRequest(request);
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
Map<String, Object> respMap = (Map<String, Object>) responseAsMap(response).values().iterator().next();
return (Map<String, Object>) respMap.get("mappings");
}

protected Map<String, Object> getIndexMappingsSAFlat(String indexName) throws IOException {
Request request = new Request("GET", MAPPER_BASE_URI + "?index_name=" + indexName);
Response response = client().performRequest(request);
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
Map<String, Object> respMap = (Map<String, Object>) responseAsMap(response).values().iterator().next();

MappingsTraverser mappingsTraverser = new MappingsTraverser((Map<String, Object>) respMap.get("mappings"), Set.of());
Map<String, Object> flatMappings = mappingsTraverser.traverseAndCopyAsFlat();
return (Map<String, Object>) flatMappings.get("properties");
}


protected void createMappingsAPI(String indexName, String topicName) throws IOException {
Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
Request request = new Request("POST", MAPPER_BASE_URI);
// both req params and req body are supported
request.setJsonEntity(
"{ \"index_name\":\"" + indexName + "\"," +
Expand Down
Loading

0 comments on commit 493f60c

Please sign in to comment.