From 1d5bb9dfcf1f3ed90c2bf993f17bb5d3814c464b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 10 Jan 2023 19:16:59 -0800 Subject: [PATCH] GetIndexMappings index pattern support (#265) (#276) Signed-off-by: Petar Dzepina --- .../mapper/MapperService.java | 52 ++-- .../mapper/MappingsTraverser.java | 80 ++++- .../TransportGetIndexMappingsAction.java | 12 +- .../SecurityAnalyticsRestTestCase.java | 25 +- .../mapper/MapperRestApiIT.java | 288 ++++++++++++------ .../mapper/MappingsTraverserTests.java | 87 +++++- 6 files changed, 414 insertions(+), 130 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index bd2c67ad0..d27494942 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -243,6 +243,27 @@ public void onFailure(Exception e) { } public void getMappingAction(String indexName, ActionListener actionListener) { + try { + // We are returning mappings view for only 1 index: writeIndex or latest from the pattern + resolveConcreteIndex(indexName, new ActionListener<>() { + @Override + public void onResponse(String concreteIndex) { + doGetMappingAction(concreteIndex, actionListener); + } + + @Override + public void onFailure(Exception e) { + actionListener.onFailure(e); + } + }); + + + } catch (IOException e) { + throw SecurityAnalyticsException.wrap(e); + } + } + + public void doGetMappingAction(String indexName, ActionListener actionListener) { GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(indexName); indicesClient.getMappings(getMappingsRequest, new ActionListener<>() { @Override @@ -287,36 +308,13 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { // Traverse mappings and do copy with excluded type=alias properties MappingsTraverser mappingsTraverser = new MappingsTraverser(mappingMetadata); - // Resulting properties after filtering - Map filteredProperties = new HashMap<>(); - - mappingsTraverser.addListener(new MappingsTraverser.MappingsTraverserListener() { - @Override - public void onLeafVisited(MappingsTraverser.Node node) { - // Skip everything except aliases we found - if (appliedAliases.contains(node.currentPath) == false) { - return; - } - MappingsTraverser.Node n = node; - while (n.parent != null) { - n = n.parent; - } - if (n == null) { - n = node; - } - filteredProperties.put(n.getNodeName(), n.getProperties()); - } + // Resulting mapping after filtering + Map filteredMapping = mappingsTraverser.traverseAndCopyWithFilter(appliedAliases); + - @Override - public void onError(String error) { - throw new IllegalArgumentException(""); - } - }); - mappingsTraverser.traverse(); // Construct filtered mappings and return them as result ImmutableOpenMap.Builder outIndexMappings = ImmutableOpenMap.builder(); - Map outRootProperties = Map.of(PROPERTIES, filteredProperties); - Map root = Map.of(org.opensearch.index.mapper.MapperService.SINGLE_MAPPING_NAME, outRootProperties); + Map root = Map.of(org.opensearch.index.mapper.MapperService.SINGLE_MAPPING_NAME, filteredMapping); MappingMetadata outMappingMetadata = new MappingMetadata(org.opensearch.index.mapper.MapperService.SINGLE_MAPPING_NAME, root); outIndexMappings.put(indexName, outMappingMetadata); diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MappingsTraverser.java b/src/main/java/org/opensearch/securityanalytics/mapper/MappingsTraverser.java index 19000101d..88d183480 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MappingsTraverser.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MappingsTraverser.java @@ -5,6 +5,9 @@ package org.opensearch.securityanalytics.mapper; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.ListIterator; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.common.xcontent.DeprecationHandler; @@ -154,7 +157,7 @@ public void traverse() { try { Map rootProperties = (Map) this.mappingsMap.get(PROPERTIES); - rootProperties.forEach((k, v) -> nodeStack.push(new Node(Map.of(k, v), ""))); + rootProperties.forEach((k, v) -> nodeStack.push(new Node(Map.of(k, v), null, rootProperties, "", ""))); while (nodeStack.size() > 0) { Node node = nodeStack.pop(); @@ -182,7 +185,7 @@ public void traverse() { node.currentPath.length() > 0 ? node.currentPath + "." + currentNodeName : currentNodeName; - nodeStack.push(new Node(Map.of(k, v), node, currentPath)); + nodeStack.push(new Node(Map.of(k, v), node, children, currentNodeName, currentPath)); }); } } @@ -211,6 +214,70 @@ private boolean shouldSkipNode(Map properties) { return false; } + public Map traverseAndCopyWithFilter(List nodePathsToCopy) { + + Map outRoot = new LinkedHashMap<>(Map.of(PROPERTIES, new LinkedHashMap())); + this.addListener(new MappingsTraverserListener() { + @Override + public void onLeafVisited(Node node) { + if (nodePathsToCopy.contains(node.currentPath) == false) { + return; + } + // Collect all nodes from root to this leaf. + List nodes = new ArrayList<>(); + Node n = node; + nodes.add(n); + while (n.parent != null) { + n = n.parent; + nodes.add(n); + } + // Iterate from root node up to this leaf and copy node in each iteration to "out" tree + ListIterator nodesIterator = nodes.listIterator(nodes.size()); + Map outNode = outRoot; + while (nodesIterator.hasPrevious()) { + Node currentNode = nodesIterator.previous(); + + appendNode(currentNode, outNode, !nodesIterator.hasPrevious()); + // Move to next output node + outNode = (Map) ((Map) outNode.get(PROPERTIES)).get(currentNode.getNodeName()); + } + } + + @Override + public void onError(String error) { + throw new IllegalArgumentException(""); + } + }); + traverse(); + return outRoot; + } + + /** + * Appends src node to dst node's properties + * @param srcNode source node + * @param dstNode destination node where source node is appended + * @param isSourceLeaf flag which indicated if source node is leaf + */ + private void appendNode(Node srcNode, Map dstNode, boolean isSourceLeaf) { + Map existingProps = (Map) ((Map) dstNode.get(PROPERTIES)).get(srcNode.getNodeName()); + if (existingProps == null) { + Map srcNodeProps = srcNode.getProperties(); + Map newProps = isSourceLeaf ? + srcNodeProps : + new LinkedHashMap(); + // In case of type="nested" node, we need to copy that type field too, beside properties + if (srcNodeProps.containsKey(TYPE) && srcNodeProps.get(TYPE).equals(NESTED)) { + ((Map) dstNode.get(PROPERTIES)).put(srcNode.getNodeName(), new LinkedHashMap(Map.of(PROPERTIES, newProps, TYPE, NESTED))); + } else { + // Append src node to dst node's properties + ((Map) dstNode.get(PROPERTIES)).put( + srcNode.getNodeName(), + isSourceLeaf ? newProps : new LinkedHashMap(Map.of(PROPERTIES, newProps)) + ); + } + } + } + /** * Traverses index mappings tree and copies it into 1-level tree with flatten nodes. (level1.level2.level3) Listeners are notified when leaves are visited, * just like during {@link #traverse()} call. @@ -254,10 +321,16 @@ private void notifyLeafVisited(Node node) { ); } + public Map getMappingsMap() { + return mappingsMap; + } + static class Node { Map node; Node parent; Map properties; + Map parentProperties; + String parentKey; String currentPath; String name; @@ -265,9 +338,10 @@ public Node(Map node, String currentPath) { this.node = node; this.currentPath = currentPath; } - public Node(Map node, Node parent, String currentPath) { + public Node(Map node, Node parent, Map parentProperties, String parentKey, String currentPath) { this.node = node; this.parent = parent; + this.parentProperties = parentProperties; this.currentPath = currentPath; } /** diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetIndexMappingsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetIndexMappingsAction.java index b638e87b5..b8b9110d8 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetIndexMappingsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetIndexMappingsAction.java @@ -45,17 +45,7 @@ public TransportGetIndexMappingsAction( @Override protected void doExecute(Task task, GetIndexMappingsRequest request, ActionListener actionListener) { this.threadPool.getThreadContext().stashContext(); - IndexMetadata index = clusterService.state().metadata().index(request.getIndexName()); - if (index == null) { - actionListener.onFailure( - SecurityAnalyticsException.wrap( - new OpenSearchStatusException( - "Could not find index [" + request.getIndexName() + "]", RestStatus.NOT_FOUND - ) - ) - ); - return; - } + mapperService.getMappingAction(request.getIndexName(), actionListener); } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 58d8329df..dfc691030 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -73,6 +73,7 @@ import java.util.stream.Collectors; import static org.opensearch.action.admin.indices.create.CreateIndexRequest.MAPPINGS; +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.MAPPER_BASE_URI; import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule; import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; @@ -1405,7 +1406,7 @@ protected void createComposableIndexTemplate(String templateName, List i assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } - protected Map getIndexMappingsFlat(String indexName) throws IOException { + protected Map getIndexMappingsAPIFlat(String indexName) throws IOException { Request request = new Request("GET", indexName + "/_mapping"); Response response = client().performRequest(request); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); @@ -1416,8 +1417,28 @@ protected Map getIndexMappingsFlat(String indexName) throws IOEx return (Map) flatMappings.get("properties"); } + protected Map getIndexMappingsAPI(String indexName) throws IOException { + Request request = new Request("GET", indexName + "/_mapping"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = (Map) responseAsMap(response).values().iterator().next(); + return (Map) respMap.get("mappings"); + } + + protected Map getIndexMappingsSAFlat(String indexName) throws IOException { + Request request = new Request("GET", MAPPER_BASE_URI + "?index_name=" + indexName); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = (Map) responseAsMap(response).values().iterator().next(); + + MappingsTraverser mappingsTraverser = new MappingsTraverser((Map) respMap.get("mappings"), Set.of()); + Map flatMappings = mappingsTraverser.traverseAndCopyAsFlat(); + return (Map) flatMappings.get("properties"); + } + + protected void createMappingsAPI(String indexName, String topicName) throws IOException { - Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + Request request = new Request("POST", MAPPER_BASE_URI); // both req params and req body are supported request.setJsonEntity( "{ \"index_name\":\"" + indexName + "\"," + diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 7dc077075..50aa9bb24 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -4,6 +4,7 @@ */ package org.opensearch.securityanalytics.mapper; +import java.util.ArrayList; import java.util.Collections; import java.util.Optional; import java.util.Set; @@ -294,88 +295,6 @@ public void testGetMappingsViewSuccess() throws IOException { assertEquals(2, unmappedFieldAliases.size()); } - public void testCreateMappings_withIndexPattern_indexTemplate_createAndUpdate_success() throws IOException { - String indexName1 = "test_index_1"; - String indexName2 = "test_index_2"; - String indexName3 = "test_index_3"; - String indexName4 = "test_index_4"; - - String indexPattern = "test_index*"; - - createIndex(indexName1, Settings.EMPTY, null); - createIndex(indexName2, Settings.EMPTY, null); - - client().performRequest(new Request("POST", "_refresh")); - - // Insert sample doc - String sampleDoc1 = "{" + - " \"netflow.destination_transport_port\":1234," + - " \"netflow.destination_ipv4_address\":\"10.53.111.14\"" + - "}"; - - indexDoc(indexName1, "1", sampleDoc1); - indexDoc(indexName2, "1", sampleDoc1); - - client().performRequest(new Request("POST", "_refresh")); - - // Execute CreateMappingsAction to add alias mapping for index - createMappingsAPI(indexPattern, "netflow"); - - // Verify that index template is up - createIndex(indexName3, Settings.EMPTY, null); - - // Execute CreateMappingsAction to add alias mapping for index - Request request = new Request("GET", indexName3 + "/_mapping"); - Response response = client().performRequest(request); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); - Map respMap = (Map) responseAsMap(response).get(indexName3); - - MappingsTraverser mappingsTraverser = new MappingsTraverser((Map) respMap.get("mappings"), Set.of()); - Map flatMappings = mappingsTraverser.traverseAndCopyAsFlat(); - // Verify mappings - Map props = (Map) flatMappings.get("properties"); - assertEquals(4, props.size()); - assertTrue(props.containsKey("destination.ip")); - assertTrue(props.containsKey("destination.port")); - assertTrue(props.containsKey("netflow.destination_transport_port")); - assertTrue(props.containsKey("netflow.destination_ipv4_address")); - - String sampleDoc2 = "{" + - " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + - " \"netflow.destination_transport_port\":1234," + - " \"netflow.destination_ipv4_address\":\"10.53.111.14\"," + - " \"netflow.source_transport_port\":4444" + - "}"; - - indexDoc(indexName3, "1", sampleDoc2); - - // Execute CreateMappingsAction to add alias mapping for index - createMappingsAPI(indexPattern, "netflow"); - - // Verify that index template is updated - createIndex(indexName4, Settings.EMPTY, null); - - // Execute CreateMappingsAction to add alias mapping for index - request = new Request("GET", indexName4 + "/_mapping"); - response = client().performRequest(request); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); - respMap = (Map) responseAsMap(response).get(indexName4); - - mappingsTraverser = new MappingsTraverser((Map) respMap.get("mappings"), Set.of()); - flatMappings = mappingsTraverser.traverseAndCopyAsFlat(); - // Verify mappings - props = (Map) flatMappings.get("properties"); - assertEquals(8, props.size()); - assertTrue(props.containsKey("source.ip")); - assertTrue(props.containsKey("destination.ip")); - assertTrue(props.containsKey("source.port")); - assertTrue(props.containsKey("destination.port")); - assertTrue(props.containsKey("netflow.source_transport_port")); - assertTrue(props.containsKey("netflow.source_ipv4_address")); - assertTrue(props.containsKey("netflow.destination_transport_port")); - assertTrue(props.containsKey("netflow.destination_ipv4_address")); - } - public void testCreateMappings_withDatastream_success() throws IOException { String datastream = "test_datastream"; @@ -391,7 +310,7 @@ public void testCreateMappings_withDatastream_success() throws IOException { createMappingsAPI(datastream, "netflow"); // Verify mappings - Map props = getIndexMappingsFlat(datastream); + Map props = getIndexMappingsAPIFlat(datastream); assertEquals(5, props.size()); assertTrue(props.containsKey("@timestamp")); assertTrue(props.containsKey("netflow.destination_transport_port")); @@ -418,7 +337,7 @@ public void testCreateMappings_withDatastream_success() throws IOException { String writeIndex = getDatastreamWriteIndex(datastream); // Verify mappings - props = getIndexMappingsFlat(writeIndex); + props = getIndexMappingsAPIFlat(writeIndex); assertEquals(9, props.size()); assertTrue(props.containsKey("@timestamp")); assertTrue(props.containsKey("netflow.source_ipv4_address")); @@ -430,6 +349,13 @@ public void testCreateMappings_withDatastream_success() throws IOException { assertTrue(props.containsKey("source.ip")); assertTrue(props.containsKey("source.port")); + // Get applied mappings + props = getIndexMappingsSAFlat(datastream); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("destination.port")); + assertTrue(props.containsKey("source.ip")); + assertTrue(props.containsKey("source.port")); + deleteDatastreamAPI(datastream); } @@ -467,13 +393,20 @@ public void testCreateMappings_withIndexPattern_existing_indexTemplate_update_su createIndex(indexName2, Settings.EMPTY, null); // Verify that template applied mappings - Map props = getIndexMappingsFlat(indexName2); + Map props = getIndexMappingsAPIFlat(indexName2); assertEquals(4, props.size()); assertTrue(props.containsKey("netflow.destination_transport_port")); assertTrue(props.containsKey("netflow.destination_ipv4_address")); assertTrue(props.containsKey("destination.ip")); assertTrue(props.containsKey("destination.port")); + // Verify our GetIndexMappings -- applied mappings + props = getIndexMappingsSAFlat(indexPattern); + assertEquals(2, props.size()); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("destination.port")); + + // Insert doc to index to add additional fields to mapping String sampleDoc = "{" + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + @@ -489,7 +422,7 @@ public void testCreateMappings_withIndexPattern_existing_indexTemplate_update_su createIndex(indexName3, Settings.EMPTY, null); // Verify mappings - props = getIndexMappingsFlat(indexName3); + props = getIndexMappingsAPIFlat(indexName3); assertEquals(8, props.size()); assertTrue(props.containsKey("source.ip")); assertTrue(props.containsKey("destination.ip")); @@ -499,6 +432,14 @@ public void testCreateMappings_withIndexPattern_existing_indexTemplate_update_su assertTrue(props.containsKey("netflow.source_ipv4_address")); assertTrue(props.containsKey("netflow.destination_transport_port")); assertTrue(props.containsKey("netflow.destination_ipv4_address")); + + // Verify our GetIndexMappings -- applied mappings + props = getIndexMappingsSAFlat(indexPattern); + assertEquals(4, props.size()); + assertTrue(props.containsKey("source.ip")); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("source.port")); + assertTrue(props.containsKey("destination.port")); } public void testCreateMappings_withIndexPattern_differentMappings_success() throws IOException { @@ -530,6 +471,96 @@ public void testCreateMappings_withIndexPattern_differentMappings_success() thro createMappingsAPI(indexPattern, "netflow"); } + public void testCreateMappings_withIndexPattern_indexTemplate_createAndUpdate_success() throws IOException { + String indexName1 = "test_index_1"; + String indexName2 = "test_index_2"; + String indexName3 = "test_index_3"; + String indexName4 = "test_index_4"; + + String indexPattern = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample doc + String sampleDoc1 = "{" + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.destination_ipv4_address\":\"10.53.111.14\"" + + "}"; + + indexDoc(indexName1, "1", sampleDoc1); + indexDoc(indexName2, "1", sampleDoc1); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction to add alias mapping for index + createMappingsAPI(indexPattern, "netflow"); + + // Verify that index template is up + createIndex(indexName3, Settings.EMPTY, null); + + // Execute CreateMappingsAction to add alias mapping for index + Request request = new Request("GET", indexName3 + "/_mapping"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = (Map) responseAsMap(response).get(indexName3); + + MappingsTraverser mappingsTraverser = new MappingsTraverser((Map) respMap.get("mappings"), Set.of()); + Map flatMappings = mappingsTraverser.traverseAndCopyAsFlat(); + // Verify mappings + Map props = (Map) flatMappings.get("properties"); + assertEquals(4, props.size()); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("destination.port")); + assertTrue(props.containsKey("netflow.destination_transport_port")); + assertTrue(props.containsKey("netflow.destination_ipv4_address")); + + String sampleDoc2 = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.destination_ipv4_address\":\"10.53.111.14\"," + + " \"netflow.source_transport_port\":4444" + + "}"; + + indexDoc(indexName3, "1", sampleDoc2); + + // Execute CreateMappingsAction to add alias mapping for index + createMappingsAPI(indexPattern, "netflow"); + + // Verify that index template is updated + createIndex(indexName4, Settings.EMPTY, null); + + // Execute CreateMappingsAction to add alias mapping for index + request = new Request("GET", indexName4 + "/_mapping"); + response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + respMap = (Map) responseAsMap(response).get(indexName4); + + mappingsTraverser = new MappingsTraverser((Map) respMap.get("mappings"), Set.of()); + flatMappings = mappingsTraverser.traverseAndCopyAsFlat(); + // Verify mappings + props = (Map) flatMappings.get("properties"); + assertEquals(8, props.size()); + assertTrue(props.containsKey("source.ip")); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("source.port")); + assertTrue(props.containsKey("destination.port")); + assertTrue(props.containsKey("netflow.source_transport_port")); + assertTrue(props.containsKey("netflow.source_ipv4_address")); + assertTrue(props.containsKey("netflow.destination_transport_port")); + assertTrue(props.containsKey("netflow.destination_ipv4_address")); + + // Verify applied mappings + props = getIndexMappingsSAFlat(indexName4); + assertEquals(4, props.size()); + assertTrue(props.containsKey("source.ip")); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("source.port")); + assertTrue(props.containsKey("destination.port")); + } + public void testCreateMappings_withIndexPattern_oneNoMatches_success() throws IOException { String indexName1 = "test_index_1"; String indexName2 = "test_index_2"; @@ -1121,6 +1152,7 @@ public void testCreateCloudTrailMapping() throws IOException { response = client().performRequest(new Request("POST", "_refresh")); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } + public void testCreateDNSMapping() throws IOException{ String INDEX_NAME = "test_create_cloudtrail_mapping_index"; @@ -1172,4 +1204,88 @@ public void testCreateDNSMapping() throws IOException{ assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } + + public void testTraverseAndCopy() { + + try { + String indexName = "my_test_index"; + + String indexMappingJSON = "" + + " \"properties\": {" + + " \"netflow.event_data.SourceAddress\": {" + + " \"type\": \"ip\"" + + " }," + + " \"type\": {" + + " \"type\": \"integer\"" + + " }," + + " \"netflow.event_data.DestinationPort\": {" + + " \"type\": \"integer\"" + + " }," + + " \"netflow.event.stop\": {" + + " \"type\": \"integer\"" + + " }," + + " \"netflow.event.start\": {" + + " \"type\": \"long\"" + + " }," + + " \"plain1\": {" + + " \"type\": \"integer\"" + + " }," + + " \"user\":{" + + " \"type\":\"nested\"," + + " \"properties\":{" + + " \"first\":{" + + " \"type\":\"long\"" + + " }," + + " \"last\":{" + + " \"type\":\"text\"," + + " \"fields\":{" + + " \"keyword\":{" + + " \"type\":\"keyword\"," + + " \"ignore_above\":256" + + " }" + + " }" + + " }" + + " }" + + " }" + + "}"; + + createIndex(indexName, Settings.EMPTY, indexMappingJSON); + + Map mappings = getIndexMappingsAPI(indexName); + + MappingsTraverser mappingsTraverser; + + mappingsTraverser = new MappingsTraverser(mappings, Set.of()); + + // Copy specific paths from mappings + Map filteredMappings = mappingsTraverser.traverseAndCopyWithFilter( + List.of("netflow.event_data.SourceAddress", "netflow.event.stop", "plain1", "user.first", "user.last") + ); + + // Now traverse filtered mapppings to confirm only copied paths are present + List paths = new ArrayList<>(); + mappingsTraverser = new MappingsTraverser(filteredMappings, Set.of()); + mappingsTraverser.addListener(new MappingsTraverser.MappingsTraverserListener() { + @Override + public void onLeafVisited(MappingsTraverser.Node node) { + paths.add(node.currentPath); + } + + @Override + public void onError(String error) { + fail("Failed traversing valid mappings"); + } + }); + mappingsTraverser.traverse(); + assertEquals(5, paths.size()); + assertTrue(paths.contains("user.first")); + assertTrue(paths.contains("user.last")); + assertTrue(paths.contains("plain1")); + assertTrue(paths.contains("netflow.event.stop")); + assertTrue(paths.contains("netflow.event_data.SourceAddress")); + + } catch (IOException e) { + fail("Error instantiating MappingsTraverser with JSON string as mappings"); + } + } } diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MappingsTraverserTests.java b/src/test/java/org/opensearch/securityanalytics/mapper/MappingsTraverserTests.java index b51575f65..19ebb9e12 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MappingsTraverserTests.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MappingsTraverserTests.java @@ -225,7 +225,7 @@ public void onError(String error) { assertEquals("netflow.event_data.SourcePort", paths.get(0)); } - public void testTraverseAndCopyValidNestedMappingsWithTypeFilter() { + public void testTraverseAndCopyAsFlatValidNestedMappingsWithTypeFilter() { String indexMappingJSON = "{" + " \"properties\": {" + " \"netflow.event_data.SourceAddress\": {" + @@ -309,4 +309,89 @@ public void onError(String error) { fail("Error instantiating MappingsTraverser with JSON string as mappings"); } } + + public void testTraverseAndCopyValidNestedMappings() { + String indexMappingJSON = "{" + + " \"properties\": {" + + " \"netflow.event_data.SourceAddress\": {" + + " \"type\": \"ip\"" + + " }," + + " \"netflow.event_data.DestinationPort\": {" + + " \"type\": \"integer\"" + + " }," + + " \"netflow.event_data.DestAddress\": {" + + " \"type\": \"ip\"" + + " }," + + " \"netflow.event_data.SourcePort\": {" + + " \"type\": \"integer\"" + + " }," + + " \"netflow.event.stop\": {" + + " \"type\": \"integer\"" + + " }," + + " \"dns.event.stop\": {" + + " \"type\": \"integer\"" + + " }," + + " \"ipx.event.stop\": {" + + " \"type\": \"integer\"" + + " }," + + " \"plain1\": {" + + " \"type\": \"integer\"" + + " }," + + " \"user\":{" + + " \"type\":\"nested\"," + + " \"properties\":{" + + " \"first\":{" + + " \"type\":\"text\"," + + " \"fields\":{" + + " \"keyword\":{" + + " \"type\":\"keyword\"," + + " \"ignore_above\":256" + + "}" + + "}" + + "}," + + " \"last\":{" + + "\"type\":\"long\"," + + "\"fields\":{" + + " \"keyword\":{" + + " \"type\":\"keyword\"," + + " \"ignore_above\":256" + + "}" + + "}" + + "}" + + "}" + + "}" + + " }" + + "}"; + + MappingsTraverser mappingsTraverser; + try { + + mappingsTraverser = new MappingsTraverser(indexMappingJSON, Set.of("ip")); + + // Copy mappings while excluding type=ip + Map filteredMappings = mappingsTraverser.traverseAndCopyWithFilter(List.of("user.first", "user.last")); + + // Now traverse filtered mapppings to confirm type=ip is not present + List paths = new ArrayList<>(); + mappingsTraverser = new MappingsTraverser(filteredMappings, Set.of("integer")); + mappingsTraverser.addListener(new MappingsTraverser.MappingsTraverserListener() { + @Override + public void onLeafVisited(MappingsTraverser.Node node) { + paths.add(node.currentPath); + } + + @Override + public void onError(String error) { + fail("Failed traversing valid mappings"); + } + }); + mappingsTraverser.traverse(); + assertEquals(2, paths.size()); + assertEquals("user.last", paths.get(0)); + assertEquals("user.first", paths.get(1)); + + } catch (IOException e) { + fail("Error instantiating MappingsTraverser with JSON string as mappings"); + } + } }