From 18899c412c2f3287e4e3b9ba942edfdb93fde291 Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Tue, 5 Dec 2023 17:09:35 -0800 Subject: [PATCH] add field based rules support in correlation engine (#737) * add field based rules support in correlation engine Signed-off-by: Subhobrata Dey * add field based rules support in correlation engine Signed-off-by: Subhobrata Dey --------- Signed-off-by: Subhobrata Dey --- .../SecurityAnalyticsPlugin.java | 1 + .../correlation/JoinEngine.java | 110 ++-- .../model/CorrelationQuery.java | 30 +- .../model/CorrelationRule.java | 20 +- .../settings/SecurityAnalyticsSettings.java | 9 + .../TransportCorrelateFindingAction.java | 6 +- .../resources/mappings/finding_mapping.json | 5 +- .../securityanalytics/TestHelpers.java | 457 ++++++++++++++- .../CorrelationEngineRestApiIT.java | 530 +++++++++++++++++- .../resthandler/DetectorMonitorRestApiIT.java | 80 +++ .../aggregation/AggregationBackendTests.java | 35 ++ 11 files changed, 1226 insertions(+), 57 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index bdf67ee50..e7fe43106 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -295,6 +295,7 @@ public List> getSettings() { SecurityAnalyticsSettings.CORRELATION_HISTORY_RETENTION_PERIOD, SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING, SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW, + SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS, SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA, SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE, SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL, diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java b/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java index 8c986fd3a..cfff7da26 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java @@ -4,7 +4,8 @@ */ package org.opensearch.securityanalytics.correlation; -import kotlin.Pair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.lang3.tuple.Triple; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.search.join.ScoreMode; @@ -49,7 +50,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.Predicate; import java.util.stream.Collectors; @@ -63,6 +63,8 @@ public class JoinEngine { private volatile long corrTimeWindow; + private volatile boolean enableAutoCorrelations; + private final TransportCorrelateFindingAction.AsyncCorrelateFindingAction correlateFindingAction; private final LogTypeService logTypeService; @@ -71,18 +73,23 @@ public class JoinEngine { public JoinEngine(Client client, PublishFindingsRequest request, NamedXContentRegistry xContentRegistry, long corrTimeWindow, TransportCorrelateFindingAction.AsyncCorrelateFindingAction correlateFindingAction, - LogTypeService logTypeService) { + LogTypeService logTypeService, boolean enableAutoCorrelations) { this.client = client; this.request = request; this.xContentRegistry = xContentRegistry; this.corrTimeWindow = corrTimeWindow; this.correlateFindingAction = correlateFindingAction; this.logTypeService = logTypeService; + this.enableAutoCorrelations = enableAutoCorrelations; } public void onSearchDetectorResponse(Detector detector, Finding finding) { try { - generateAutoCorrelations(detector, finding); + if (enableAutoCorrelations) { + generateAutoCorrelations(detector, finding); + } else { + onAutoCorrelations(detector, finding, Map.of()); + } } catch (IOException ex) { correlateFindingAction.onFailures(ex); } @@ -265,6 +272,7 @@ public void onFailure(Exception e) { private void getValidDocuments(String detectorType, List indices, List correlationRules, List relatedDocIds, Map> autoCorrelations) { MultiSearchRequest mSearchRequest = new MultiSearchRequest(); List validCorrelationRules = new ArrayList<>(); + List validFields = new ArrayList<>(); for (CorrelationRule rule: correlationRules) { Optional query = rule.getCorrelationQueries().stream() @@ -272,12 +280,19 @@ private void getValidDocuments(String detectorType, List indices, List indices, List indices, List filteredCorrelationRules = new ArrayList<>(); + List filteredCorrelationRules = new ArrayList<>(); int idx = 0; for (MultiSearchResponse.Item response : responses) { @@ -304,14 +320,17 @@ public void onResponse(MultiSearchResponse items) { } if (response.getResponse().getHits().getTotalHits().value > 0L) { - filteredCorrelationRules.add(validCorrelationRules.get(idx)); + filteredCorrelationRules.add(new FilteredCorrelationRule(validCorrelationRules.get(idx), + response.getResponse().getHits().getHits(), validFields.get(idx))); } ++idx; } Map> categoryToQueriesMap = new HashMap<>(); - for (CorrelationRule rule: filteredCorrelationRules) { - List queries = rule.getCorrelationQueries(); + Map categoryToTimeWindowMap = new HashMap<>(); + for (FilteredCorrelationRule rule: filteredCorrelationRules) { + List queries = rule.correlationRule.getCorrelationQueries(); + Long timeWindow = rule.correlationRule.getCorrTimeWindow(); for (CorrelationQuery query: queries) { List correlationQueries; @@ -320,12 +339,36 @@ public void onResponse(MultiSearchResponse items) { } else { correlationQueries = new ArrayList<>(); } - correlationQueries.add(query); + if (categoryToTimeWindowMap.containsKey(query.getCategory())) { + categoryToTimeWindowMap.put(query.getCategory(), Math.max(timeWindow, categoryToTimeWindowMap.get(query.getCategory()))); + } else { + categoryToTimeWindowMap.put(query.getCategory(), timeWindow); + } + + if (query.getField() == null) { + correlationQueries.add(query); + } else { + SearchHit[] hits = rule.filteredDocs; + StringBuilder qb = new StringBuilder(query.getField()).append(":("); + for (int i = 0; i < hits.length; ++i) { + String value = hits[i].field(rule.field).getValue(); + qb.append(value); + if (i < hits.length-1) { + qb.append(" OR "); + } else { + qb.append(")"); + } + } + if (query.getQuery() != null) { + qb.append(" AND ").append(query.getQuery()); + } + correlationQueries.add(new CorrelationQuery(query.getIndex(), qb.toString(), query.getCategory(), null)); + } categoryToQueriesMap.put(query.getCategory(), correlationQueries); } } - searchFindingsByTimestamp(detectorType, categoryToQueriesMap, - filteredCorrelationRules.stream().map(CorrelationRule::getId).collect(Collectors.toList()), + searchFindingsByTimestamp(detectorType, categoryToQueriesMap, categoryToTimeWindowMap, + filteredCorrelationRules.stream().map(it -> it.correlationRule).map(CorrelationRule::getId).collect(Collectors.toList()), autoCorrelations ); } @@ -348,15 +391,15 @@ public void onFailure(Exception e) { * this method searches for parent findings given the log category & correlation time window & collects all related docs * for them. */ - private void searchFindingsByTimestamp(String detectorType, Map> categoryToQueriesMap, List correlationRules, Map> autoCorrelations) { + private void searchFindingsByTimestamp(String detectorType, Map> categoryToQueriesMap, Map categoryToTimeWindowMap, List correlationRules, Map> autoCorrelations) { long findingTimestamp = request.getFinding().getTimestamp().toEpochMilli(); MultiSearchRequest mSearchRequest = new MultiSearchRequest(); List>> categoryToQueriesPairs = new ArrayList<>(); for (Map.Entry> categoryToQueries: categoryToQueriesMap.entrySet()) { RangeQueryBuilder queryBuilder = QueryBuilders.rangeQuery("timestamp") - .gte(findingTimestamp - corrTimeWindow) - .lte(findingTimestamp + corrTimeWindow); + .gte(findingTimestamp - categoryToTimeWindowMap.get(categoryToQueries.getKey())) + .lte(findingTimestamp + categoryToTimeWindowMap.get(categoryToQueries.getKey())); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(queryBuilder); @@ -368,7 +411,7 @@ private void searchFindingsByTimestamp(String detectorType, Map(categoryToQueries.getKey(), categoryToQueries.getValue())); + categoryToQueriesPairs.add(Pair.of(categoryToQueries.getKey(), categoryToQueries.getValue())); } if (!mSearchRequest.requests().isEmpty()) { @@ -392,17 +435,17 @@ public void onResponse(MultiSearchResponse items) { .map(Object::toString).collect(Collectors.toList())); } - List correlationQueries = categoryToQueriesPairs.get(idx).getSecond(); + List correlationQueries = categoryToQueriesPairs.get(idx).getValue(); List indices = correlationQueries.stream().map(CorrelationQuery::getIndex).collect(Collectors.toList()); List queries = correlationQueries.stream().map(CorrelationQuery::getQuery).collect(Collectors.toList()); - relatedDocsMap.put(categoryToQueriesPairs.get(idx).getFirst(), + relatedDocsMap.put(categoryToQueriesPairs.get(idx).getKey(), new DocSearchCriteria( indices, queries, relatedDocIds)); ++idx; } - searchDocsWithFilterKeys(detectorType, relatedDocsMap, correlationRules, autoCorrelations); + searchDocsWithFilterKeys(detectorType, relatedDocsMap, categoryToTimeWindowMap, correlationRules, autoCorrelations); } @Override @@ -422,7 +465,7 @@ public void onFailure(Exception e) { /** * Given the related docs from parent findings, this method filters only those related docs which match parent join criteria. */ - private void searchDocsWithFilterKeys(String detectorType, Map relatedDocsMap, List correlationRules, Map> autoCorrelations) { + private void searchDocsWithFilterKeys(String detectorType, Map relatedDocsMap, Map categoryToTimeWindowMap, List correlationRules, Map> autoCorrelations) { MultiSearchRequest mSearchRequest = new MultiSearchRequest(); List categories = new ArrayList<>(); @@ -433,6 +476,7 @@ private void searchDocsWithFilterKeys(String detectorType, Map> filteredRelatedDocIds, List correlationRules, Map> autoCorrelations) { + private void getCorrelatedFindings(String detectorType, Map> filteredRelatedDocIds, Map categoryToTimeWindowMap, List correlationRules, Map> autoCorrelations) { long findingTimestamp = request.getFinding().getTimestamp().toEpochMilli(); MultiSearchRequest mSearchRequest = new MultiSearchRequest(); List categories = new ArrayList<>(); @@ -499,8 +543,8 @@ private void getCorrelatedFindings(String detectorType, Map for (Map.Entry> relatedDocIds: filteredRelatedDocIds.entrySet()) { BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() .filter(QueryBuilders.rangeQuery("timestamp") - .gte(findingTimestamp - corrTimeWindow) - .lte(findingTimestamp + corrTimeWindow)) + .gte(findingTimestamp - categoryToTimeWindowMap.get(relatedDocIds.getKey())) + .lte(findingTimestamp + categoryToTimeWindowMap.get(relatedDocIds.getKey()))) .must(QueryBuilders.termsQuery("correlated_doc_ids", relatedDocIds.getValue())); if (relatedDocIds.getKey().equals(detectorType)) { @@ -586,15 +630,15 @@ public DocSearchCriteria(List indices, List queries, List correlationQueries; - public CorrelationRule(String id, Long version, String name, List correlationQueries) { + private Long corrTimeWindow; + + public CorrelationRule(String id, Long version, String name, List correlationQueries, Long corrTimeWindow) { this.id = id != null ? id : NO_ID; this.version = version != null ? version : NO_VERSION; this.name = name; this.correlationQueries = correlationQueries; + this.corrTimeWindow = corrTimeWindow != null? corrTimeWindow: 300000L; } public CorrelationRule(StreamInput sin) throws IOException { - this(sin.readString(), sin.readLong(), sin.readString(), sin.readList(CorrelationQuery::readFrom)); + this(sin.readString(), sin.readLong(), sin.readString(), sin.readList(CorrelationQuery::readFrom), sin.readLong()); } @Override @@ -57,6 +61,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws CorrelationQuery[] correlationQueries = new CorrelationQuery[] {}; correlationQueries = this.correlationQueries.toArray(correlationQueries); builder.field(CORRELATION_QUERIES, correlationQueries); + builder.field(CORRELATION_TIME_WINDOW, corrTimeWindow); return builder.endObject(); } @@ -69,6 +74,7 @@ public void writeTo(StreamOutput out) throws IOException { for (CorrelationQuery query : correlationQueries) { query.writeTo(out); } + out.writeLong(corrTimeWindow); } public static CorrelationRule parse(XContentParser xcp, String id, Long version) throws IOException { @@ -81,6 +87,7 @@ public static CorrelationRule parse(XContentParser xcp, String id, Long version) String name = null; List correlationQueries = new ArrayList<>(); + Long corrTimeWindow = null; XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { @@ -98,11 +105,14 @@ public static CorrelationRule parse(XContentParser xcp, String id, Long version) correlationQueries.add(query); } break; + case CORRELATION_TIME_WINDOW: + corrTimeWindow = xcp.longValue(); + break; default: xcp.skipChildren(); } } - return new CorrelationRule(id, version, name, correlationQueries); + return new CorrelationRule(id, version, name, correlationQueries, corrTimeWindow); } public static CorrelationRule readFrom(StreamInput sin) throws IOException { @@ -137,6 +147,10 @@ public List getCorrelationQueries() { return correlationQueries; } + public Long getCorrTimeWindow() { + return corrTimeWindow; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index b55548069..fefe7c288 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -138,6 +138,15 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + /** + * Setting which enables auto correlations + */ + public static final Setting ENABLE_AUTO_CORRELATIONS = Setting.boolSetting( + "plugins.security_analytics.auto_correlations_enabled", + false, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting DEFAULT_MAPPING_SCHEMA = Setting.simpleString( "plugins.security_analytics.mappings.default_schema", "ecs", diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java index db2b0287d..b7a906159 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java @@ -96,6 +96,8 @@ public class TransportCorrelateFindingAction extends HandledTransportAction indexTimeout = it); this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW, it -> corrTimeWindow = it.getMillis()); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS, it -> enableAutoCorrelation = it); this.setupTimestamp = System.currentTimeMillis(); } @@ -220,7 +224,7 @@ public class AsyncCorrelateFindingAction { this.response =new AtomicReference<>(); - this.joinEngine = new JoinEngine(client, request, xContentRegistry, corrTimeWindow, this, logTypeService); + this.joinEngine = new JoinEngine(client, request, xContentRegistry, corrTimeWindow, this, logTypeService, enableAutoCorrelation); this.vectorEmbeddingsEngine = new VectorEmbeddingsEngine(client, indexTimeout, corrTimeWindow, this); } diff --git a/src/main/resources/mappings/finding_mapping.json b/src/main/resources/mappings/finding_mapping.json index 18faa22f6..fcb2cc152 100644 --- a/src/main/resources/mappings/finding_mapping.json +++ b/src/main/resources/mappings/finding_mapping.json @@ -1,7 +1,7 @@ { "dynamic": "strict", "_meta" : { - "schema_version": 3 + "schema_version": 4 }, "properties": { "schema_version": { @@ -46,6 +46,9 @@ "type" : "keyword" } } + }, + "fields": { + "type": "text" } } }, diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 6f87900c5..a12b5dae4 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -225,9 +225,9 @@ public static CorrelationRule randomCorrelationRule(String name) { name = name.isEmpty()? ">": name; return new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, name, List.of( - new CorrelationQuery("vpc_flow1", "dstaddr:192.168.1.*", "network"), - new CorrelationQuery("ad_logs1", "azure.platformlogs.result_type:50126", "ad_ldap") - )); + new CorrelationQuery("vpc_flow1", "dstaddr:192.168.1.*", "network", null), + new CorrelationQuery("ad_logs1", "azure.platformlogs.result_type:50126", "ad_ldap", null) + ), 300000L); } public static String randomRule() { @@ -259,6 +259,59 @@ public static String randomRule() { "level: high"; } + public static String randomNullRule() { + return "title: null field\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firew all to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " EventID: 22\n" + + " RecordNumber: null\n" + + " condition: selection\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + + public static String randomCloudtrailRuleForCorrelations(String value) { + return "id: 5f92fff9-82e2-48ab-8fc1-8b133556a551\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: AWS User Created\n" + + "description: AWS User Created\n" + + "tags:\n" + + " - attack.test1\n" + + "falsepositives:\n" + + " - Legit User Account Administration\n" + + "level: high\n" + + "date: 2022/01/01\n" + + "status: experimental\n" + + "references:\n" + + " - 'https://github.com/RhinoSecurityLabs/AWS-IAM-Privilege-Escalation'\n" + + "author: toffeebr33k\n" + + "detection:\n" + + " condition: selection_source\n" + + " selection_source:\n" + + " EventName:\n" + + " - " + value; + } + public static String randomRuleForMappingView(String field) { return "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + @@ -837,6 +890,29 @@ public static String randomAggregationRule(String aggFunction, String signAndVa return String.format(Locale.ROOT, rule, opCode, aggFunction, signAndValue); } + public static String randomCloudtrailAggrRule() { + return "id: c64c5175-5189-431b-a55e-6d9882158250\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: Accounts created and deleted within 24h\n" + + "description: Flag suspicious activity of accounts created and deleted within 24h\n" + + "date: 2021/09/23\n" + + "tags:\n" + + " - attack.exfiltration\n" + + "falsepositives: [ ]\n" + + "level: high\n" + + "status: test\n" + + "references: [ ]\n" + + "author: Sashank\n" + + "detection:\n" + + " selection:\n" + + " EventName:\n" + + " - CREATED\n" + + " - DELETED\n" + + " timeframe: 24h\n" + + " condition: selection | count(*) by AccountName >= 2"; + } + public static String windowsIndexMapping() { return "\"properties\": {\n" + " \"@timestamp\": {\"type\":\"date\"},\n" + @@ -850,7 +926,10 @@ public static String windowsIndexMapping() { " \"type\": \"text\"\n" + " },\n" + " \"AccountName\": {\n" + - " \"type\": \"text\"\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"EventName\": {\n" + + " \"type\": \"keyword\"\n" + " },\n" + " \"AccountType\": {\n" + " \"type\": \"text\",\n" + @@ -991,7 +1070,7 @@ public static String windowsIndexMapping() { " \"type\": \"date\"\n" + " },\n" + " \"EventType\": {\n" + - " \"type\": \"integer\"\n" + + " \"type\": \"keyword\"\n" + " },\n" + " \"ExecutionProcessID\": {\n" + " \"type\": \"long\"\n" + @@ -1614,6 +1693,7 @@ public static String randomDoc() { "\"Severity\":\"INFO\",\n" + "\"EventID\":22,\n" + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"SourceIp\":\"1.2.3.4\",\n" + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + "\"Version\":5,\n" + "\"TaskValue\":22,\n" + @@ -1642,6 +1722,13 @@ public static String randomDoc() { "}"; } + public static String randomCloudtrailAggrDoc(String eventType, String accountId) { + return "{\n" + + " \"AccountName\": \"" + accountId + "\",\n" + + " \"EventType\": \"" + eventType + "\"\n" + + "}"; + } + public static String randomVpcFlowDoc() { return "{\n" + " \"version\": 1,\n" + @@ -1664,6 +1751,60 @@ public static String randomAdLdapDoc() { "}"; } + public static String randomCloudtrailDoc(String user, String event) { + return "{\n" + + " \"eventVersion\": \"1.08\",\n" + + " \"userIdentity\": {\n" + + " \"type\": \"IAMUser\",\n" + + " \"principalId\": \"AIDA6ON6E4XEGITEXAMPLE\",\n" + + " \"arn\": \"arn:aws:iam::888888888888:user/Mary\",\n" + + " \"accountId\": \"888888888888\",\n" + + " \"accessKeyId\": \"AKIAIOSFODNN7EXAMPLE\",\n" + + " \"userName\": \"Mary\",\n" + + " \"sessionContext\": {\n" + + " \"sessionIssuer\": {},\n" + + " \"webIdFederationData\": {},\n" + + " \"attributes\": {\n" + + " \"creationDate\": \"2023-07-19T21:11:57Z\",\n" + + " \"mfaAuthenticated\": \"false\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventTime\": \"2023-07-19T21:25:09Z\",\n" + + " \"eventSource\": \"iam.amazonaws.com\",\n" + + " \"EventName\": \"" + event + "\",\n" + + " \"awsRegion\": \"us-east-1\",\n" + + " \"sourceIPAddress\": \"192.0.2.0\",\n" + + " \"AccountName\": \"" + user + "\",\n" + + " \"userAgent\": \"aws-cli/2.13.5 Python/3.11.4 Linux/4.14.255-314-253.539.amzn2.x86_64 exec-env/CloudShell exe/x86_64.amzn.2 prompt/off command/iam.create-user\",\n" + + " \"requestParameters\": {\n" + + " \"userName\": \"" + user + "\"\n" + + " },\n" + + " \"responseElements\": {\n" + + " \"user\": {\n" + + " \"path\": \"/\",\n" + + " \"arn\": \"arn:aws:iam::888888888888:user/Richard\",\n" + + " \"userId\": \"AIDA6ON6E4XEP7EXAMPLE\",\n" + + " \"createDate\": \"Jul 19, 2023 9:25:09 PM\",\n" + + " \"userName\": \"Richard\"\n" + + " }\n" + + " },\n" + + " \"requestID\": \"2d528c76-329e-410b-9516-EXAMPLE565dc\",\n" + + " \"eventID\": \"ba0801a1-87ec-4d26-be87-EXAMPLE75bbb\",\n" + + " \"readOnly\": false,\n" + + " \"eventType\": \"AwsApiCall\",\n" + + " \"managementEvent\": true,\n" + + " \"recipientAccountId\": \"888888888888\",\n" + + " \"eventCategory\": \"Management\",\n" + + " \"tlsDetails\": {\n" + + " \"tlsVersion\": \"TLSv1.2\",\n" + + " \"cipherSuite\": \"ECDHE-RSA-AES128-GCM-SHA256\",\n" + + " \"clientProvidedHostHeader\": \"iam.amazonaws.com\"\n" + + " },\n" + + " \"sessionCredentialFromConsole\": \"true\"\n" + + "}"; + } + public static String randomAppLogDoc() { return "{\n" + " \"endpoint\": \"/customer_records.txt\",\n" + @@ -1694,6 +1835,312 @@ public static String adLdapLogMappings() { " }"; } + public static String cloudtrailMappings() { + return "\"properties\": {\n" + + " \"Records\": {\n" + + " \"properties\": {\n" + + " \"awsRegion\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventCategory\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventID\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventSource\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventTime\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"eventType\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventVersion\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"managementEvent\": {\n" + + " \"type\": \"boolean\"\n" + + " },\n" + + " \"readOnly\": {\n" + + " \"type\": \"boolean\"\n" + + " },\n" + + " \"recipientAccountId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"requestID\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"requestParameters\": {\n" + + " \"properties\": {\n" + + " \"userName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"responseElements\": {\n" + + " \"properties\": {\n" + + " \"user\": {\n" + + " \"properties\": {\n" + + " \"arn\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"createDate\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"path\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sessionCredentialFromConsole\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sourceIPAddress\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"tlsDetails\": {\n" + + " \"properties\": {\n" + + " \"cipherSuite\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"clientProvidedHostHeader\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"tlsVersion\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userAgent\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userIdentity\": {\n" + + " \"properties\": {\n" + + " \"accessKeyId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"accountId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"arn\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"principalId\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sessionContext\": {\n" + + " \"properties\": {\n" + + " \"attributes\": {\n" + + " \"properties\": {\n" + + " \"creationDate\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"mfaAuthenticated\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"sessionIssuer\": {\n" + + " \"type\": \"object\"\n" + + " },\n" + + " \"webIdFederationData\": {\n" + + " \"type\": \"object\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"type\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"userName\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }}"; + } + public static String s3AccessLogMappings() { return " \"properties\": {" + " \"aws.cloudtrail.eventSource\": {" + diff --git a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java index b15fd6277..9e1622402 100644 --- a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java @@ -4,6 +4,9 @@ */ package org.opensearch.securityanalytics.correlation; +import org.apache.http.HttpStatus; +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHeader; import org.junit.Assert; import org.opensearch.client.Request; import org.opensearch.client.Response; @@ -11,14 +14,15 @@ import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.TestHelpers; import org.opensearch.securityanalytics.model.CorrelationQuery; import org.opensearch.securityanalytics.model.CorrelationRule; +import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; -import org.opensearch.securityanalytics.util.CorrelationIndices; import java.io.IOException; import java.util.Collections; @@ -33,6 +37,8 @@ public class CorrelationEngineRestApiIT extends SecurityAnalyticsRestTestCase { @SuppressWarnings("unchecked") public void testBasicCorrelationEngineWorkflow() throws IOException, InterruptedException { + updateClusterSetting(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS.getKey(), "true"); + LogIndices indices = createIndices(); String vpcFlowMonitorId = createVpcFlowDetector(indices.vpcFlowsIndex); @@ -164,6 +170,7 @@ public void testListCorrelationsWorkflow() throws IOException, InterruptedExcept @SuppressWarnings("unchecked") public void testBasicCorrelationEngineWorkflowWithoutRules() throws IOException, InterruptedException { + updateClusterSetting(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS.getKey(), "true"); LogIndices indices = createIndices(); String vpcFlowMonitorId = createVpcFlowDetector(indices.vpcFlowsIndex); @@ -231,6 +238,7 @@ public void testBasicCorrelationEngineWorkflowWithoutRules() throws IOException, @SuppressWarnings("unchecked") public void testBasicCorrelationEngineWorkflowWithRolloverByMaxAge() throws IOException, InterruptedException { + updateClusterSetting(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS.getKey(), "true"); updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_INDEX_MAX_AGE.getKey(), "1s"); @@ -325,6 +333,7 @@ public void testBasicCorrelationEngineWorkflowWithRolloverByMaxAge() throws IOEx } public void testBasicCorrelationEngineWorkflowWithRolloverByMaxDoc() throws IOException, InterruptedException { + updateClusterSetting(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS.getKey(), "true"); updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_MAX_DOCS.getKey(), "1"); @@ -419,6 +428,7 @@ public void testBasicCorrelationEngineWorkflowWithRolloverByMaxDoc() throws IOEx } public void testBasicCorrelationEngineWorkflowWithRolloverByMaxDocAndShortRetention() throws IOException, InterruptedException { + updateClusterSetting(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS.getKey(), "true"); updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); updateClusterSetting(SecurityAnalyticsSettings.CORRELATION_HISTORY_MAX_DOCS.getKey(), "1"); @@ -521,6 +531,456 @@ public void testBasicCorrelationEngineWorkflowWithRolloverByMaxDocAndShortRetent } } + public void testBasicCorrelationEngineWorkflowWithFieldBasedRules() throws IOException, InterruptedException { + Long startTime = System.currentTimeMillis(); + String index = createTestIndex("cloudtrail", cloudtrailMappings()); + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{\n" + + " \"index_name\": \"" + index + "\",\n" + + " \"rule_topic\": \"cloudtrail\",\n" + + " \"partial\": true,\n" + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"aws.cloudtrail.event_name\": {\n" + + " \"path\": \"Records.eventName\",\n" + + " \"type\": \"alias\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode()); + + String rule1 = randomCloudtrailRuleForCorrelations("CreateUser"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule1), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId1 = responseBody.get("_id").toString(); + + String rule2 = randomCloudtrailRuleForCorrelations("DeleteUser"); + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule2), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + responseBody = asMap(createResponse); + String createdId2 = responseBody.get("_id").toString(); + + createCloudtrailFieldBasedRule(index, "requestParameters.userName", null); + + Detector cloudtrailDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("cloudtrail detector for security analytics", List.of(index), + List.of(new DetectorRule(createdId1), new DetectorRule(createdId2)), + List.of())), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("cloudtrail"), List.of(), List.of(), List.of(), List.of(), List.of())), "cloudtrail"); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(cloudtrailDetector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomCloudtrailDoc("Richard", "CreateUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + Thread.sleep(1000); + indexDoc(index, "4", randomCloudtrailDoc("deysubho", "CreateUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + Thread.sleep(1000); + + indexDoc(index, "2", randomCloudtrailDoc("Richard", "DeleteUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detectorType", "cloudtrail"); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + Thread.sleep(5000); + + int count = 0; + while (true) { + try { + Long endTime = System.currentTimeMillis(); + Request restRequest = new Request("GET", "/_plugins/_security_analytics/correlations?start_timestamp=" + startTime + "&end_timestamp=" + endTime); + response = client().performRequest(restRequest); + + Map responseMap = entityAsMap(response); + List results = (List) responseMap.get("findings"); + if (results.size() == 1) { + Assert.assertTrue(true); + break; + } + } catch (Exception ex) { + // suppress ex + } + ++count; + Thread.sleep(5000); + if (count >= 12) { + Assert.assertTrue(false); + break; + } + } + } + + public void testBasicCorrelationEngineWorkflowWithFieldBasedRulesOnMultipleLogTypes() throws IOException, InterruptedException { + updateClusterSetting(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS.getKey(), "false"); + + LogIndices indices = new LogIndices(); + indices.windowsIndex = createTestIndex(randomIndex(), windowsIndexMapping()); + indices.vpcFlowsIndex = createTestIndex("vpc_flow", vpcFlowMappings()); + + String vpcFlowMonitorId = createVpcFlowDetector(indices.vpcFlowsIndex); + String testWindowsMonitorId = createTestWindowsDetector(indices.windowsIndex); + + String ruleId = createNetworkToWindowsFieldBasedRule(indices); + + indexDoc(indices.windowsIndex, "2", randomDoc()); + Response executeResponse = executeAlertingMonitor(testWindowsMonitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + indexDoc(indices.vpcFlowsIndex, "1", randomVpcFlowDoc()); + executeResponse = executeAlertingMonitor(vpcFlowMonitorId, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + Thread.sleep(5000); + + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detectorType", "test_windows"); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + String finding = ((List>) getFindingsBody.get("findings")).get(0).get("id").toString(); + + int count = 0; + while (true) { + try { + List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); + if (correlatedFindings.size() == 1) { + Assert.assertTrue(true); + + Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); + + for (var correlatedFinding: correlatedFindings) { + if (correlatedFinding.get("detector_type").equals("network")) { + Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); + Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + } + } + break; + } + } catch (Exception ex) { + // suppress ex + } + ++count; + Thread.sleep(5000); + if (count >= 12) { + Assert.assertTrue(false); + break; + } + } + } + + public void testBasicCorrelationEngineWorkflowWithIndexPatterns() throws IOException, InterruptedException { + updateClusterSetting(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS.getKey(), "false"); + + LogIndices indices = new LogIndices(); + createTestIndex("windows1", windowsIndexMapping()); + createTestIndex("windows2", windowsIndexMapping()); + indices.windowsIndex = "windows*"; + createTestIndex("vpc_flow1", vpcFlowMappings()); + createTestIndex("vpc_flow2", vpcFlowMappings()); + indices.vpcFlowsIndex = "vpc_flow*"; + + String vpcFlowMonitorId = createVpcFlowDetector(indices.vpcFlowsIndex); + String testWindowsMonitorId = createTestWindowsDetector(indices.windowsIndex); + + String ruleId = createNetworkToWindowsFilterQueryBasedRule(indices); + + indexDoc("windows2", "2", randomDoc()); + Response executeResponse = executeAlertingMonitor(testWindowsMonitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + indexDoc("vpc_flow1", "1", randomVpcFlowDoc()); + executeResponse = executeAlertingMonitor(vpcFlowMonitorId, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + Thread.sleep(5000); + + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detectorType", "test_windows"); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + String finding = ((List>) getFindingsBody.get("findings")).get(0).get("id").toString(); + + int count = 0; + while (true) { + try { + List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); + if (correlatedFindings.size() == 1) { + Assert.assertTrue(true); + + Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); + + for (var correlatedFinding: correlatedFindings) { + if (correlatedFinding.get("detector_type").equals("network")) { + Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); + Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + } + } + break; + } + } catch (Exception ex) { + // suppress ex + } + ++count; + Thread.sleep(5000); + if (count >= 12) { + Assert.assertTrue(false); + break; + } + } + } + + public void testBasicCorrelationEngineWorkflowWithFieldBasedRulesAndDynamicTimeWindow() throws IOException, InterruptedException { + Long startTime = System.currentTimeMillis(); + String index = createTestIndex("cloudtrail", cloudtrailMappings()); + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{\n" + + " \"index_name\": \"" + index + "\",\n" + + " \"rule_topic\": \"cloudtrail\",\n" + + " \"partial\": true,\n" + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"aws.cloudtrail.event_name\": {\n" + + " \"path\": \"Records.eventName\",\n" + + " \"type\": \"alias\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode()); + + String rule1 = randomCloudtrailRuleForCorrelations("CreateUser"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule1), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId1 = responseBody.get("_id").toString(); + + String rule2 = randomCloudtrailRuleForCorrelations("DeleteUser"); + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule2), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + responseBody = asMap(createResponse); + String createdId2 = responseBody.get("_id").toString(); + + createCloudtrailFieldBasedRule(index, "requestParameters.userName", 5000L); + + Detector cloudtrailDetector = randomDetectorWithInputsAndTriggersAndType(List.of(new DetectorInput("cloudtrail detector for security analytics", List.of(index), + List.of(new DetectorRule(createdId1), new DetectorRule(createdId2)), + List.of())), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("cloudtrail"), List.of(), List.of(), List.of(), List.of(), List.of())), "cloudtrail"); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(cloudtrailDetector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomCloudtrailDoc("Richard", "CreateUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + Thread.sleep(30000); + indexDoc(index, "4", randomCloudtrailDoc("deysubho", "CreateUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + Thread.sleep(1000); + + indexDoc(index, "2", randomCloudtrailDoc("Richard", "DeleteUser")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detectorType", "cloudtrail"); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + Thread.sleep(5000); + + int count = 0; + while (true) { + try { + Long endTime = System.currentTimeMillis(); + Request restRequest = new Request("GET", "/_plugins/_security_analytics/correlations?start_timestamp=" + startTime + "&end_timestamp=" + endTime); + response = client().performRequest(restRequest); + + Map responseMap = entityAsMap(response); + List results = (List) responseMap.get("findings"); + if (results.size() == 1) { + Assert.assertTrue(true); + break; + } + } catch (Exception ex) { + // suppress ex + } + ++count; + Thread.sleep(5000); + if (count >= 2) { + break; + } + } + Assert.assertEquals(2, count); + } + + public void testBasicCorrelationEngineWorkflowWithCustomLogTypes() throws IOException, InterruptedException { + LogIndices indices = new LogIndices(); + indices.vpcFlowsIndex = createTestIndex("vpc_flow1", vpcFlowMappings()); + + String vpcFlowMonitorId = createVpcFlowDetector(indices.vpcFlowsIndex); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + customLogType.getName() + "\", " + + " \"partial\":true, " + + " \"alias_mappings\":{}" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomRule(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("custom log type detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputs(List.of(input), customLogType.getName()); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + createdId = responseBody.get("_id").toString(); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", customLogType.getName(), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + String ruleId = createNetworkToCustomLogTypeFieldBasedRule(indices, customLogType.getName(), index); + + indexDoc(index, "1", randomDoc()); + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + + indexDoc(indices.vpcFlowsIndex, "1", randomVpcFlowDoc()); + executeResponse = executeAlertingMonitor(vpcFlowMonitorId, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + Thread.sleep(5000); + + Map params = new HashMap<>(); + params.put("detectorType", customLogType.getName()); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + String finding = ((List>) getFindingsBody.get("findings")).get(0).get("id").toString(); + + int count = 0; + while (true) { + try { + List> correlatedFindings = searchCorrelatedFindings(finding, customLogType.getName(), 300000L, 10); + if (correlatedFindings.size() == 1) { + Assert.assertTrue(true); + + Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); + + for (var correlatedFinding: correlatedFindings) { + if (correlatedFinding.get("detector_type").equals("network")) { + Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); + Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); + } + } + break; + } + } catch (Exception ex) { + // suppress ex + } + ++count; + Thread.sleep(5000); + if (count >= 12) { + Assert.assertTrue(false); + break; + } + } + } + private LogIndices createIndices() throws IOException { LogIndices indices = new LogIndices(); indices.adLdapLogsIndex = createTestIndex("ad_logs", adLdapLogMappings()); @@ -531,12 +991,51 @@ private LogIndices createIndices() throws IOException { return indices; } + private String createNetworkToWindowsFieldBasedRule(LogIndices indices) throws IOException { + CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, null, "network", "srcaddr"); + CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, null, "test_windows", "SourceIp"); + + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L); + Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); + request.setJsonEntity(toJsonString(rule)); + Response response = client().performRequest(request); + + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + return entityAsMap(response).get("_id").toString(); + } + + private String createNetworkToWindowsFilterQueryBasedRule(LogIndices indices) throws IOException { + CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "srcaddr:1.2.3.4", "network", null); + CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "SourceIp:1.2.3.4", "test_windows", null); + + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to windows", List.of(query1, query4), 300000L); + Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); + request.setJsonEntity(toJsonString(rule)); + Response response = client().performRequest(request); + + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + return entityAsMap(response).get("_id").toString(); + } + + private String createNetworkToCustomLogTypeFieldBasedRule(LogIndices indices, String customLogTypeName, String customLogTypeIndex) throws IOException { + CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, null, "network", "srcaddr"); + CorrelationQuery query4 = new CorrelationQuery(customLogTypeIndex, null, customLogTypeName, "SourceIp"); + + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to custom log type", List.of(query1, query4), 300000L); + Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); + request.setJsonEntity(toJsonString(rule)); + Response response = client().performRequest(request); + + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + return entityAsMap(response).get("_id").toString(); + } + private String createNetworkToAdLdapToWindowsRule(LogIndices indices) throws IOException { - CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "dstaddr:4.5.6.7", "network"); - CorrelationQuery query2 = new CorrelationQuery(indices.adLdapLogsIndex, "ResultType:50126", "ad_ldap"); - CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "Domain:NTAUTHORI*", "test_windows"); + CorrelationQuery query1 = new CorrelationQuery(indices.vpcFlowsIndex, "dstaddr:4.5.6.7", "network", null); + CorrelationQuery query2 = new CorrelationQuery(indices.adLdapLogsIndex, "ResultType:50126", "ad_ldap", null); + CorrelationQuery query4 = new CorrelationQuery(indices.windowsIndex, "Domain:NTAUTHORI*", "test_windows", null); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to ad_ldap to windows", List.of(query1, query2, query4)); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "network to ad_ldap to windows", List.of(query1, query2, query4), 300000L); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request); @@ -546,11 +1045,24 @@ private String createNetworkToAdLdapToWindowsRule(LogIndices indices) throws IOE } private String createWindowsToAppLogsToS3LogsRule(LogIndices indices) throws IOException { - CorrelationQuery query1 = new CorrelationQuery(indices.windowsIndex, "HostName:EC2AMAZ*", "test_windows"); - CorrelationQuery query2 = new CorrelationQuery(indices.appLogsIndex, "endpoint:\\/customer_records.txt", "others_application"); - CorrelationQuery query4 = new CorrelationQuery(indices.s3AccessLogsIndex, "aws.cloudtrail.eventName:ReplicateObject", "s3"); + CorrelationQuery query1 = new CorrelationQuery(indices.windowsIndex, "HostName:EC2AMAZ*", "test_windows", null); + CorrelationQuery query2 = new CorrelationQuery(indices.appLogsIndex, "endpoint:\\/customer_records.txt", "others_application", null); + CorrelationQuery query4 = new CorrelationQuery(indices.s3AccessLogsIndex, "aws.cloudtrail.eventName:ReplicateObject", "s3", null); + + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "windows to app_logs to s3 logs", List.of(query1, query2, query4), 300000L); + Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); + request.setJsonEntity(toJsonString(rule)); + Response response = client().performRequest(request); + + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + return entityAsMap(response).get("_id").toString(); + } + + private String createCloudtrailFieldBasedRule(String index, String field, Long timeWindow) throws IOException { + CorrelationQuery query1 = new CorrelationQuery(index, "EventName:CreateUser", "cloudtrail", field); + CorrelationQuery query2 = new CorrelationQuery(index, "EventName:DeleteUser", "cloudtrail", field); - CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "windows to app_logs to s3 logs", List.of(query1, query2, query4)); + CorrelationRule rule = new CorrelationRule(CorrelationRule.NO_ID, CorrelationRule.NO_VERSION, "cloudtrail field based", List.of(query1, query2), timeWindow); Request request = new Request("POST", "/_plugins/_security_analytics/correlation/rules"); request.setJsonEntity(toJsonString(rule)); Response response = client().performRequest(request); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index b91317da6..f952ecaf9 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -5,6 +5,8 @@ package org.opensearch.securityanalytics.resthandler; import org.apache.http.HttpStatus; +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHeader; import org.junit.Assert; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; @@ -28,12 +30,15 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import static java.util.Collections.emptyList; import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRule; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailDoc; import static org.opensearch.securityanalytics.TestHelpers.randomDetector; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; @@ -1902,6 +1907,81 @@ public void testCreateDetectorWithKeywordsRule_ensureNoFindingsWithoutDateMappin assertEquals(0, noOfSigmaRuleMatches); } + @SuppressWarnings("unchecked") + public void testCreateDetectorWithCloudtrailAggrRule() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + indexDoc(index, "0", randomCloudtrailDoc("A12346", "CREATED")); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomCloudtrailAggrRule(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", randomDetectorType()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String workflowId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("workflow_ids")).get(0); + + indexDoc(index, "1", randomCloudtrailDoc("A12345", "CREATED")); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + indexDoc(index, "2", randomCloudtrailDoc("A12345", "DELETED")); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + Map params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + assertEquals(1, getFindingsBody.get("total_findings")); + } + private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { List> buckets = ((List>) (((Map) ((Map) ((Map) ((List) ((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); Integer docCount = buckets.stream().mapToInt(it -> (Integer) it.get("doc_count")).sum(); diff --git a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java index 395f15a79..9103d03ab 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java @@ -216,4 +216,39 @@ public void testAvgAggregationWithGroupBy() throws IOException, SigmaError { Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"avg\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } + + public void testCloudtrailAggregationRule() throws IOException, SigmaError { + OSQueryBackend queryBackend = new OSQueryBackend(Map.of(), true, true); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + "id: c64c5175-5189-431b-a55e-6d9882158250\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: Accounts created and deleted within 24h\n" + + "description: Flag suspicious activity of accounts created and deleted within 24h\n" + + "date: 2021/09/23\n" + + "tags:\n" + + " - attack.exfiltration\n" + + "falsepositives: [ ]\n" + + "level: high\n" + + "status: test\n" + + "references: [ ]\n" + + "author: Sashank\n" + + "detection:\n" + + " selection:\n" + + " event:\n" + + " - CREATED\n" + + " - DELETED\n" + + " timeframe: 24h\n" + + " condition: selection | count(*) by accountid > 2", true)); + + String query = queries.get(0).toString(); + Assert.assertEquals("(event: \"CREATED\") OR (event: \"DELETED\")", query); + + OSQueryBackend.AggregationQueries aggQueries = (OSQueryBackend.AggregationQueries) queries.get(1); + String aggQuery = aggQueries.getAggQuery(); + String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); + + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"accountid\"}}}", aggQuery); + Assert.assertEquals("{\"buckets_path\":{\"_cnt\":\"_count\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params._cnt > 2.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); + } } \ No newline at end of file