diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 7410e9c3e..d7b7ae879 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -4,18 +4,6 @@ */ package org.opensearch.securityanalytics.transport; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -41,6 +29,8 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.io.stream.NamedWriteableRegistry; @@ -68,9 +58,11 @@ import org.opensearch.commons.alerting.model.SearchInput; import org.opensearch.commons.alerting.model.action.Action; import org.opensearch.commons.authuser.User; +import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.RangeQueryBuilder; import org.opensearch.index.reindex.BulkByScrollResponse; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.rest.RestRequest; @@ -80,11 +72,15 @@ import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.action.GetIndexMappingsAction; +import org.opensearch.securityanalytics.action.GetIndexMappingsRequest; +import org.opensearch.securityanalytics.action.GetIndexMappingsResponse; import org.opensearch.securityanalytics.action.IndexDetectorAction; import org.opensearch.securityanalytics.action.IndexDetectorRequest; import org.opensearch.securityanalytics.action.IndexDetectorResponse; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.mapper.MapperService; +import org.opensearch.securityanalytics.mapper.MapperUtils; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; @@ -105,10 +101,24 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + public class TransportIndexDetectorAction extends HandledTransportAction implements SecureTransportAction { public static final String PLUGIN_OWNER_FIELD = "security_analytics"; private static final Logger log = LogManager.getLogger(TransportIndexDetectorAction.class); + public static final String TIMESTAMP_FIELD_ALIAS = "timestamp"; private final Client client; @@ -133,6 +143,8 @@ public class TransportIndexDetectorAction extends HandledTransportAction> pairs = MapperUtils.getAllAliasPathPairs(mappingMetadata); + boolean timeStampAliasPresent = pairs. + stream() + .anyMatch(p -> + TIMESTAMP_FIELD_ALIAS.equals(p.getLeft()) || TIMESTAMP_FIELD_ALIAS.equals(p.getRight())); + if(timeStampAliasPresent) { + BoolQueryBuilder boolQueryBuilder = searchSourceBuilder.query() == null + ? new BoolQueryBuilder() + : QueryBuilders.boolQuery().must(searchSourceBuilder.query()); + RangeQueryBuilder timeRangeFilter = QueryBuilders.rangeQuery(TIMESTAMP_FIELD_ALIAS) + .gt("{{period_end}}||-1h") + .lte("{{period_end}}") + .format("epoch_millis"); + boolQueryBuilder.must(timeRangeFilter); + searchSourceBuilder.query(boolQueryBuilder); + } + } catch (Exception e) { + log.error( + String.format(Locale.getDefault(), + "Unable to verify presence of timestamp alias for index [%s] in detector [%s]. Not setting time range filter for bucket level monitor.", + concreteIndex, detector.getName()), e); + } List bucketLevelMonitorInputs = new ArrayList<>(); bucketLevelMonitorInputs.add(new SearchInput(Arrays.asList(index), searchSourceBuilder)); diff --git a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java index 4388d3a44..3358f5d12 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java @@ -7,11 +7,13 @@ import java.util.SortedMap; import org.opensearch.action.ActionListener; import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.IndicesAdminClient; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexAbstraction; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.NamedXContentRegistry; import org.opensearch.common.xcontent.XContentParser; @@ -154,4 +156,10 @@ public static String getNewestIndexByCreationDate(String[] concreteIndices, Clus } return newestIndex; } + + public static String getNewIndexByCreationDate(ClusterState state, IndexNameExpressionResolver i, String index) { + String[] strings = i.concreteIndexNames(state, IndicesOptions.LENIENT_EXPAND_OPEN, index); + return getNewestIndexByCreationDate(strings, state); + } + } \ No newline at end of file diff --git a/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml b/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml index 9ab79f08f..e519f3963 100644 --- a/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml +++ b/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml @@ -1,7 +1,4 @@ # this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under windows log group to their corresponding ECS Fields. fieldmappings: - EventID: event_uid - HiveName: unmapped.HiveName - fieldB: mappedB - fieldA1: mappedA - CommandLine: windows-event_data-CommandLine + TargetUserName: winlog-event_data-TargetUserName + creationTime: timestamp diff --git a/src/main/resources/OSMapping/ad_ldap/mappings.json b/src/main/resources/OSMapping/ad_ldap/mappings.json index ea77f2460..f51459bb7 100644 --- a/src/main/resources/OSMapping/ad_ldap/mappings.json +++ b/src/main/resources/OSMapping/ad_ldap/mappings.json @@ -1,12 +1,12 @@ { "properties": { - "windows-event_data-CommandLine": { - "type": "alias", - "path": "CommandLine" + "winlog-event_data-TargetUserName": { + "path": "winlog.event_data.TargetUserName", + "type": "alias" }, - "event_uid": { - "type": "alias", - "path": "EventID" + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/apache_access/fieldmappings.yml b/src/main/resources/OSMapping/apache_access/fieldmappings.yml index 9ab79f08f..9f649aeee 100644 --- a/src/main/resources/OSMapping/apache_access/fieldmappings.yml +++ b/src/main/resources/OSMapping/apache_access/fieldmappings.yml @@ -5,3 +5,4 @@ fieldmappings: fieldB: mappedB fieldA1: mappedA CommandLine: windows-event_data-CommandLine + creationTime: timestamp diff --git a/src/main/resources/OSMapping/apache_access/mappings.json b/src/main/resources/OSMapping/apache_access/mappings.json index ea77f2460..dc7fc31b5 100644 --- a/src/main/resources/OSMapping/apache_access/mappings.json +++ b/src/main/resources/OSMapping/apache_access/mappings.json @@ -7,6 +7,10 @@ "event_uid": { "type": "alias", "path": "EventID" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml b/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml index eb6350d9e..6399929b6 100644 --- a/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml +++ b/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml @@ -7,4 +7,5 @@ fieldmappings: requestParameters.containerDefinitions.command: aws-cloudtrail-requestParameters-containerDefinitions-command userIdentity.sessionContext.sessionIssuer.type: userIdentity-sessionContext-sessionIssuer-type userIdentity.type: aws-cloudtrail-userIdentity-type - userIdentity.arn: aws-cloudtrail-userIdentity-arn \ No newline at end of file + userIdentity.arn: aws-cloudtrail-userIdentity-arn + eventTime: timestamp \ No newline at end of file diff --git a/src/main/resources/OSMapping/cloudtrail/mappings.json b/src/main/resources/OSMapping/cloudtrail/mappings.json index 7a9a35c47..ac8d1898c 100644 --- a/src/main/resources/OSMapping/cloudtrail/mappings.json +++ b/src/main/resources/OSMapping/cloudtrail/mappings.json @@ -35,6 +35,10 @@ "userIdentity-sessionContext-sessionIssuer-type": { "type": "alias", "path": "aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.type" + }, + "timestamp": { + "path": "aws.cloudtrail.eventTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/dns/fieldmappings.yml b/src/main/resources/OSMapping/dns/fieldmappings.yml index fc85e1ee7..7672ee77a 100644 --- a/src/main/resources/OSMapping/dns/fieldmappings.yml +++ b/src/main/resources/OSMapping/dns/fieldmappings.yml @@ -1,4 +1,5 @@ fieldmappings: record_type: dns-answers-type query: dns-question-name - parent_domain: dns-question-registered_domain \ No newline at end of file + parent_domain: dns-question-registered_domain + creationTime: timestamp \ No newline at end of file diff --git a/src/main/resources/OSMapping/dns/mappings.json b/src/main/resources/OSMapping/dns/mappings.json index 6f9f869ea..0905e2c9b 100644 --- a/src/main/resources/OSMapping/dns/mappings.json +++ b/src/main/resources/OSMapping/dns/mappings.json @@ -11,6 +11,10 @@ "dns-question-registered_domain": { "type": "alias", "path": "dns.question.registered_domain" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } diff --git a/src/main/resources/OSMapping/linux/fieldmappings.yml b/src/main/resources/OSMapping/linux/fieldmappings.yml index aedb0394f..d58617df6 100644 --- a/src/main/resources/OSMapping/linux/fieldmappings.yml +++ b/src/main/resources/OSMapping/linux/fieldmappings.yml @@ -12,4 +12,4 @@ fieldmappings: ParentImage: process-parent-executable CurrentDirectory: process-working_directory LogonId: process-real_user-id - + creationTime: timestamp diff --git a/src/main/resources/OSMapping/linux/mappings.json b/src/main/resources/OSMapping/linux/mappings.json index 17fb80188..5c69c0c48 100644 --- a/src/main/resources/OSMapping/linux/mappings.json +++ b/src/main/resources/OSMapping/linux/mappings.json @@ -47,6 +47,10 @@ "process-real_user-id": { "path": "process.real_user.id", "type": "alias" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/network/NetFlowMapping.json b/src/main/resources/OSMapping/network/NetFlowMapping.json index 2bc0d4d76..9e404a67e 100644 --- a/src/main/resources/OSMapping/network/NetFlowMapping.json +++ b/src/main/resources/OSMapping/network/NetFlowMapping.json @@ -23,6 +23,10 @@ "http.response.status_code": { "type": "alias", "path": "netflow.http_status_code" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/network/fieldmappings.yml b/src/main/resources/OSMapping/network/fieldmappings.yml index 2a2753832..2a10f24f4 100644 --- a/src/main/resources/OSMapping/network/fieldmappings.yml +++ b/src/main/resources/OSMapping/network/fieldmappings.yml @@ -14,4 +14,5 @@ fieldmappings: client_header_names: zeek-http-client_header_names resp_mime_types: zeek-http-resp_mime_types cipher: zeek-kerberos-cipher - request_type: zeek-kerberos-request_type \ No newline at end of file + request_type: zeek-kerberos-request_type + creationTime: timestamp \ No newline at end of file diff --git a/src/main/resources/OSMapping/network/mappings.json b/src/main/resources/OSMapping/network/mappings.json index 0a521ae75..97f5f5d74 100644 --- a/src/main/resources/OSMapping/network/mappings.json +++ b/src/main/resources/OSMapping/network/mappings.json @@ -59,6 +59,10 @@ "zeek-kerberos-request_type": { "path": "zeek.kerberos.request_type", "type": "alias" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } diff --git a/src/main/resources/OSMapping/others_application/fieldmappings.yml b/src/main/resources/OSMapping/others_application/fieldmappings.yml index e1dba4476..65ab8b638 100644 --- a/src/main/resources/OSMapping/others_application/fieldmappings.yml +++ b/src/main/resources/OSMapping/others_application/fieldmappings.yml @@ -4,4 +4,4 @@ fieldmappings: HiveName: unmapped.HiveName fieldB: mappedB fieldA1: mappedA - + creationTime: timestamp diff --git a/src/main/resources/OSMapping/others_application/mappings.json b/src/main/resources/OSMapping/others_application/mappings.json index 48cdda71d..a3ccdca77 100644 --- a/src/main/resources/OSMapping/others_application/mappings.json +++ b/src/main/resources/OSMapping/others_application/mappings.json @@ -23,6 +23,10 @@ "windows-servicename": { "type": "alias", "path": "ServiceName" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_apt/fieldmappings.yml b/src/main/resources/OSMapping/others_apt/fieldmappings.yml index a25dd9693..41f5d688f 100644 --- a/src/main/resources/OSMapping/others_apt/fieldmappings.yml +++ b/src/main/resources/OSMapping/others_apt/fieldmappings.yml @@ -4,4 +4,4 @@ fieldmappings: HiveName: unmapped.HiveName fieldB: mappedB fieldA1: mappedA - + creationTime: timestamp diff --git a/src/main/resources/OSMapping/others_apt/mappings.json b/src/main/resources/OSMapping/others_apt/mappings.json index 48cdda71d..a3ccdca77 100644 --- a/src/main/resources/OSMapping/others_apt/mappings.json +++ b/src/main/resources/OSMapping/others_apt/mappings.json @@ -23,6 +23,10 @@ "windows-servicename": { "type": "alias", "path": "ServiceName" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_cloud/fieldmappings.yml b/src/main/resources/OSMapping/others_cloud/fieldmappings.yml index 85af60cec..8d5691ea4 100644 --- a/src/main/resources/OSMapping/others_cloud/fieldmappings.yml +++ b/src/main/resources/OSMapping/others_cloud/fieldmappings.yml @@ -4,4 +4,4 @@ fieldmappings: HiveName: unmapped.HiveName fieldB: mappedB fieldA1: mappedA - + creationTime: timestamp diff --git a/src/main/resources/OSMapping/others_cloud/mappings.json b/src/main/resources/OSMapping/others_cloud/mappings.json index 48cdda71d..0e9426a6e 100644 --- a/src/main/resources/OSMapping/others_cloud/mappings.json +++ b/src/main/resources/OSMapping/others_cloud/mappings.json @@ -23,6 +23,10 @@ "windows-servicename": { "type": "alias", "path": "ServiceName" + }, + "creationTime": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_compliance/fieldmappings.yml b/src/main/resources/OSMapping/others_compliance/fieldmappings.yml index ddf9081d6..d1ec0d91a 100644 --- a/src/main/resources/OSMapping/others_compliance/fieldmappings.yml +++ b/src/main/resources/OSMapping/others_compliance/fieldmappings.yml @@ -4,4 +4,4 @@ fieldmappings: HiveName: unmapped.HiveName fieldB: mappedB fieldA1: mappedA - + creationTime: timestamp diff --git a/src/main/resources/OSMapping/others_compliance/mappings.json b/src/main/resources/OSMapping/others_compliance/mappings.json index 48cdda71d..a3ccdca77 100644 --- a/src/main/resources/OSMapping/others_compliance/mappings.json +++ b/src/main/resources/OSMapping/others_compliance/mappings.json @@ -23,6 +23,10 @@ "windows-servicename": { "type": "alias", "path": "ServiceName" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_macos/fieldmappings.yml b/src/main/resources/OSMapping/others_macos/fieldmappings.yml index 35a88e285..1754fcddc 100644 --- a/src/main/resources/OSMapping/others_macos/fieldmappings.yml +++ b/src/main/resources/OSMapping/others_macos/fieldmappings.yml @@ -4,3 +4,4 @@ fieldmappings: HiveName: unmapped.HiveName fieldB: mappedB fieldA1: mappedA + creationTime: timestamp \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_macos/mappings.json b/src/main/resources/OSMapping/others_macos/mappings.json index 0804434f1..5da91e3d9 100644 --- a/src/main/resources/OSMapping/others_macos/mappings.json +++ b/src/main/resources/OSMapping/others_macos/mappings.json @@ -3,6 +3,10 @@ "macos.event_data.CommandLine": { "type": "alias", "path": "CommandLine" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_proxy/fieldmappings.yml b/src/main/resources/OSMapping/others_proxy/fieldmappings.yml index ddaaa6956..08b377f19 100644 --- a/src/main/resources/OSMapping/others_proxy/fieldmappings.yml +++ b/src/main/resources/OSMapping/others_proxy/fieldmappings.yml @@ -4,4 +4,4 @@ fieldmappings: HiveName: unmapped.HiveName fieldB: mappedB fieldA1: mappedA - + creationTime: timestamp diff --git a/src/main/resources/OSMapping/others_proxy/mappings.json b/src/main/resources/OSMapping/others_proxy/mappings.json index 48cdda71d..a3ccdca77 100644 --- a/src/main/resources/OSMapping/others_proxy/mappings.json +++ b/src/main/resources/OSMapping/others_proxy/mappings.json @@ -23,6 +23,10 @@ "windows-servicename": { "type": "alias", "path": "ServiceName" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_web/fieldmappings.yml b/src/main/resources/OSMapping/others_web/fieldmappings.yml index a1cc130a2..8d7fd290e 100644 --- a/src/main/resources/OSMapping/others_web/fieldmappings.yml +++ b/src/main/resources/OSMapping/others_web/fieldmappings.yml @@ -4,4 +4,5 @@ fieldmappings: HiveName: unmapped.HiveName fieldB: mappedB fieldA1: mappedA + creationTime: timestamp diff --git a/src/main/resources/OSMapping/others_web/mappings.json b/src/main/resources/OSMapping/others_web/mappings.json index 48cdda71d..a3ccdca77 100644 --- a/src/main/resources/OSMapping/others_web/mappings.json +++ b/src/main/resources/OSMapping/others_web/mappings.json @@ -23,6 +23,10 @@ "windows-servicename": { "type": "alias", "path": "ServiceName" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/s3/fieldmappings.yml b/src/main/resources/OSMapping/s3/fieldmappings.yml index d2cbd9dc9..06ce921d8 100644 --- a/src/main/resources/OSMapping/s3/fieldmappings.yml +++ b/src/main/resources/OSMapping/s3/fieldmappings.yml @@ -1,3 +1,4 @@ fieldmappings: eventName: aws-cloudtrail-eventName eventSource: aws-cloudtrail-eventSource + eventTime: timestamp diff --git a/src/main/resources/OSMapping/s3/mappings.json b/src/main/resources/OSMapping/s3/mappings.json index 2559999ad..a13f40236 100644 --- a/src/main/resources/OSMapping/s3/mappings.json +++ b/src/main/resources/OSMapping/s3/mappings.json @@ -7,6 +7,10 @@ "aws-cloudtrail-eventName": { "type": "alias", "path": "aws.cloudtrail.eventName" + }, + "timestamp": { + "path": "aws.cloudtrail.eventTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/test_windows/fieldmappings.yml b/src/main/resources/OSMapping/test_windows/fieldmappings.yml index 7567e715b..04fff5ead 100644 --- a/src/main/resources/OSMapping/test_windows/fieldmappings.yml +++ b/src/main/resources/OSMapping/test_windows/fieldmappings.yml @@ -9,3 +9,4 @@ fieldmappings: Message: windows-message Provider_Name: windows-provider-name ServiceName: windows-servicename + creationTime: timestamp diff --git a/src/main/resources/OSMapping/test_windows/mappings.json b/src/main/resources/OSMapping/test_windows/mappings.json index 48cdda71d..a3ccdca77 100644 --- a/src/main/resources/OSMapping/test_windows/mappings.json +++ b/src/main/resources/OSMapping/test_windows/mappings.json @@ -23,6 +23,10 @@ "windows-servicename": { "type": "alias", "path": "ServiceName" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/windows/fieldmappings.yml b/src/main/resources/OSMapping/windows/fieldmappings.yml index 7f259f8a7..8e905ad52 100644 --- a/src/main/resources/OSMapping/windows/fieldmappings.yml +++ b/src/main/resources/OSMapping/windows/fieldmappings.yml @@ -61,5 +61,5 @@ fieldmappings: windows-hostname: winlog-computer_name windows-provider-name: winlog-provider_name windows-servicename: winlog-event_data-ServiceName - + creationTime: timestamp diff --git a/src/main/resources/OSMapping/windows/mappings.json b/src/main/resources/OSMapping/windows/mappings.json index e2075c854..f2eef2a51 100644 --- a/src/main/resources/OSMapping/windows/mappings.json +++ b/src/main/resources/OSMapping/windows/mappings.json @@ -187,6 +187,10 @@ "winlog-event_data-param2": { "path": "winlog.event_data.param2", "type": "alias" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index d236f28a7..939b118f5 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -170,67 +170,76 @@ public static String randomRule() { public static String countAggregationTestRule() { return " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA: valueA\n" + - " fieldB: valueB\n" + - " fieldC: valueC\n" + - " condition: sel | count(*) > 1"; + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA: valueA\n" + + " fieldB: valueB\n" + + " fieldC: valueC\n" + + " condition: sel | count(*) > 1"; } public static String sumAggregationTestRule() { return " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA: 123\n" + - " fieldB: 111\n" + - " fieldC: valueC\n" + - " condition: sel | sum(fieldA) by fieldB > 110"; + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA: 123\n" + + " fieldB: 111\n" + + " fieldC: valueC\n" + + " condition: sel | sum(fieldA) by fieldB > 110"; } public static String productIndexMaxAggRule() { return " title: Test\n" + - " id: 5f92fff9-82e3-48eb-8fc1-8b133556a551\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA: 123\n" + - " fieldB: 111\n" + - " fieldC: valueC\n" + - " condition: sel | max(fieldA) by fieldB > 110"; + " id: 5f92fff9-82e3-48eb-8fc1-8b133556a551\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA: 123\n" + + " fieldB: 111\n" + + " fieldC: valueC\n" + + " condition: sel | max(fieldA) by fieldB > 110"; } public static String randomProductDocument(){ return "{\n" + - " \"fieldA\": 123,\n" + - " \"mappedB\": 111,\n" + - " \"fieldC\": \"valueC\"\n" + - "}\n"; + " \"fieldA\": 123,\n" + + " \"mappedB\": 111,\n" + + " \"fieldC\": \"valueC\"\n" + + "}\n"; + } + + public static String randomProductDocumentWithTime(long time){ + return "{\n" + + " \"fieldA\": 123,\n" + + " \"mappedB\": 111,\n" + + " \"time\": " + (time) + ",\n" + + " \"fieldC\": \"valueC\"\n" + + "}\n"; } public static String randomEditedRule() { @@ -429,95 +438,98 @@ public static String netFlowMappings() { public static String productIndexMapping(){ return "\"properties\":{\n" + - " \"fieldA\":{\n" + - " \"type\":\"long\"\n" + - " },\n" + - " \"mappedB\":{\n" + - " \"type\":\"long\"\n" + - " },\n" + - " \"fieldC\":{\n" + - " \"type\":\"keyword\"\n" + - " }\n" + - "}\n" + - "}"; + " \"fieldA\":{\n" + + " \"type\":\"long\"\n" + + " },\n" + + " \"mappedB\":{\n" + + " \"type\":\"long\"\n" + + " },\n" + + " \"time\":{\n" + + " \"type\":\"date\"\n" + + " },\n" + + " \"fieldC\":{\n" + + " \"type\":\"keyword\"\n" + + " }\n" + + "}\n" + + "}"; } public static String productIndexAvgAggRule(){ return " title: Test\n" + - " id: 39f918f3-981b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA: 123\n" + - " fieldB: 111\n" + - " fieldC: valueC\n" + - " condition: sel | avg(fieldA) by fieldC > 110"; + " id: 39f918f3-981b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA: 123\n" + + " fieldB: 111\n" + + " fieldC: valueC\n" + + " condition: sel | avg(fieldA) by fieldC > 110"; } public static String randomAggregationRule(String aggFunction, String signAndValue) { String rule = "title: Remote Encrypting File System Abuse\n" + - "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + - "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + - "references:\n" + - " - https://attack.mitre.org/tactics/TA0008/\n" + - " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + - " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + - " - https://github.com/zeronetworks/rpcfirewall\n" + - " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + - "tags:\n" + - " - attack.defense_evasion\n" + - "status: experimental\n" + - "author: Sagie Dulce, Dekel Paz\n" + - "date: 2022/01/01\n" + - "modified: 2022/01/01\n" + - "logsource:\n" + - " product: rpc_firewall\n" + - " category: application\n" + - " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + - "detection:\n" + - " sel:\n" + - " Opcode: Info\n" + - " condition: sel | %s(SeverityValue) by Version %s\n" + - "falsepositives:\n" + - " - Legitimate usage of remote file encryption\n" + - "level: high"; + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " sel:\n" + + " Opcode: Info\n" + + " condition: sel | %s(SeverityValue) by Version %s\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; return String.format(Locale.ROOT, rule, aggFunction, signAndValue); } public static String randomAggregationRule(String aggFunction, String signAndValue, String opCode) { String rule = "title: Remote Encrypting File System Abuse\n" + - "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + - "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + - "references:\n" + - " - https://attack.mitre.org/tactics/TA0008/\n" + - " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + - " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + - " - https://github.com/zeronetworks/rpcfirewall\n" + - " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + - "tags:\n" + - " - attack.defense_evasion\n" + - "status: experimental\n" + - "author: Sagie Dulce, Dekel Paz\n" + - "date: 2022/01/01\n" + - "modified: 2022/01/01\n" + - "logsource:\n" + - " product: rpc_firewall\n" + - " category: application\n" + - " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + - "detection:\n" + - " sel:\n" + - " Opcode: %s\n" + - " condition: sel | %s(SeverityValue) by Version %s\n" + - "falsepositives:\n" + - " - Legitimate usage of remote file encryption\n" + - "level: high"; + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " sel:\n" + + " Opcode: %s\n" + + " condition: sel | %s(SeverityValue) by Version %s\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; return String.format(Locale.ROOT, rule, opCode, aggFunction, signAndValue); } @@ -1146,39 +1158,39 @@ public static String windowsIndexMapping() { public static String randomDoc(int severity, int version, String opCode) { String doc = "{\n" + - "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + - "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + - "\"Keywords\":\"9223372036854775808\",\n" + - "\"SeverityValue\":%s,\n" + - "\"Severity\":\"INFO\",\n" + - "\"EventID\":22,\n" + - "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + - "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + - "\"Version\":%s,\n" + - "\"TaskValue\":22,\n" + - "\"OpcodeValue\":0,\n" + - "\"RecordNumber\":9532,\n" + - "\"ExecutionProcessID\":1996,\n" + - "\"ExecutionThreadID\":2616,\n" + - "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + - "\"Domain\":\"NT AUTHORITY\",\n" + - "\"AccountName\":\"SYSTEM\",\n" + - "\"UserID\":\"S-1-5-18\",\n" + - "\"AccountType\":\"User\",\n" + - "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + - "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + - "\"Opcode\":\"%s\",\n" + - "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + - "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + - "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + - "\"QueryResults\":\"172.31.46.38;\",\n" + - "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + - "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + - "\"SourceModuleName\":\"in\",\n" + - "\"SourceModuleType\":\"im_msvistalog\",\n" + - "\"CommandLine\": \"eachtest\",\n" + - "\"Initiated\": \"true\"\n" + - "}"; + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + + "\"Keywords\":\"9223372036854775808\",\n" + + "\"SeverityValue\":%s,\n" + + "\"Severity\":\"INFO\",\n" + + "\"EventID\":22,\n" + + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + "\"Version\":%s,\n" + + "\"TaskValue\":22,\n" + + "\"OpcodeValue\":0,\n" + + "\"RecordNumber\":9532,\n" + + "\"ExecutionProcessID\":1996,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + + "\"Domain\":\"NT AUTHORITY\",\n" + + "\"AccountName\":\"SYSTEM\",\n" + + "\"UserID\":\"S-1-5-18\",\n" + + "\"AccountType\":\"User\",\n" + + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + + "\"Opcode\":\"%s\",\n" + + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + + "\"QueryResults\":\"172.31.46.38;\",\n" + + "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + + "\"SourceModuleName\":\"in\",\n" + + "\"SourceModuleType\":\"im_msvistalog\",\n" + + "\"CommandLine\": \"eachtest\",\n" + + "\"Initiated\": \"true\"\n" + + "}"; return String.format(Locale.ROOT, doc, severity, version, opCode); } diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 50aa9bb24..d639e070b 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -4,10 +4,8 @@ */ package org.opensearch.securityanalytics.mapper; -import java.util.ArrayList; import java.util.Collections; import java.util.Optional; -import java.util.Set; import org.apache.http.HttpStatus; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; @@ -50,9 +48,9 @@ public void testCreateMappingSuccess() throws IOException { // both req params and req body are supported request.setJsonEntity( "{ \"index_name\":\"" + testIndexName + "\"," + - " \"rule_topic\":\"netflow\", " + - " \"partial\":true" + - "}" + " \"rule_topic\":\"netflow\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(request); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); @@ -89,22 +87,22 @@ public void testCreateMappingWithAliasesSuccess() throws IOException { // both req params and req body are supported request.setJsonEntity( "{\n" + - " \"index_name\": \"my_index\",\n" + - " \"rule_topic\":\"netflow\", " + - " \"partial\":true," + - " \"alias_mappings\": {\n" + - " \"properties\": {\n" + - " \"source.ip\": {\n" + - " \"type\": \"alias\",\n" + - " \"path\": \"netflow.source_ipv4_address\"\n" + - " },\n" + - " \"source.port\": {\n" + - " \"type\": \"alias\",\n" + - " \"path\": \"netflow.source_transport_port\"\n" + - " }\n" + - " }\n" + - " }\n" + - "}" + " \"index_name\": \"my_index\",\n" + + " \"rule_topic\":\"netflow\", " + + " \"partial\":true," + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"source.ip\": {\n" + + " \"type\": \"alias\",\n" + + " \"path\": \"netflow.source_ipv4_address\"\n" + + " },\n" + + " \"source.port\": {\n" + + " \"type\": \"alias\",\n" + + " \"path\": \"netflow.source_transport_port\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" ); // request.addParameter("indexName", testIndexName); // request.addParameter("ruleTopic", "netflow"); @@ -225,7 +223,7 @@ public void testExistingMappingsAreUntouched() throws IOException { GetMappingsResponse getMappingsResponse = SecurityAnalyticsClientUtils.executeGetMappingsRequest(testIndexName); Map properties = (Map) getMappingsResponse.getMappings().get(testIndexName) - .getSourceAsMap().get("properties"); + .getSourceAsMap().get("properties"); // Verify that there is still mapping for integer field "plain1" assertTrue(((Map)properties.get("plain1")).get("type").equals("integer")); } @@ -272,7 +270,7 @@ public void testGetMappingsViewSuccess() throws IOException { createSampleIndex(testIndexName); - // Execute GetMappingsViewAction to add alias mapping for index + // Execute CreateMappingsAction to add alias mapping for index Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); // both req params and req body are supported request.addParameter("index_name", testIndexName); @@ -292,332 +290,7 @@ public void testGetMappingsViewSuccess() throws IOException { assertEquals(6, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(2, unmappedFieldAliases.size()); - } - - public void testCreateMappings_withDatastream_success() throws IOException { - String datastream = "test_datastream"; - - String datastreamMappings = "\"properties\": {" + - " \"@timestamp\":{ \"type\": \"date\" }," + - " \"netflow.destination_transport_port\":{ \"type\": \"long\" }," + - " \"netflow.destination_ipv4_address\":{ \"type\": \"ip\" }" + - "}"; - - createSampleDatastream(datastream, datastreamMappings); - - // Execute CreateMappingsAction to add alias mapping for index - createMappingsAPI(datastream, "netflow"); - - // Verify mappings - Map props = getIndexMappingsAPIFlat(datastream); - assertEquals(5, props.size()); - assertTrue(props.containsKey("@timestamp")); - assertTrue(props.containsKey("netflow.destination_transport_port")); - assertTrue(props.containsKey("netflow.destination_ipv4_address")); - assertTrue(props.containsKey("destination.ip")); - assertTrue(props.containsKey("destination.port")); - - // Verify that index template applied mappings - Response response = makeRequest(client(), "POST", datastream + "/_rollover", Collections.emptyMap(), null); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); - - // Insert doc to index to add additional fields to mapping - String sampleDoc = "{" + - " \"@timestamp\":\"2023-01-06T00:05:00\"," + - " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + - " \"netflow.source_transport_port\":4444" + - "}"; - - indexDoc(datastream, "2", sampleDoc); - - // Execute CreateMappingsAction to add alias mapping for index - createMappingsAPI(datastream, "netflow"); - - String writeIndex = getDatastreamWriteIndex(datastream); - - // Verify mappings - props = getIndexMappingsAPIFlat(writeIndex); - assertEquals(9, props.size()); - assertTrue(props.containsKey("@timestamp")); - assertTrue(props.containsKey("netflow.source_ipv4_address")); - assertTrue(props.containsKey("netflow.source_transport_port")); - assertTrue(props.containsKey("netflow.destination_transport_port")); - assertTrue(props.containsKey("netflow.destination_ipv4_address")); - assertTrue(props.containsKey("destination.ip")); - assertTrue(props.containsKey("destination.port")); - assertTrue(props.containsKey("source.ip")); - assertTrue(props.containsKey("source.port")); - - // Get applied mappings - props = getIndexMappingsSAFlat(datastream); - assertTrue(props.containsKey("destination.ip")); - assertTrue(props.containsKey("destination.port")); - assertTrue(props.containsKey("source.ip")); - assertTrue(props.containsKey("source.port")); - - deleteDatastreamAPI(datastream); - } - - public void testCreateMappings_withIndexPattern_existing_indexTemplate_update_success() throws IOException { - String indexName1 = "test_index_1"; - String indexName2 = "test_index_2"; - String indexName3 = "test_index_3"; - - String indexPattern = "test_index*"; - - String componentTemplateMappings = "\"properties\": {" + - " \"netflow.destination_transport_port\":{ \"type\": \"long\" }," + - " \"netflow.destination_ipv4_address\":{ \"type\": \"ip\" }" + - "}"; - - // Setup index_template - createComponentTemplateWithMappings( - IndexTemplateManager.computeComponentTemplateName(indexPattern), - componentTemplateMappings - ); - - createComposableIndexTemplate( - IndexTemplateManager.computeIndexTemplateName(indexPattern), - List.of(indexPattern), - IndexTemplateManager.computeComponentTemplateName(indexPattern), - false - ); - - createIndex(indexName1, Settings.EMPTY, null); - - // Execute CreateMappingsAction to apply alias mappings - index template should be updated - createMappingsAPI(indexPattern, "netflow"); - - // Create new index to verify that index template is updated - createIndex(indexName2, Settings.EMPTY, null); - - // Verify that template applied mappings - Map props = getIndexMappingsAPIFlat(indexName2); - assertEquals(4, props.size()); - assertTrue(props.containsKey("netflow.destination_transport_port")); - assertTrue(props.containsKey("netflow.destination_ipv4_address")); - assertTrue(props.containsKey("destination.ip")); - assertTrue(props.containsKey("destination.port")); - - // Verify our GetIndexMappings -- applied mappings - props = getIndexMappingsSAFlat(indexPattern); - assertEquals(2, props.size()); - assertTrue(props.containsKey("destination.ip")); - assertTrue(props.containsKey("destination.port")); - - - // Insert doc to index to add additional fields to mapping - String sampleDoc = "{" + - " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + - " \"netflow.source_transport_port\":4444" + - "}"; - - indexDoc(indexName2, "1", sampleDoc); - - // Call CreateMappings API and expect index template to be updated with 2 additional aliases - createMappingsAPI(indexPattern, "netflow"); - - // Create new index to verify that index template was updated correctly - createIndex(indexName3, Settings.EMPTY, null); - - // Verify mappings - props = getIndexMappingsAPIFlat(indexName3); - assertEquals(8, props.size()); - assertTrue(props.containsKey("source.ip")); - assertTrue(props.containsKey("destination.ip")); - assertTrue(props.containsKey("source.port")); - assertTrue(props.containsKey("destination.port")); - assertTrue(props.containsKey("netflow.source_transport_port")); - assertTrue(props.containsKey("netflow.source_ipv4_address")); - assertTrue(props.containsKey("netflow.destination_transport_port")); - assertTrue(props.containsKey("netflow.destination_ipv4_address")); - - // Verify our GetIndexMappings -- applied mappings - props = getIndexMappingsSAFlat(indexPattern); - assertEquals(4, props.size()); - assertTrue(props.containsKey("source.ip")); - assertTrue(props.containsKey("destination.ip")); - assertTrue(props.containsKey("source.port")); - assertTrue(props.containsKey("destination.port")); - } - - public void testCreateMappings_withIndexPattern_differentMappings_success() throws IOException { - String indexName1 = "test_index_1"; - String indexName2 = "test_index_2"; - String indexPattern = "test_index*"; - - createIndex(indexName1, Settings.EMPTY, null); - createIndex(indexName2, Settings.EMPTY, null); - - client().performRequest(new Request("POST", "_refresh")); - - // Insert sample docs - String sampleDoc1 = "{" + - " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + - " \"netflow.destination_transport_port\":1234," + - " \"netflow.source_transport_port\":4444" + - "}"; - String sampleDoc2 = "{" + - " \"netflow.destination_transport_port\":1234," + - " \"netflow.destination_ipv4_address\":\"10.53.111.14\"" + - "}"; - indexDoc(indexName1, "1", sampleDoc1); - indexDoc(indexName2, "1", sampleDoc2); - - client().performRequest(new Request("POST", "_refresh")); - - // Execute CreateMappingsAction to add alias mapping for index - createMappingsAPI(indexPattern, "netflow"); - } - - public void testCreateMappings_withIndexPattern_indexTemplate_createAndUpdate_success() throws IOException { - String indexName1 = "test_index_1"; - String indexName2 = "test_index_2"; - String indexName3 = "test_index_3"; - String indexName4 = "test_index_4"; - - String indexPattern = "test_index*"; - - createIndex(indexName1, Settings.EMPTY, null); - createIndex(indexName2, Settings.EMPTY, null); - - client().performRequest(new Request("POST", "_refresh")); - - // Insert sample doc - String sampleDoc1 = "{" + - " \"netflow.destination_transport_port\":1234," + - " \"netflow.destination_ipv4_address\":\"10.53.111.14\"" + - "}"; - - indexDoc(indexName1, "1", sampleDoc1); - indexDoc(indexName2, "1", sampleDoc1); - - client().performRequest(new Request("POST", "_refresh")); - - // Execute CreateMappingsAction to add alias mapping for index - createMappingsAPI(indexPattern, "netflow"); - - // Verify that index template is up - createIndex(indexName3, Settings.EMPTY, null); - - // Execute CreateMappingsAction to add alias mapping for index - Request request = new Request("GET", indexName3 + "/_mapping"); - Response response = client().performRequest(request); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); - Map respMap = (Map) responseAsMap(response).get(indexName3); - - MappingsTraverser mappingsTraverser = new MappingsTraverser((Map) respMap.get("mappings"), Set.of()); - Map flatMappings = mappingsTraverser.traverseAndCopyAsFlat(); - // Verify mappings - Map props = (Map) flatMappings.get("properties"); - assertEquals(4, props.size()); - assertTrue(props.containsKey("destination.ip")); - assertTrue(props.containsKey("destination.port")); - assertTrue(props.containsKey("netflow.destination_transport_port")); - assertTrue(props.containsKey("netflow.destination_ipv4_address")); - - String sampleDoc2 = "{" + - " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + - " \"netflow.destination_transport_port\":1234," + - " \"netflow.destination_ipv4_address\":\"10.53.111.14\"," + - " \"netflow.source_transport_port\":4444" + - "}"; - - indexDoc(indexName3, "1", sampleDoc2); - - // Execute CreateMappingsAction to add alias mapping for index - createMappingsAPI(indexPattern, "netflow"); - - // Verify that index template is updated - createIndex(indexName4, Settings.EMPTY, null); - - // Execute CreateMappingsAction to add alias mapping for index - request = new Request("GET", indexName4 + "/_mapping"); - response = client().performRequest(request); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); - respMap = (Map) responseAsMap(response).get(indexName4); - - mappingsTraverser = new MappingsTraverser((Map) respMap.get("mappings"), Set.of()); - flatMappings = mappingsTraverser.traverseAndCopyAsFlat(); - // Verify mappings - props = (Map) flatMappings.get("properties"); - assertEquals(8, props.size()); - assertTrue(props.containsKey("source.ip")); - assertTrue(props.containsKey("destination.ip")); - assertTrue(props.containsKey("source.port")); - assertTrue(props.containsKey("destination.port")); - assertTrue(props.containsKey("netflow.source_transport_port")); - assertTrue(props.containsKey("netflow.source_ipv4_address")); - assertTrue(props.containsKey("netflow.destination_transport_port")); - assertTrue(props.containsKey("netflow.destination_ipv4_address")); - - // Verify applied mappings - props = getIndexMappingsSAFlat(indexName4); - assertEquals(4, props.size()); - assertTrue(props.containsKey("source.ip")); - assertTrue(props.containsKey("destination.ip")); - assertTrue(props.containsKey("source.port")); - assertTrue(props.containsKey("destination.port")); - } - - public void testCreateMappings_withIndexPattern_oneNoMatches_success() throws IOException { - String indexName1 = "test_index_1"; - String indexName2 = "test_index_2"; - String indexPattern = "test_index*"; - - createIndex(indexName1, Settings.EMPTY, null); - createIndex(indexName2, Settings.EMPTY, null); - - client().performRequest(new Request("POST", "_refresh")); - - // Insert sample docs - String sampleDoc1 = "{" + - " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + - " \"netflow.destination_transport_port\":1234," + - " \"netflow.source_transport_port\":4444" + - "}"; - String sampleDoc2 = "{" + - " \"netflow11.destination33_transport_port\":1234," + - " \"netflow11.destination33_ipv4_address\":\"10.53.111.14\"" + - "}"; - indexDoc(indexName1, "1", sampleDoc1); - indexDoc(indexName2, "1", sampleDoc2); - - client().performRequest(new Request("POST", "_refresh")); - - // Execute CreateMappingsAction to add alias mapping for index - createMappingsAPI(indexPattern, "netflow"); - } - - public void testCreateMappings_withIndexPattern_oneNoMappings_failure() throws IOException { - String indexName1 = "test_index_1"; - String indexName2 = "test_index_2"; - String indexPattern = "test_index*"; - - createIndex(indexName1, Settings.EMPTY, null); - createIndex(indexName2, Settings.EMPTY, null); - - client().performRequest(new Request("POST", "_refresh")); - - // Insert sample docs - String sampleDoc1 = "{" + - " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + - " \"netflow.destination_transport_port\":1234," + - " \"netflow.source_transport_port\":4444" + - "}"; - indexDoc(indexName1, "1", sampleDoc1); - - client().performRequest(new Request("POST", "_refresh")); - - // Execute CreateMappingsAction to add alias mapping for index - try { - createMappingsAPI(indexPattern, "netflow"); - fail("expected 500 failure!"); - } catch (ResponseException e) { - assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getResponse().getStatusLine().getStatusCode()); - } - + assertEquals(3, unmappedFieldAliases.size()); } public void testGetMappingsView_index_pattern_two_indices_Success() throws IOException { @@ -652,7 +325,7 @@ public void testGetMappingsView_index_pattern_two_indices_Success() throws IOExc assertTrue(extraField.isPresent()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(2, unmappedFieldAliases.size()); + assertEquals(3, unmappedFieldAliases.size()); } public void testGetMappingsView_alias_without_writeindex_Success() throws IOException { @@ -687,7 +360,7 @@ public void testGetMappingsView_alias_without_writeindex_Success() throws IOExce assertTrue(extraField.isPresent()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(2, unmappedFieldAliases.size()); + assertEquals(3, unmappedFieldAliases.size()); } public void testGetMappingsView_alias_with_writeindex_Success() throws IOException { @@ -725,7 +398,7 @@ public void testGetMappingsView_alias_with_writeindex_Success() throws IOExcepti assertTrue(extraField.isPresent()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(2, unmappedFieldAliases.size()); + assertEquals(3, unmappedFieldAliases.size()); } public void testGetMappingsView_datastream_one_backing_index_Success() throws IOException { @@ -752,7 +425,7 @@ public void testGetMappingsView_datastream_one_backing_index_Success() throws IO assertEquals(7, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(2, unmappedFieldAliases.size()); + assertEquals(3, unmappedFieldAliases.size()); deleteDatastream(datastreamName); } @@ -802,11 +475,10 @@ public void testGetMappingsView_datastream_two_backing_index_Success() throws IO assertEquals(1, props.size()); assertTrue(props.containsKey("source.ip")); // Verify unmapped index fields - List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); - assertEquals(1, unmappedIndexFields.size()); + // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(5, unmappedFieldAliases.size()); + assertEquals(6, unmappedFieldAliases.size()); deleteDatastream(datastreamName); } @@ -847,6 +519,120 @@ public void testCreateMappings_withIndexPattern_success() throws IOException { assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } + public void testCreateMappings_withIndexPattern_differentMappings_success() throws IOException { + String indexName1 = "test_index_1"; + String indexName2 = "test_index_2"; + String indexPattern = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample docs + String sampleDoc1 = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.source_transport_port\":4444" + + "}"; + String sampleDoc2 = "{" + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.destination_ipv4_address\":\"10.53.111.14\"" + + "}"; + indexDoc(indexName1, "1", sampleDoc1); + indexDoc(indexName2, "1", sampleDoc2); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction to add alias mapping for index + Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + request.setJsonEntity( + "{ \"index_name\":\"" + indexPattern + "\"," + + " \"rule_topic\":\"netflow\", " + + " \"partial\":true" + + "}" + ); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + + public void testCreateMappings_withIndexPattern_oneNoMatches_success() throws IOException { + String indexName1 = "test_index_1"; + String indexName2 = "test_index_2"; + String indexPattern = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample docs + String sampleDoc1 = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.source_transport_port\":4444" + + "}"; + String sampleDoc2 = "{" + + " \"netflow11.destination33_transport_port\":1234," + + " \"netflow11.destination33_ipv4_address\":\"10.53.111.14\"" + + "}"; + indexDoc(indexName1, "1", sampleDoc1); + indexDoc(indexName2, "1", sampleDoc2); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction to add alias mapping for index + Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + request.setJsonEntity( + "{ \"index_name\":\"" + indexPattern + "\"," + + " \"rule_topic\":\"netflow\", " + + " \"partial\":true" + + "}" + ); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + + public void testCreateMappings_withIndexPattern_oneNoMappings_failure() throws IOException { + String indexName1 = "test_index_1"; + String indexName2 = "test_index_2"; + String indexPattern = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample docs + String sampleDoc1 = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.source_transport_port\":4444" + + "}"; + indexDoc(indexName1, "1", sampleDoc1); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction to add alias mapping for index + Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + request.setJsonEntity( + "{ \"index_name\":\"" + indexPattern + "\"," + + " \"rule_topic\":\"netflow\", " + + " \"partial\":true" + + "}" + ); + try { + client().performRequest(request); + fail("expected 500 failure!"); + } catch (ResponseException e) { + assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getResponse().getStatusLine().getStatusCode()); + } + + } + private void createSampleIndex(String indexName) throws IOException { createSampleIndex(indexName, Settings.EMPTY, null); } @@ -887,20 +673,20 @@ private void createSampleIndex(String indexName, Settings settings, String alias " \"keyword\":{" + " \"type\":\"keyword\"," + " \"ignore_above\":256" + - "}" + - "}" + - "}," + + "}" + + "}" + + "}," + " \"last\":{" + - "\"type\":\"text\"," + - "\"fields\":{" + + "\"type\":\"text\"," + + "\"fields\":{" + " \"keyword\":{" + " \"type\":\"keyword\"," + " \"ignore_above\":256" + - "}" + - "}" + - "}" + - "}" + - "}" + + "}" + + "}" + + "}" + + "}" + + "}" + " }"; createIndex(indexName, settings, indexMapping, aliases); @@ -1152,7 +938,6 @@ public void testCreateCloudTrailMapping() throws IOException { response = client().performRequest(new Request("POST", "_refresh")); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } - public void testCreateDNSMapping() throws IOException{ String INDEX_NAME = "test_create_cloudtrail_mapping_index"; @@ -1184,6 +969,8 @@ public void testCreateDNSMapping() throws IOException{ //Loop over the mappings and run update request for each one specifying the index to be updated mappings.entrySet().forEach(entry -> { String key = entry.getKey(); + if("timestamp".equals(key)) + return; String path = ((Map) entry.getValue()).get("path").toString(); try { Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); @@ -1204,88 +991,4 @@ public void testCreateDNSMapping() throws IOException{ assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } - - public void testTraverseAndCopy() { - - try { - String indexName = "my_test_index"; - - String indexMappingJSON = "" + - " \"properties\": {" + - " \"netflow.event_data.SourceAddress\": {" + - " \"type\": \"ip\"" + - " }," + - " \"type\": {" + - " \"type\": \"integer\"" + - " }," + - " \"netflow.event_data.DestinationPort\": {" + - " \"type\": \"integer\"" + - " }," + - " \"netflow.event.stop\": {" + - " \"type\": \"integer\"" + - " }," + - " \"netflow.event.start\": {" + - " \"type\": \"long\"" + - " }," + - " \"plain1\": {" + - " \"type\": \"integer\"" + - " }," + - " \"user\":{" + - " \"type\":\"nested\"," + - " \"properties\":{" + - " \"first\":{" + - " \"type\":\"long\"" + - " }," + - " \"last\":{" + - " \"type\":\"text\"," + - " \"fields\":{" + - " \"keyword\":{" + - " \"type\":\"keyword\"," + - " \"ignore_above\":256" + - " }" + - " }" + - " }" + - " }" + - " }" + - "}"; - - createIndex(indexName, Settings.EMPTY, indexMappingJSON); - - Map mappings = getIndexMappingsAPI(indexName); - - MappingsTraverser mappingsTraverser; - - mappingsTraverser = new MappingsTraverser(mappings, Set.of()); - - // Copy specific paths from mappings - Map filteredMappings = mappingsTraverser.traverseAndCopyWithFilter( - List.of("netflow.event_data.SourceAddress", "netflow.event.stop", "plain1", "user.first", "user.last") - ); - - // Now traverse filtered mapppings to confirm only copied paths are present - List paths = new ArrayList<>(); - mappingsTraverser = new MappingsTraverser(filteredMappings, Set.of()); - mappingsTraverser.addListener(new MappingsTraverser.MappingsTraverserListener() { - @Override - public void onLeafVisited(MappingsTraverser.Node node) { - paths.add(node.currentPath); - } - - @Override - public void onError(String error) { - fail("Failed traversing valid mappings"); - } - }); - mappingsTraverser.traverse(); - assertEquals(5, paths.size()); - assertTrue(paths.contains("user.first")); - assertTrue(paths.contains("user.last")); - assertTrue(paths.contains("plain1")); - assertTrue(paths.contains("netflow.event.stop")); - assertTrue(paths.contains("netflow.event_data.SourceAddress")); - - } catch (IOException e) { - fail("Error instantiating MappingsTraverser with JSON string as mappings"); - } - } } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index 3bfaa0930..e20f69161 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -17,8 +17,10 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; +import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.client.ResponseException; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.commons.alerting.model.Monitor.MonitorType; import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; @@ -37,17 +39,7 @@ import java.util.stream.Collectors; import org.opensearch.securityanalytics.model.DetectorTrigger; -import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule; -import static org.opensearch.securityanalytics.TestHelpers.productIndexMapping; -import static org.opensearch.securityanalytics.TestHelpers.randomDetector; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; -import static org.opensearch.securityanalytics.TestHelpers.randomDoc; -import static org.opensearch.securityanalytics.TestHelpers.randomIndex; -import static org.opensearch.securityanalytics.TestHelpers.randomProductDocument; -import static org.opensearch.securityanalytics.TestHelpers.randomRule; -import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.TestHelpers.*; public class DetectorRestApiIT extends SecurityAnalyticsRestTestCase { @@ -156,10 +148,10 @@ public void testCreateDetectorWithoutRules() throws IOException { Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response createMappingResponse = client().performRequest(createMappingRequest); @@ -174,11 +166,11 @@ public void testCreateDetectorWithoutRules() throws IOException { // Verify rules String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); Assert.assertEquals(0, response.getHits().getTotalHits().value); @@ -341,10 +333,10 @@ public void testCreatingADetectorWithAggregationRules() throws IOException { Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"test_windows\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"test_windows\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -353,7 +345,7 @@ public void testCreatingADetectorWithAggregationRules() throws IOException { String customAvgRuleId = createRule(productIndexAvgAggRule()); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(customAvgRuleId)), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -371,12 +363,12 @@ public void testCreatingADetectorWithAggregationRules() throws IOException { Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -699,4 +691,201 @@ public void testDeletingANonExistingDetector() throws IOException { Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode()); } } + + public void testCreatingADetectorWithTimestampFieldAliasMapping() throws IOException { + String index = createTestIndex(randomIndex(), productIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"test_windows\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + updateRequest.setJsonEntity(Strings.toString(XContentFactory.jsonBuilder().map(Map.of( + "index_name", index, + "field", "time", + "alias", "timestamp")))); + Response apiResponse = client().performRequest(updateRequest); + assertEquals(HttpStatus.SC_OK, apiResponse.getStatusLine().getStatusCode()); + + String customAvgRuleId = createRule(productIndexAvgAggRule()); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(customAvgRuleId)), + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, detectorId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, detectorId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + List monitorTypes = new ArrayList<>(); + + Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); + + String bucketLevelMonitorId = ""; + + // Verify that doc level monitor is created + List monitorIds = (List) (detectorAsMap).get("monitor_id"); + + String firstMonitorId = monitorIds.get(0); + String firstMonitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + firstMonitorId))).get("monitor")).get("monitor_type"); + + if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(firstMonitorType)){ + bucketLevelMonitorId = firstMonitorId; + } + monitorTypes.add(firstMonitorType); + + String secondMonitorId = monitorIds.get(1); + String secondMonitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + secondMonitorId))).get("monitor")).get("monitor_type"); + monitorTypes.add(secondMonitorType); + if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(secondMonitorType)){ + bucketLevelMonitorId = secondMonitorId; + } + Assert.assertTrue(Arrays.asList(MonitorType.BUCKET_LEVEL_MONITOR.getValue(), MonitorType.DOC_LEVEL_MONITOR.getValue()).containsAll(monitorTypes)); + + indexDoc(index, "1", randomProductDocumentWithTime(System.currentTimeMillis())); + + Response executeResponse = executeAlertingMonitor(bucketLevelMonitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + // verify bucket level monitor findings + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + assertNotNull(getFindingsBody); + Assert.assertEquals(1, getFindingsBody.get("total_findings")); + List findings = (List) getFindingsBody.get("findings"); + Assert.assertEquals(findings.size(), 1); + HashMap finding = (HashMap) findings.get(0); + Assert.assertTrue(finding.containsKey("queries")); + HashMap docLevelQuery = (HashMap) ((List) finding.get("queries")).get(0); + String ruleId = docLevelQuery.get("id").toString(); + // Verify if the rule id in bucket level finding is the same as rule used for bucket monitor creation + assertEquals(customAvgRuleId, ruleId); + Response getResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), null); + String getDetectorResponseString = new String(getResponse.getEntity().getContent().readAllBytes()); + Assert.assertTrue(getDetectorResponseString.contains(ruleId)); + } + + public void testCreatingADetectorWithTimestampFieldAliasMapping_verifyTimeRangeInBucketMonitor() throws IOException { + String index = createTestIndex(randomIndex(), productIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"test_windows\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + updateRequest.setJsonEntity(Strings.toString(XContentFactory.jsonBuilder().map(Map.of( + "index_name", index, + "field", "time", + "alias", "timestamp")))); + Response apiResponse = client().performRequest(updateRequest); + assertEquals(HttpStatus.SC_OK, apiResponse.getStatusLine().getStatusCode()); + + String customAvgRuleId = createRule(productIndexAvgAggRule()); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(customAvgRuleId)), + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, detectorId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, detectorId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + List monitorTypes = new ArrayList<>(); + + Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); + + String bucketLevelMonitorId = ""; + + // Verify that doc level monitor is created + List monitorIds = (List) (detectorAsMap).get("monitor_id"); + + String firstMonitorId = monitorIds.get(0); + String firstMonitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + firstMonitorId))).get("monitor")).get("monitor_type"); + + if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(firstMonitorType)){ + bucketLevelMonitorId = firstMonitorId; + } + monitorTypes.add(firstMonitorType); + + String secondMonitorId = monitorIds.get(1); + String secondMonitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + secondMonitorId))).get("monitor")).get("monitor_type"); + monitorTypes.add(secondMonitorType); + if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(secondMonitorType)){ + bucketLevelMonitorId = secondMonitorId; + } + Assert.assertTrue(Arrays.asList(MonitorType.BUCKET_LEVEL_MONITOR.getValue(), MonitorType.DOC_LEVEL_MONITOR.getValue()).containsAll(monitorTypes)); + + indexDoc(index, "1", randomProductDocumentWithTime(System.currentTimeMillis()-1000*60*70)); // doc's timestamp is older than 1 hr + + Response executeResponse = executeAlertingMonitor(bucketLevelMonitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + // verify bucket level monitor findings + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + assertNotNull(getFindingsBody); + Assert.assertEquals(0, getFindingsBody.get("total_findings")); + List findings = (List) getFindingsBody.get("findings"); + Assert.assertEquals(findings.size(), 0); //there should be no findings as doc is not in time range of current run + } } \ No newline at end of file