diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml new file mode 100644 index 000000000..9dcc7020d --- /dev/null +++ b/.github/workflows/add-untriaged.yml @@ -0,0 +1,19 @@ +name: Apply 'untriaged' label during issue lifecycle + +on: + issues: + types: [opened, reopened, transferred] + +jobs: + apply-label: + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v6 + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['untriaged'] + }) diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 000000000..3bf1986a0 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,7 @@ +@amsiglan +@awshurneyt +@getsaurabh02 +@lezzago +@praveensameneni +@sbcd90 +@eirsep diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 32bd185fb..a8735dcce 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -4,8 +4,12 @@ This document contains a list of maintainers in this repo. See [opensearch-proje ## Current Maintainers -| Maintainer | GitHub ID | Affiliation | -| ---------------------- | ----------------------------------------------- | ----------- | -| Saurabh Singh | [getsaurabh02](https://github.com/getsaurabh02) | Amazon | -| Subhobrata Dey | [sbcd90](https://github.com/sbcd90) | Amazon | -| Surya Sashank Nistalai | [eirsep](https://github.com/eirsep) | Amazon | +| Maintainer | GitHub ID | Affiliation | +| ---------------- | ----------------------------------------------------- | ----------- | +| Ashish Agrawal | [lezzago](https://github.com/lezzago) | Amazon | +| Subhobrata Dey | [sbcd90](https://github.com/sbcd90) | Amazon | +| Thomas Hurney | [awshurneyt](https://github.com/AWSHurneyt) | Amazon | +| Surya Sashank Nistala | [eirsep](https://github.com/eirsep) | Amazon | +| Praveen Sameneni | [praveensameneni](https://github.com/praveensameneni) | Amazon | +| Amardeepsingh Siglani | [amsiglan](https://github.com/amsiglan) | Amazon | +| Saurabh Singh | [getsaurabh02](https://github.com/getsaurabh02) | Amazon | diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/IndexTemplateManager.java b/src/main/java/org/opensearch/securityanalytics/mapper/IndexTemplateManager.java index bbdd06fe1..1ba9a1825 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/IndexTemplateManager.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/IndexTemplateManager.java @@ -99,6 +99,7 @@ public void onResponse(AcknowledgedResponse acknowledgedResponse) { if (acknowledgedResponse.isAcknowledged() == false) { log.warn("Upserting component template not ack'd!"); } + boolean updateConflictingTemplate = false; // Find template which matches input index best String templateName = MetadataIndexTemplateService.findV2Template( @@ -106,12 +107,39 @@ public void onResponse(AcknowledgedResponse acknowledgedResponse) { normalizeIndexName(indexName), false ); + // If we find conflicting templates(regardless of priority) and that template was created by us, + // we will silently update index_pattern of that template. + // Otherwise, we will fail since we don't want to change index_pattern of user created index template + Map> conflictingTemplates = + MetadataIndexTemplateService.findConflictingV2Templates( + state, + computeIndexTemplateName(indexName), + List.of(computeIndexPattern(indexName)) + ); + + // If there is 1 conflict with our own template, we will update that template's index_pattern field + if (conflictingTemplates.size() == 1) { + String conflictingTemplateName = conflictingTemplates.keySet().iterator().next(); + if (conflictingTemplateName.startsWith(OPENSEARCH_SAP_INDEX_TEMPLATE_PREFIX)) { + templateName = conflictingTemplateName; + updateConflictingTemplate = true; + } + } + + if (templateName == null && conflictingTemplates.size() > 0) { + String errorMessage = "Found conflicting templates: [" + + String.join(", ", conflictingTemplates.keySet()) + "]"; + log.error(errorMessage); + actionListener.onFailure(SecurityAnalyticsException.wrap(new IllegalStateException(errorMessage))); + return; + } + String componentName = computeComponentTemplateName(indexName); ComposableIndexTemplate template; if (templateName == null) { template = new ComposableIndexTemplate( - List.of(indexName.endsWith("*") == false ? indexName + "*": indexName), + List.of(computeIndexPattern(indexName)), null, List.of(componentName), null, @@ -123,10 +151,18 @@ public void onResponse(AcknowledgedResponse acknowledgedResponse) { template = state.metadata().templatesV2().get(templateName); // Check if we need to append our component to composedOf list if (template.composedOf().contains(componentName) == false) { - List newComposedOf = new ArrayList<>(template.composedOf()); - newComposedOf.add(componentName); + List newComposedOf; + List indexPatterns; + if (updateConflictingTemplate) { + newComposedOf = new ArrayList<>(template.composedOf()); + newComposedOf.add(componentName); + indexPatterns = List.of(computeIndexPattern(indexName)); + } else { + newComposedOf = List.of(componentName); + indexPatterns = template.indexPatterns(); + } template = new ComposableIndexTemplate( - template.indexPatterns(), + indexPatterns, template.template(), newComposedOf, template.priority(), @@ -155,6 +191,10 @@ public void onFailure(Exception e) { } + private String computeIndexPattern(String indexName) { + return indexName.endsWith("*") == false ? indexName + "*" : indexName; + } + private void upsertIndexTemplate( IndicesAdminClient indicesClient, boolean create, diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index d27494942..0f55daecd 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -5,9 +5,16 @@ package org.opensearch.securityanalytics.mapper; +import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -21,21 +28,12 @@ import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.IndicesAdminClient; -import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.collect.ImmutableOpenMap; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.rest.RestStatus; import org.opensearch.securityanalytics.action.GetIndexMappingsResponse; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import org.opensearch.securityanalytics.action.GetMappingsViewResponse; import org.opensearch.securityanalytics.model.CreateMappingResult; import org.opensearch.securityanalytics.util.IndexUtils; @@ -248,7 +246,7 @@ public void getMappingAction(String indexName, ActionListener() { @Override public void onResponse(String concreteIndex) { - doGetMappingAction(concreteIndex, actionListener); + doGetMappingAction(indexName, concreteIndex, actionListener); } @Override @@ -263,17 +261,16 @@ public void onFailure(Exception e) { } } - public void doGetMappingAction(String indexName, ActionListener actionListener) { - GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(indexName); + public void doGetMappingAction(String indexName, String concreteIndexName, ActionListener actionListener) { + GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(concreteIndexName); indicesClient.getMappings(getMappingsRequest, new ActionListener<>() { @Override public void onResponse(GetMappingsResponse getMappingsResponse) { try { - // Extract indexName and MappingMetadata - String indexName = getMappingsResponse.mappings().iterator().next().key; + // Extract MappingMetadata MappingMetadata mappingMetadata = getMappingsResponse.mappings().iterator().next().value; // List of all found applied aliases on index - List appliedAliases = new ArrayList<>(); + Set appliedAliases = new HashSet<>(); // Get list of alias -> path pairs from index mappings List> indexAliasPathPairs = MapperUtils.getAllAliasPathPairs(mappingMetadata); @@ -293,10 +290,6 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { } } } - // If we found all aliases we can stop searching further - if (indexAliasPathPairs.size() == appliedAliases.size()) { - break; - } } if (appliedAliases.size() == 0) { diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MappingsTraverser.java b/src/main/java/org/opensearch/securityanalytics/mapper/MappingsTraverser.java index 88d183480..16335894b 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MappingsTraverser.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MappingsTraverser.java @@ -214,7 +214,7 @@ private boolean shouldSkipNode(Map properties) { return false; } - public Map traverseAndCopyWithFilter(List nodePathsToCopy) { + public Map traverseAndCopyWithFilter(Set nodePathsToCopy) { Map outRoot = new LinkedHashMap<>(Map.of(PROPERTIES, new LinkedHashMap())); this.addListener(new MappingsTraverserListener() { diff --git a/src/main/java/org/opensearch/securityanalytics/rules/objects/SigmaRule.java b/src/main/java/org/opensearch/securityanalytics/rules/objects/SigmaRule.java index 4b507a863..6236713d1 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/objects/SigmaRule.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/objects/SigmaRule.java @@ -11,8 +11,11 @@ import org.opensearch.securityanalytics.rules.exceptions.SigmaLevelError; import org.opensearch.securityanalytics.rules.exceptions.SigmaLogsourceError; import org.opensearch.securityanalytics.rules.exceptions.SigmaStatusError; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.LoaderOptions; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.SafeConstructor; +import org.yaml.snakeyaml.representer.Representer; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -168,7 +171,10 @@ protected static SigmaRule fromDict(Map rule, boolean collectErr } public static SigmaRule fromYaml(String rule, boolean collectErrors) throws SigmaError { - Yaml yaml = new Yaml(new SafeConstructor()); + LoaderOptions loaderOptions = new LoaderOptions(); + loaderOptions.setNestingDepthLimit(10); + + Yaml yaml = new Yaml(new SafeConstructor(), new Representer(), new DumperOptions(), loaderOptions); Map ruleMap = yaml.load(rule); return fromDict(ruleMap, collectErrors); } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 7410e9c3e..e882a9ff8 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -4,18 +4,6 @@ */ package org.opensearch.securityanalytics.transport; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -41,6 +29,8 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.io.stream.NamedWriteableRegistry; @@ -59,6 +49,7 @@ import org.opensearch.commons.alerting.action.IndexMonitorRequest; import org.opensearch.commons.alerting.action.IndexMonitorResponse; import org.opensearch.commons.alerting.model.BucketLevelTrigger; +import org.opensearch.commons.alerting.model.CronSchedule; import org.opensearch.commons.alerting.model.DataSources; import org.opensearch.commons.alerting.model.DocLevelMonitorInput; import org.opensearch.commons.alerting.model.DocLevelQuery; @@ -69,8 +60,10 @@ import org.opensearch.commons.alerting.model.action.Action; import org.opensearch.commons.authuser.User; import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.RangeQueryBuilder; import org.opensearch.index.reindex.BulkByScrollResponse; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.rest.RestRequest; @@ -80,11 +73,15 @@ import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.action.GetIndexMappingsAction; +import org.opensearch.securityanalytics.action.GetIndexMappingsRequest; +import org.opensearch.securityanalytics.action.GetIndexMappingsResponse; import org.opensearch.securityanalytics.action.IndexDetectorAction; import org.opensearch.securityanalytics.action.IndexDetectorRequest; import org.opensearch.securityanalytics.action.IndexDetectorResponse; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.mapper.MapperService; +import org.opensearch.securityanalytics.mapper.MapperUtils; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; @@ -105,10 +102,24 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + public class TransportIndexDetectorAction extends HandledTransportAction implements SecureTransportAction { public static final String PLUGIN_OWNER_FIELD = "security_analytics"; private static final Logger log = LogManager.getLogger(TransportIndexDetectorAction.class); + public static final String TIMESTAMP_FIELD_ALIAS = "timestamp"; private final Client client; @@ -133,6 +144,8 @@ public class TransportIndexDetectorAction extends HandledTransportAction> pairs = MapperUtils.getAllAliasPathPairs(mappingMetadata); + boolean timeStampAliasPresent = pairs. + stream() + .anyMatch(p -> + TIMESTAMP_FIELD_ALIAS.equals(p.getLeft()) || TIMESTAMP_FIELD_ALIAS.equals(p.getRight())); + if(timeStampAliasPresent) { + BoolQueryBuilder boolQueryBuilder = searchSourceBuilder.query() == null + ? new BoolQueryBuilder() + : QueryBuilders.boolQuery().must(searchSourceBuilder.query()); + RangeQueryBuilder timeRangeFilter = QueryBuilders.rangeQuery(TIMESTAMP_FIELD_ALIAS) + .gt("{{period_end}}||-1h") + .lte("{{period_end}}") + .format("epoch_millis"); + boolQueryBuilder.must(timeRangeFilter); + searchSourceBuilder.query(boolQueryBuilder); + } + } catch (Exception e) { + log.error( + String.format(Locale.getDefault(), + "Unable to verify presence of timestamp alias for index [%s] in detector [%s]. Not setting time range filter for bucket level monitor.", + concreteIndex, detector.getName()), e); + } List bucketLevelMonitorInputs = new ArrayList<>(); bucketLevelMonitorInputs.add(new SearchInput(Arrays.asList(index), searchSourceBuilder)); diff --git a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java index 4388d3a44..3358f5d12 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java @@ -7,11 +7,13 @@ import java.util.SortedMap; import org.opensearch.action.ActionListener; import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.IndicesAdminClient; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexAbstraction; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.NamedXContentRegistry; import org.opensearch.common.xcontent.XContentParser; @@ -154,4 +156,10 @@ public static String getNewestIndexByCreationDate(String[] concreteIndices, Clus } return newestIndex; } + + public static String getNewIndexByCreationDate(ClusterState state, IndexNameExpressionResolver i, String index) { + String[] strings = i.concreteIndexNames(state, IndicesOptions.LENIENT_EXPAND_OPEN, index); + return getNewestIndexByCreationDate(strings, state); + } + } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/util/SecurityAnalyticsException.java b/src/main/java/org/opensearch/securityanalytics/util/SecurityAnalyticsException.java index a5ed50f82..696862b25 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/SecurityAnalyticsException.java +++ b/src/main/java/org/opensearch/securityanalytics/util/SecurityAnalyticsException.java @@ -39,7 +39,7 @@ public RestStatus status() { } public static OpenSearchException wrap(Exception ex) { - log.error(String.format(Locale.getDefault(), "Security Analytics error: %s", ex.getMessage())); + log.error("Security Analytics error:", ex); String friendlyMsg = "Unknown error"; RestStatus status = RestStatus.INTERNAL_SERVER_ERROR; @@ -52,7 +52,7 @@ public static OpenSearchException wrap(Exception ex) { } public static OpenSearchException wrap(OpenSearchException ex) { - log.error(String.format(Locale.getDefault(), "Security Analytics error: %s", ex.getMessage())); + log.error("Security Analytics error:", ex); String friendlyMsg = "Unknown error"; RestStatus status = ex.status(); @@ -71,10 +71,10 @@ public static OpenSearchException wrap(List ex) { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); for (Exception e: ex) { builder.field("error", e.getMessage()); + log.error("Security Analytics error:", e); } builder.endObject(); String friendlyMsg = Strings.toString(builder); - log.error(String.format(Locale.getDefault(), "Security Analytics error: %s", friendlyMsg)); return new SecurityAnalyticsException(friendlyMsg, status, new Exception(String.format(Locale.getDefault(), "%s: %s", ex.getClass().getName(), friendlyMsg))); } catch (IOException e) { diff --git a/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml b/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml index 70ebfe168..d949a735e 100644 --- a/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml +++ b/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml @@ -1,2 +1,3 @@ fieldmappings: TargetUserName: winlog-event_data-TargetUserName + creationTime: timestamp \ No newline at end of file diff --git a/src/main/resources/OSMapping/ad_ldap/mappings.json b/src/main/resources/OSMapping/ad_ldap/mappings.json index f68692f5a..f51459bb7 100644 --- a/src/main/resources/OSMapping/ad_ldap/mappings.json +++ b/src/main/resources/OSMapping/ad_ldap/mappings.json @@ -3,6 +3,10 @@ "winlog-event_data-TargetUserName": { "path": "winlog.event_data.TargetUserName", "type": "alias" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/apache_access/fieldmappings.yml b/src/main/resources/OSMapping/apache_access/fieldmappings.yml index 9ab79f08f..9f649aeee 100644 --- a/src/main/resources/OSMapping/apache_access/fieldmappings.yml +++ b/src/main/resources/OSMapping/apache_access/fieldmappings.yml @@ -5,3 +5,4 @@ fieldmappings: fieldB: mappedB fieldA1: mappedA CommandLine: windows-event_data-CommandLine + creationTime: timestamp diff --git a/src/main/resources/OSMapping/apache_access/mappings.json b/src/main/resources/OSMapping/apache_access/mappings.json index ea77f2460..dc7fc31b5 100644 --- a/src/main/resources/OSMapping/apache_access/mappings.json +++ b/src/main/resources/OSMapping/apache_access/mappings.json @@ -7,6 +7,10 @@ "event_uid": { "type": "alias", "path": "EventID" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml b/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml index eb6350d9e..6399929b6 100644 --- a/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml +++ b/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml @@ -7,4 +7,5 @@ fieldmappings: requestParameters.containerDefinitions.command: aws-cloudtrail-requestParameters-containerDefinitions-command userIdentity.sessionContext.sessionIssuer.type: userIdentity-sessionContext-sessionIssuer-type userIdentity.type: aws-cloudtrail-userIdentity-type - userIdentity.arn: aws-cloudtrail-userIdentity-arn \ No newline at end of file + userIdentity.arn: aws-cloudtrail-userIdentity-arn + eventTime: timestamp \ No newline at end of file diff --git a/src/main/resources/OSMapping/cloudtrail/mappings.json b/src/main/resources/OSMapping/cloudtrail/mappings.json index 7a9a35c47..ac8d1898c 100644 --- a/src/main/resources/OSMapping/cloudtrail/mappings.json +++ b/src/main/resources/OSMapping/cloudtrail/mappings.json @@ -35,6 +35,10 @@ "userIdentity-sessionContext-sessionIssuer-type": { "type": "alias", "path": "aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.type" + }, + "timestamp": { + "path": "aws.cloudtrail.eventTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/dns/fieldmappings.yml b/src/main/resources/OSMapping/dns/fieldmappings.yml index fc85e1ee7..7672ee77a 100644 --- a/src/main/resources/OSMapping/dns/fieldmappings.yml +++ b/src/main/resources/OSMapping/dns/fieldmappings.yml @@ -1,4 +1,5 @@ fieldmappings: record_type: dns-answers-type query: dns-question-name - parent_domain: dns-question-registered_domain \ No newline at end of file + parent_domain: dns-question-registered_domain + creationTime: timestamp \ No newline at end of file diff --git a/src/main/resources/OSMapping/dns/mappings.json b/src/main/resources/OSMapping/dns/mappings.json index 6f9f869ea..0905e2c9b 100644 --- a/src/main/resources/OSMapping/dns/mappings.json +++ b/src/main/resources/OSMapping/dns/mappings.json @@ -11,6 +11,10 @@ "dns-question-registered_domain": { "type": "alias", "path": "dns.question.registered_domain" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } diff --git a/src/main/resources/OSMapping/linux/fieldmappings.yml b/src/main/resources/OSMapping/linux/fieldmappings.yml index aedb0394f..d58617df6 100644 --- a/src/main/resources/OSMapping/linux/fieldmappings.yml +++ b/src/main/resources/OSMapping/linux/fieldmappings.yml @@ -12,4 +12,4 @@ fieldmappings: ParentImage: process-parent-executable CurrentDirectory: process-working_directory LogonId: process-real_user-id - + creationTime: timestamp diff --git a/src/main/resources/OSMapping/linux/mappings.json b/src/main/resources/OSMapping/linux/mappings.json index 17fb80188..5c69c0c48 100644 --- a/src/main/resources/OSMapping/linux/mappings.json +++ b/src/main/resources/OSMapping/linux/mappings.json @@ -47,6 +47,10 @@ "process-real_user-id": { "path": "process.real_user.id", "type": "alias" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/network/NetFlowMapping.json b/src/main/resources/OSMapping/network/NetFlowMapping.json index 2bc0d4d76..9e404a67e 100644 --- a/src/main/resources/OSMapping/network/NetFlowMapping.json +++ b/src/main/resources/OSMapping/network/NetFlowMapping.json @@ -23,6 +23,10 @@ "http.response.status_code": { "type": "alias", "path": "netflow.http_status_code" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/network/fieldmappings.yml b/src/main/resources/OSMapping/network/fieldmappings.yml index 2a2753832..2a10f24f4 100644 --- a/src/main/resources/OSMapping/network/fieldmappings.yml +++ b/src/main/resources/OSMapping/network/fieldmappings.yml @@ -14,4 +14,5 @@ fieldmappings: client_header_names: zeek-http-client_header_names resp_mime_types: zeek-http-resp_mime_types cipher: zeek-kerberos-cipher - request_type: zeek-kerberos-request_type \ No newline at end of file + request_type: zeek-kerberos-request_type + creationTime: timestamp \ No newline at end of file diff --git a/src/main/resources/OSMapping/network/mappings.json b/src/main/resources/OSMapping/network/mappings.json index 0a521ae75..97f5f5d74 100644 --- a/src/main/resources/OSMapping/network/mappings.json +++ b/src/main/resources/OSMapping/network/mappings.json @@ -59,6 +59,10 @@ "zeek-kerberos-request_type": { "path": "zeek.kerberos.request_type", "type": "alias" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } diff --git a/src/main/resources/OSMapping/others_application/fieldmappings.yml b/src/main/resources/OSMapping/others_application/fieldmappings.yml index e1dba4476..853eb1cba 100644 --- a/src/main/resources/OSMapping/others_application/fieldmappings.yml +++ b/src/main/resources/OSMapping/others_application/fieldmappings.yml @@ -1,7 +1,4 @@ # this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under application log group to their corresponding ECS Fields. fieldmappings: - EventID: event_uid - HiveName: unmapped.HiveName - fieldB: mappedB - fieldA1: mappedA - + Signature: abusech-malware-signature + Filename: file-name \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_application/mappings.json b/src/main/resources/OSMapping/others_application/mappings.json index 48cdda71d..33a27986b 100644 --- a/src/main/resources/OSMapping/others_application/mappings.json +++ b/src/main/resources/OSMapping/others_application/mappings.json @@ -1,28 +1,12 @@ { "properties": { - "windows-event_data-CommandLine": { + "abusech-malware-signature": { "type": "alias", - "path": "CommandLine" + "path": "abusech.malware.signature" }, - "event_uid": { + "file-name": { "type": "alias", - "path": "EventID" - }, - "windows-hostname": { - "type": "alias", - "path": "HostName" - }, - "windows-message": { - "type": "alias", - "path": "Message" - }, - "windows-provider-name": { - "type": "alias", - "path": "Provider_Name" - }, - "windows-servicename": { - "type": "alias", - "path": "ServiceName" + "path": "file.name" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_apt/fieldmappings.yml b/src/main/resources/OSMapping/others_apt/fieldmappings.yml index c9bd2ec90..26234ae87 100644 --- a/src/main/resources/OSMapping/others_apt/fieldmappings.yml +++ b/src/main/resources/OSMapping/others_apt/fieldmappings.yml @@ -2,5 +2,3 @@ fieldmappings: Image: process-exe CommandLine: process-command_line - - diff --git a/src/main/resources/OSMapping/others_cloud/fieldmappings.yml b/src/main/resources/OSMapping/others_cloud/fieldmappings.yml index 85af60cec..8d5691ea4 100644 --- a/src/main/resources/OSMapping/others_cloud/fieldmappings.yml +++ b/src/main/resources/OSMapping/others_cloud/fieldmappings.yml @@ -4,4 +4,4 @@ fieldmappings: HiveName: unmapped.HiveName fieldB: mappedB fieldA1: mappedA - + creationTime: timestamp diff --git a/src/main/resources/OSMapping/others_cloud/mappings.json b/src/main/resources/OSMapping/others_cloud/mappings.json index 48cdda71d..0e9426a6e 100644 --- a/src/main/resources/OSMapping/others_cloud/mappings.json +++ b/src/main/resources/OSMapping/others_cloud/mappings.json @@ -23,6 +23,10 @@ "windows-servicename": { "type": "alias", "path": "ServiceName" + }, + "creationTime": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_compliance/fieldmappings.yml b/src/main/resources/OSMapping/others_compliance/fieldmappings.yml index ddf9081d6..d1ec0d91a 100644 --- a/src/main/resources/OSMapping/others_compliance/fieldmappings.yml +++ b/src/main/resources/OSMapping/others_compliance/fieldmappings.yml @@ -4,4 +4,4 @@ fieldmappings: HiveName: unmapped.HiveName fieldB: mappedB fieldA1: mappedA - + creationTime: timestamp diff --git a/src/main/resources/OSMapping/others_compliance/mappings.json b/src/main/resources/OSMapping/others_compliance/mappings.json index 48cdda71d..a3ccdca77 100644 --- a/src/main/resources/OSMapping/others_compliance/mappings.json +++ b/src/main/resources/OSMapping/others_compliance/mappings.json @@ -23,6 +23,10 @@ "windows-servicename": { "type": "alias", "path": "ServiceName" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_macos/fieldmappings.yml b/src/main/resources/OSMapping/others_macos/fieldmappings.yml index 35a88e285..1754fcddc 100644 --- a/src/main/resources/OSMapping/others_macos/fieldmappings.yml +++ b/src/main/resources/OSMapping/others_macos/fieldmappings.yml @@ -4,3 +4,4 @@ fieldmappings: HiveName: unmapped.HiveName fieldB: mappedB fieldA1: mappedA + creationTime: timestamp \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_macos/mappings.json b/src/main/resources/OSMapping/others_macos/mappings.json index 0804434f1..5da91e3d9 100644 --- a/src/main/resources/OSMapping/others_macos/mappings.json +++ b/src/main/resources/OSMapping/others_macos/mappings.json @@ -3,6 +3,10 @@ "macos.event_data.CommandLine": { "type": "alias", "path": "CommandLine" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_proxy/fieldmappings.yml b/src/main/resources/OSMapping/others_proxy/fieldmappings.yml index ddaaa6956..08b377f19 100644 --- a/src/main/resources/OSMapping/others_proxy/fieldmappings.yml +++ b/src/main/resources/OSMapping/others_proxy/fieldmappings.yml @@ -4,4 +4,4 @@ fieldmappings: HiveName: unmapped.HiveName fieldB: mappedB fieldA1: mappedA - + creationTime: timestamp diff --git a/src/main/resources/OSMapping/others_proxy/mappings.json b/src/main/resources/OSMapping/others_proxy/mappings.json index 48cdda71d..a3ccdca77 100644 --- a/src/main/resources/OSMapping/others_proxy/mappings.json +++ b/src/main/resources/OSMapping/others_proxy/mappings.json @@ -23,6 +23,10 @@ "windows-servicename": { "type": "alias", "path": "ServiceName" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/others_web/fieldmappings.yml b/src/main/resources/OSMapping/others_web/fieldmappings.yml index a1cc130a2..8d7fd290e 100644 --- a/src/main/resources/OSMapping/others_web/fieldmappings.yml +++ b/src/main/resources/OSMapping/others_web/fieldmappings.yml @@ -4,4 +4,5 @@ fieldmappings: HiveName: unmapped.HiveName fieldB: mappedB fieldA1: mappedA + creationTime: timestamp diff --git a/src/main/resources/OSMapping/others_web/mappings.json b/src/main/resources/OSMapping/others_web/mappings.json index 48cdda71d..a3ccdca77 100644 --- a/src/main/resources/OSMapping/others_web/mappings.json +++ b/src/main/resources/OSMapping/others_web/mappings.json @@ -23,6 +23,10 @@ "windows-servicename": { "type": "alias", "path": "ServiceName" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/s3/fieldmappings.yml b/src/main/resources/OSMapping/s3/fieldmappings.yml index d2cbd9dc9..06ce921d8 100644 --- a/src/main/resources/OSMapping/s3/fieldmappings.yml +++ b/src/main/resources/OSMapping/s3/fieldmappings.yml @@ -1,3 +1,4 @@ fieldmappings: eventName: aws-cloudtrail-eventName eventSource: aws-cloudtrail-eventSource + eventTime: timestamp diff --git a/src/main/resources/OSMapping/s3/mappings.json b/src/main/resources/OSMapping/s3/mappings.json index 2559999ad..a13f40236 100644 --- a/src/main/resources/OSMapping/s3/mappings.json +++ b/src/main/resources/OSMapping/s3/mappings.json @@ -7,6 +7,10 @@ "aws-cloudtrail-eventName": { "type": "alias", "path": "aws.cloudtrail.eventName" + }, + "timestamp": { + "path": "aws.cloudtrail.eventTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/test_windows/fieldmappings.yml b/src/main/resources/OSMapping/test_windows/fieldmappings.yml index 7567e715b..04fff5ead 100644 --- a/src/main/resources/OSMapping/test_windows/fieldmappings.yml +++ b/src/main/resources/OSMapping/test_windows/fieldmappings.yml @@ -9,3 +9,4 @@ fieldmappings: Message: windows-message Provider_Name: windows-provider-name ServiceName: windows-servicename + creationTime: timestamp diff --git a/src/main/resources/OSMapping/test_windows/mappings.json b/src/main/resources/OSMapping/test_windows/mappings.json index 48cdda71d..a3ccdca77 100644 --- a/src/main/resources/OSMapping/test_windows/mappings.json +++ b/src/main/resources/OSMapping/test_windows/mappings.json @@ -23,6 +23,10 @@ "windows-servicename": { "type": "alias", "path": "ServiceName" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/windows/fieldmappings.yml b/src/main/resources/OSMapping/windows/fieldmappings.yml index 7f259f8a7..8e905ad52 100644 --- a/src/main/resources/OSMapping/windows/fieldmappings.yml +++ b/src/main/resources/OSMapping/windows/fieldmappings.yml @@ -61,5 +61,5 @@ fieldmappings: windows-hostname: winlog-computer_name windows-provider-name: winlog-provider_name windows-servicename: winlog-event_data-ServiceName - + creationTime: timestamp diff --git a/src/main/resources/OSMapping/windows/mappings.json b/src/main/resources/OSMapping/windows/mappings.json index e2075c854..f2eef2a51 100644 --- a/src/main/resources/OSMapping/windows/mappings.json +++ b/src/main/resources/OSMapping/windows/mappings.json @@ -187,6 +187,10 @@ "winlog-event_data-param2": { "path": "winlog.event_data.param2", "type": "alias" + }, + "timestamp": { + "path": "creationTime", + "type": "alias" } } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index dfc691030..a4f2df778 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -1386,6 +1386,10 @@ protected void createComponentTemplateWithMappings(String componentTemplateName, } protected void createComposableIndexTemplate(String templateName, List indexPatterns, String componentTemplateName, boolean isDatastream) throws IOException { + createComposableIndexTemplate(templateName, indexPatterns, componentTemplateName, isDatastream, 0); + } + + protected void createComposableIndexTemplate(String templateName, List indexPatterns, String componentTemplateName, boolean isDatastream, int priority) throws IOException { String body = "{\n" + (isDatastream ? "\"data_stream\": { }," : "") + @@ -1393,7 +1397,8 @@ protected void createComposableIndexTemplate(String templateName, List i indexPatterns.stream().collect( Collectors.joining(",", "\"", "\"")) + " ]," + - "\"composed_of\": [\"" + componentTemplateName + "\"]" + + "\"composed_of\": [\"" + componentTemplateName + "\"]," + + "\"priority\":" + priority + "}"; Response response = makeRequest( client(), @@ -1437,6 +1442,7 @@ protected Map getIndexMappingsSAFlat(String indexName) throws IO } + protected void createMappingsAPI(String indexName, String topicName) throws IOException { Request request = new Request("POST", MAPPER_BASE_URI); // both req params and req body are supported diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index d236f28a7..34afda245 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -170,67 +170,76 @@ public static String randomRule() { public static String countAggregationTestRule() { return " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA: valueA\n" + - " fieldB: valueB\n" + - " fieldC: valueC\n" + - " condition: sel | count(*) > 1"; + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA: valueA\n" + + " fieldB: valueB\n" + + " fieldC: valueC\n" + + " condition: sel | count(*) > 1"; } public static String sumAggregationTestRule() { return " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA: 123\n" + - " fieldB: 111\n" + - " fieldC: valueC\n" + - " condition: sel | sum(fieldA) by fieldB > 110"; + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA: 123\n" + + " fieldB: 111\n" + + " fieldC: valueC\n" + + " condition: sel | sum(fieldA) by fieldB > 110"; } public static String productIndexMaxAggRule() { return " title: Test\n" + - " id: 5f92fff9-82e3-48eb-8fc1-8b133556a551\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA: 123\n" + - " fieldB: 111\n" + - " fieldC: valueC\n" + - " condition: sel | max(fieldA) by fieldB > 110"; + " id: 5f92fff9-82e3-48eb-8fc1-8b133556a551\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA: 123\n" + + " fieldB: 111\n" + + " fieldC: valueC\n" + + " condition: sel | max(fieldA) by fieldB > 110"; } public static String randomProductDocument(){ return "{\n" + - " \"fieldA\": 123,\n" + - " \"mappedB\": 111,\n" + - " \"fieldC\": \"valueC\"\n" + - "}\n"; + " \"fieldA\": 123,\n" + + " \"mappedB\": 111,\n" + + " \"fieldC\": \"valueC\"\n" + + "}\n"; + } + + public static String randomProductDocumentWithTime(long time){ + return "{\n" + + " \"fieldA\": 123,\n" + + " \"mappedB\": 111,\n" + + " \"time\": " + (time) + ",\n" + + " \"fieldC\": \"valueC\"\n" + + "}\n"; } public static String randomEditedRule() { @@ -311,7 +320,7 @@ public static User randomUserEmpty() { } public static String randomDetectorType() { - return "TEST_WINDOWS"; + return "test_windows"; } public static DetectorInput randomDetectorInput() { @@ -429,95 +438,98 @@ public static String netFlowMappings() { public static String productIndexMapping(){ return "\"properties\":{\n" + - " \"fieldA\":{\n" + - " \"type\":\"long\"\n" + - " },\n" + - " \"mappedB\":{\n" + - " \"type\":\"long\"\n" + - " },\n" + - " \"fieldC\":{\n" + - " \"type\":\"keyword\"\n" + - " }\n" + - "}\n" + - "}"; + " \"fieldA\":{\n" + + " \"type\":\"long\"\n" + + " },\n" + + " \"mappedB\":{\n" + + " \"type\":\"long\"\n" + + " },\n" + + " \"time\":{\n" + + " \"type\":\"date\"\n" + + " },\n" + + " \"fieldC\":{\n" + + " \"type\":\"keyword\"\n" + + " }\n" + + "}\n" + + "}"; } public static String productIndexAvgAggRule(){ return " title: Test\n" + - " id: 39f918f3-981b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA: 123\n" + - " fieldB: 111\n" + - " fieldC: valueC\n" + - " condition: sel | avg(fieldA) by fieldC > 110"; + " id: 39f918f3-981b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA: 123\n" + + " fieldB: 111\n" + + " fieldC: valueC\n" + + " condition: sel | avg(fieldA) by fieldC > 110"; } public static String randomAggregationRule(String aggFunction, String signAndValue) { String rule = "title: Remote Encrypting File System Abuse\n" + - "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + - "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + - "references:\n" + - " - https://attack.mitre.org/tactics/TA0008/\n" + - " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + - " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + - " - https://github.com/zeronetworks/rpcfirewall\n" + - " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + - "tags:\n" + - " - attack.defense_evasion\n" + - "status: experimental\n" + - "author: Sagie Dulce, Dekel Paz\n" + - "date: 2022/01/01\n" + - "modified: 2022/01/01\n" + - "logsource:\n" + - " product: rpc_firewall\n" + - " category: application\n" + - " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + - "detection:\n" + - " sel:\n" + - " Opcode: Info\n" + - " condition: sel | %s(SeverityValue) by Version %s\n" + - "falsepositives:\n" + - " - Legitimate usage of remote file encryption\n" + - "level: high"; + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " sel:\n" + + " Opcode: Info\n" + + " condition: sel | %s(SeverityValue) by Version %s\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; return String.format(Locale.ROOT, rule, aggFunction, signAndValue); } public static String randomAggregationRule(String aggFunction, String signAndValue, String opCode) { String rule = "title: Remote Encrypting File System Abuse\n" + - "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + - "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + - "references:\n" + - " - https://attack.mitre.org/tactics/TA0008/\n" + - " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + - " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + - " - https://github.com/zeronetworks/rpcfirewall\n" + - " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + - "tags:\n" + - " - attack.defense_evasion\n" + - "status: experimental\n" + - "author: Sagie Dulce, Dekel Paz\n" + - "date: 2022/01/01\n" + - "modified: 2022/01/01\n" + - "logsource:\n" + - " product: rpc_firewall\n" + - " category: application\n" + - " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + - "detection:\n" + - " sel:\n" + - " Opcode: %s\n" + - " condition: sel | %s(SeverityValue) by Version %s\n" + - "falsepositives:\n" + - " - Legitimate usage of remote file encryption\n" + - "level: high"; + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " sel:\n" + + " Opcode: %s\n" + + " condition: sel | %s(SeverityValue) by Version %s\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; return String.format(Locale.ROOT, rule, opCode, aggFunction, signAndValue); } @@ -1146,39 +1158,39 @@ public static String windowsIndexMapping() { public static String randomDoc(int severity, int version, String opCode) { String doc = "{\n" + - "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + - "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + - "\"Keywords\":\"9223372036854775808\",\n" + - "\"SeverityValue\":%s,\n" + - "\"Severity\":\"INFO\",\n" + - "\"EventID\":22,\n" + - "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + - "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + - "\"Version\":%s,\n" + - "\"TaskValue\":22,\n" + - "\"OpcodeValue\":0,\n" + - "\"RecordNumber\":9532,\n" + - "\"ExecutionProcessID\":1996,\n" + - "\"ExecutionThreadID\":2616,\n" + - "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + - "\"Domain\":\"NT AUTHORITY\",\n" + - "\"AccountName\":\"SYSTEM\",\n" + - "\"UserID\":\"S-1-5-18\",\n" + - "\"AccountType\":\"User\",\n" + - "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + - "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + - "\"Opcode\":\"%s\",\n" + - "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + - "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + - "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + - "\"QueryResults\":\"172.31.46.38;\",\n" + - "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + - "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + - "\"SourceModuleName\":\"in\",\n" + - "\"SourceModuleType\":\"im_msvistalog\",\n" + - "\"CommandLine\": \"eachtest\",\n" + - "\"Initiated\": \"true\"\n" + - "}"; + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + + "\"Keywords\":\"9223372036854775808\",\n" + + "\"SeverityValue\":%s,\n" + + "\"Severity\":\"INFO\",\n" + + "\"EventID\":22,\n" + + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + "\"Version\":%s,\n" + + "\"TaskValue\":22,\n" + + "\"OpcodeValue\":0,\n" + + "\"RecordNumber\":9532,\n" + + "\"ExecutionProcessID\":1996,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + + "\"Domain\":\"NT AUTHORITY\",\n" + + "\"AccountName\":\"SYSTEM\",\n" + + "\"UserID\":\"S-1-5-18\",\n" + + "\"AccountType\":\"User\",\n" + + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + + "\"Opcode\":\"%s\",\n" + + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + + "\"QueryResults\":\"172.31.46.38;\",\n" + + "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + + "\"SourceModuleName\":\"in\",\n" + + "\"SourceModuleType\":\"im_msvistalog\",\n" + + "\"CommandLine\": \"eachtest\",\n" + + "\"Initiated\": \"true\"\n" + + "}"; return String.format(Locale.ROOT, doc, severity, version, opCode); } diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 50aa9bb24..9e7337a92 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -36,9 +36,62 @@ import java.util.Map; import java.util.stream.Collectors; + +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.MAPPER_BASE_URI; + public class MapperRestApiIT extends SecurityAnalyticsRestTestCase { + public void testGetMappingSuccess() throws IOException { + String testIndexName1 = "my_index_1"; + String testIndexName2 = "my_index_2"; + String testIndexPattern = "my_index*"; + + createSampleIndex(testIndexName1); + createSampleIndex(testIndexName2); + + createMappingsAPI(testIndexName2, "netflow"); + + Request request = new Request("GET", MAPPER_BASE_URI + "?index_name=" + testIndexPattern); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = (Map) responseAsMap(response); + // Assert that indexName returned is one passed by user + assertTrue(respMap.containsKey(testIndexPattern)); + } + + public void testGetMappingSuccess_1() throws IOException { + String testIndexName1 = "my_index_1"; + String testIndexPattern = "my_index*"; + + createIndex(testIndexName1, Settings.EMPTY); + + String sampleDoc = "{\n" + + " \"lvl1field\": 12345,\n" + + " \"source1.ip\": \"12345\",\n" + + " \"source1.port\": 55,\n" + + " \"some.very.long.field.name\": \"test\"\n" + + "}"; + + indexDoc(testIndexName1, "1", sampleDoc); + // puts mappings with timestamp alias + String createMappingsRequest = "{\"index_name\":\"my_index*\",\"rule_topic\":\"windows\",\"partial\":true,\"alias_mappings\":{\"properties\":{\"timestamp\":{\"type\":\"alias\",\"path\":\"lvl1field\"},\"winlog-computer_name\":{\"type\":\"alias\",\"path\":\"source1.port\"},\"winlog-event_data-AuthenticationPackageName\":{\"type\":\"alias\",\"path\":\"source1.ip\"},\"winlog-event_data-Company\":{\"type\":\"alias\",\"path\":\"some.very.long.field.name\"}}}}"; + + Request request = new Request("POST", MAPPER_BASE_URI); + // both req params and req body are supported + request.setJsonEntity(createMappingsRequest); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + request = new Request("GET", MAPPER_BASE_URI + "?index_name=" + testIndexPattern); + response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = (Map) responseAsMap(response); + Map props = (Map)((Map) respMap.get(testIndexPattern)).get("mappings"); + props = (Map) props.get("properties"); + assertEquals(4, props.size()); + } + public void testCreateMappingSuccess() throws IOException { String testIndexName = "my_index"; @@ -50,9 +103,9 @@ public void testCreateMappingSuccess() throws IOException { // both req params and req body are supported request.setJsonEntity( "{ \"index_name\":\"" + testIndexName + "\"," + - " \"rule_topic\":\"netflow\", " + - " \"partial\":true" + - "}" + " \"rule_topic\":\"netflow\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(request); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); @@ -89,22 +142,22 @@ public void testCreateMappingWithAliasesSuccess() throws IOException { // both req params and req body are supported request.setJsonEntity( "{\n" + - " \"index_name\": \"my_index\",\n" + - " \"rule_topic\":\"netflow\", " + - " \"partial\":true," + - " \"alias_mappings\": {\n" + - " \"properties\": {\n" + - " \"source.ip\": {\n" + - " \"type\": \"alias\",\n" + - " \"path\": \"netflow.source_ipv4_address\"\n" + - " },\n" + - " \"source.port\": {\n" + - " \"type\": \"alias\",\n" + - " \"path\": \"netflow.source_transport_port\"\n" + - " }\n" + - " }\n" + - " }\n" + - "}" + " \"index_name\": \"my_index\",\n" + + " \"rule_topic\":\"netflow\", " + + " \"partial\":true," + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"source.ip\": {\n" + + " \"type\": \"alias\",\n" + + " \"path\": \"netflow.source_ipv4_address\"\n" + + " },\n" + + " \"source.port\": {\n" + + " \"type\": \"alias\",\n" + + " \"path\": \"netflow.source_transport_port\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" ); // request.addParameter("indexName", testIndexName); // request.addParameter("ruleTopic", "netflow"); @@ -225,7 +278,7 @@ public void testExistingMappingsAreUntouched() throws IOException { GetMappingsResponse getMappingsResponse = SecurityAnalyticsClientUtils.executeGetMappingsRequest(testIndexName); Map properties = (Map) getMappingsResponse.getMappings().get(testIndexName) - .getSourceAsMap().get("properties"); + .getSourceAsMap().get("properties"); // Verify that there is still mapping for integer field "plain1" assertTrue(((Map)properties.get("plain1")).get("type").equals("integer")); } @@ -292,7 +345,7 @@ public void testGetMappingsViewSuccess() throws IOException { assertEquals(6, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(2, unmappedFieldAliases.size()); + assertEquals(3, unmappedFieldAliases.size()); } public void testCreateMappings_withDatastream_success() throws IOException { @@ -561,35 +614,6 @@ public void testCreateMappings_withIndexPattern_indexTemplate_createAndUpdate_su assertTrue(props.containsKey("destination.port")); } - public void testCreateMappings_withIndexPattern_oneNoMatches_success() throws IOException { - String indexName1 = "test_index_1"; - String indexName2 = "test_index_2"; - String indexPattern = "test_index*"; - - createIndex(indexName1, Settings.EMPTY, null); - createIndex(indexName2, Settings.EMPTY, null); - - client().performRequest(new Request("POST", "_refresh")); - - // Insert sample docs - String sampleDoc1 = "{" + - " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + - " \"netflow.destination_transport_port\":1234," + - " \"netflow.source_transport_port\":4444" + - "}"; - String sampleDoc2 = "{" + - " \"netflow11.destination33_transport_port\":1234," + - " \"netflow11.destination33_ipv4_address\":\"10.53.111.14\"" + - "}"; - indexDoc(indexName1, "1", sampleDoc1); - indexDoc(indexName2, "1", sampleDoc2); - - client().performRequest(new Request("POST", "_refresh")); - - // Execute CreateMappingsAction to add alias mapping for index - createMappingsAPI(indexPattern, "netflow"); - } - public void testCreateMappings_withIndexPattern_oneNoMappings_failure() throws IOException { String indexName1 = "test_index_1"; String indexName2 = "test_index_2"; @@ -622,9 +646,13 @@ public void testCreateMappings_withIndexPattern_oneNoMappings_failure() throws I public void testGetMappingsView_index_pattern_two_indices_Success() throws IOException { - String testIndexName1 = "get_mappings_view_index11"; - String testIndexName2 = "get_mappings_view_index22"; - String indexPattern = "get_mappings_view_index*"; + String testIndexName1 = "get_mappings_view_index111"; + String testIndexName2 = "get_mappings_view_index122"; + String testIndexName3 = "get_mappings_view_index"; + + String indexPattern = "get_mappings_view_index1*"; + String indexPattern2 = "get_mappings_view_index*"; + createSampleIndex(testIndexName1); createSampleIndex(testIndexName2); indexDoc(testIndexName2, "987654", "{ \"extra_field\": 12345 }"); @@ -652,7 +680,7 @@ public void testGetMappingsView_index_pattern_two_indices_Success() throws IOExc assertTrue(extraField.isPresent()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(2, unmappedFieldAliases.size()); + assertEquals(3, unmappedFieldAliases.size()); } public void testGetMappingsView_alias_without_writeindex_Success() throws IOException { @@ -687,7 +715,7 @@ public void testGetMappingsView_alias_without_writeindex_Success() throws IOExce assertTrue(extraField.isPresent()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(2, unmappedFieldAliases.size()); + assertEquals(3, unmappedFieldAliases.size()); } public void testGetMappingsView_alias_with_writeindex_Success() throws IOException { @@ -725,7 +753,7 @@ public void testGetMappingsView_alias_with_writeindex_Success() throws IOExcepti assertTrue(extraField.isPresent()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(2, unmappedFieldAliases.size()); + assertEquals(3, unmappedFieldAliases.size()); } public void testGetMappingsView_datastream_one_backing_index_Success() throws IOException { @@ -752,7 +780,7 @@ public void testGetMappingsView_datastream_one_backing_index_Success() throws IO assertEquals(7, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(2, unmappedFieldAliases.size()); + assertEquals(3, unmappedFieldAliases.size()); deleteDatastream(datastreamName); } @@ -802,11 +830,10 @@ public void testGetMappingsView_datastream_two_backing_index_Success() throws IO assertEquals(1, props.size()); assertTrue(props.containsKey("source.ip")); // Verify unmapped index fields - List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); - assertEquals(1, unmappedIndexFields.size()); + // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); - assertEquals(5, unmappedFieldAliases.size()); + assertEquals(6, unmappedFieldAliases.size()); deleteDatastream(datastreamName); } @@ -847,6 +874,176 @@ public void testCreateMappings_withIndexPattern_success() throws IOException { assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } + public void testCreateMappings_withIndexPattern_conflictingTemplates_success() throws IOException { + String indexName1 = "test_index_11"; + String indexName2 = "test_index_12"; + String indexName3 = "test_index_13"; + String indexName4 = "test_index44"; + String indexPattern1 = "test_index_1*"; + String indexPattern2 = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample doc + String sampleDoc = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234" + + "}"; + + indexDoc(indexName1, "1", sampleDoc); + indexDoc(indexName2, "1", sampleDoc); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction with first index pattern + createMappingsAPI(indexPattern1, "netflow"); + + createIndex(indexName3, Settings.EMPTY, null); + + // Insert sample doc + String sampleDoc2 = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.destination_ipv4_address\":\"10.53.111.14\"," + + " \"netflow.source_transport_port\":4444" + + "}"; + + indexDoc(indexName3, "1", sampleDoc2); + + // Execute CreateMappingsAction with conflicting index pattern - expect template to be updated + createMappingsAPI(indexPattern2, "netflow"); + + createIndex(indexName4, Settings.EMPTY, null); + // Verify with GET _mapping + Map props = getIndexMappingsAPIFlat(indexName4); + assertEquals(8, props.size()); + // Verify with SA's GetIndexMappings + props = getIndexMappingsSAFlat(indexName4); + assertEquals(4, props.size()); + assertTrue(props.containsKey("source.ip")); + assertTrue(props.containsKey("source.port")); + assertTrue(props.containsKey("destination.ip")); + assertTrue(props.containsKey("destination.port")); + } + + public void testCreateMappings_withIndexPattern_conflictingTemplates_failure_1() throws IOException { + String indexName1 = "test_index_11"; + String indexName2 = "test_index_12"; + String indexName3 = "test_index_13"; + String indexName4 = "test_index44"; + String indexPattern1 = "test_index_1*"; + String indexPattern2 = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample doc + String sampleDoc = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234" + + "}"; + + indexDoc(indexName1, "1", sampleDoc); + indexDoc(indexName2, "1", sampleDoc); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction with first index pattern + createMappingsAPI(indexPattern1, "netflow"); + + // User-create template with conflicting pattern but higher priority + createComponentTemplateWithMappings("user_component_template", "\"properties\": { \"some_field\": { \"type\": \"long\" } }"); + createComposableIndexTemplate("user_custom_template", List.of("test_index_111111*"), "user_component_template", false, 100); + + // Execute CreateMappingsAction and expect 2 conflicting templates and failure + try { + createMappingsAPI(indexPattern2, "netflow"); + } catch (ResponseException e) { + assertTrue(e.getMessage().contains("Found conflicting templates: [user_custom_template, .opensearch-sap-alias-mappings-index-template-test_index_1]")); + } + } + + public void testCreateMappings_withIndexPattern_conflictingTemplates_failure_2() throws IOException { + String indexName1 = "test_index_11"; + String indexName2 = "test_index_12"; + String indexName3 = "test_index_13"; + String indexName4 = "test_index44"; + String indexPattern1 = "test_index_1*"; + String indexPattern2 = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample doc + String sampleDoc = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234" + + "}"; + + indexDoc(indexName1, "1", sampleDoc); + indexDoc(indexName2, "1", sampleDoc); + + client().performRequest(new Request("POST", "_refresh")); + + + // User-create template with conflicting pattern but higher priority + createComponentTemplateWithMappings("user_component_template", "\"properties\": { \"some_field\": { \"type\": \"long\" } }"); + createComposableIndexTemplate("user_custom_template", List.of("test_index_111111*"), "user_component_template", false, 100); + + // Execute CreateMappingsAction and expect conflict with 1 user template + try { + createMappingsAPI(indexPattern2, "netflow"); + } catch (ResponseException e) { + assertTrue(e.getMessage().contains("Found conflicting templates: [user_custom_template]")); + } + } + + + public void testCreateMappings_withIndexPattern_oneNoMatches_success() throws IOException { + String indexName1 = "test_index_1"; + String indexName2 = "test_index_2"; + String indexPattern = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample docs + String sampleDoc1 = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.source_transport_port\":4444" + + "}"; + String sampleDoc2 = "{" + + " \"netflow11.destination33_transport_port\":1234," + + " \"netflow11.destination33_ipv4_address\":\"10.53.111.14\"" + + "}"; + indexDoc(indexName1, "1", sampleDoc1); + indexDoc(indexName2, "1", sampleDoc2); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction to add alias mapping for index + Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + request.setJsonEntity( + "{ \"index_name\":\"" + indexPattern + "\"," + + " \"rule_topic\":\"netflow\", " + + " \"partial\":true" + + "}" + ); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + private void createSampleIndex(String indexName) throws IOException { createSampleIndex(indexName, Settings.EMPTY, null); } @@ -887,20 +1084,20 @@ private void createSampleIndex(String indexName, Settings settings, String alias " \"keyword\":{" + " \"type\":\"keyword\"," + " \"ignore_above\":256" + - "}" + - "}" + - "}," + + "}" + + "}" + + "}," + " \"last\":{" + - "\"type\":\"text\"," + - "\"fields\":{" + + "\"type\":\"text\"," + + "\"fields\":{" + " \"keyword\":{" + " \"type\":\"keyword\"," + " \"ignore_above\":256" + - "}" + - "}" + - "}" + - "}" + - "}" + + "}" + + "}" + + "}" + + "}" + + "}" + " }"; createIndex(indexName, settings, indexMapping, aliases); @@ -1152,7 +1349,6 @@ public void testCreateCloudTrailMapping() throws IOException { response = client().performRequest(new Request("POST", "_refresh")); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } - public void testCreateDNSMapping() throws IOException{ String INDEX_NAME = "test_create_cloudtrail_mapping_index"; @@ -1184,6 +1380,8 @@ public void testCreateDNSMapping() throws IOException{ //Loop over the mappings and run update request for each one specifying the index to be updated mappings.entrySet().forEach(entry -> { String key = entry.getKey(); + if("timestamp".equals(key)) + return; String path = ((Map) entry.getValue()).get("path").toString(); try { Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); @@ -1259,7 +1457,7 @@ public void testTraverseAndCopy() { // Copy specific paths from mappings Map filteredMappings = mappingsTraverser.traverseAndCopyWithFilter( - List.of("netflow.event_data.SourceAddress", "netflow.event.stop", "plain1", "user.first", "user.last") + Set.of("netflow.event_data.SourceAddress", "netflow.event.stop", "plain1", "user.first", "user.last") ); // Now traverse filtered mapppings to confirm only copied paths are present diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MappingsTraverserTests.java b/src/test/java/org/opensearch/securityanalytics/mapper/MappingsTraverserTests.java index 19ebb9e12..1c71df93c 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MappingsTraverserTests.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MappingsTraverserTests.java @@ -369,7 +369,7 @@ public void testTraverseAndCopyValidNestedMappings() { mappingsTraverser = new MappingsTraverser(indexMappingJSON, Set.of("ip")); // Copy mappings while excluding type=ip - Map filteredMappings = mappingsTraverser.traverseAndCopyWithFilter(List.of("user.first", "user.last")); + Map filteredMappings = mappingsTraverser.traverseAndCopyWithFilter(Set.of("user.first", "user.last")); // Now traverse filtered mapppings to confirm type=ip is not present List paths = new ArrayList<>(); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index 22ada45e2..1711a6ca3 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -17,8 +17,10 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; +import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.client.ResponseException; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.commons.alerting.model.Monitor.MonitorType; import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; @@ -36,21 +38,8 @@ import java.util.Map; import java.util.stream.Collectors; import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.model.Rule; - -import static org.opensearch.securityanalytics.TestHelpers.productIndexMaxAggRule; -import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule; -import static org.opensearch.securityanalytics.TestHelpers.productIndexMapping; -import static org.opensearch.securityanalytics.TestHelpers.randomDetector; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; -import static org.opensearch.securityanalytics.TestHelpers.randomDoc; -import static org.opensearch.securityanalytics.TestHelpers.randomIndex; -import static org.opensearch.securityanalytics.TestHelpers.randomProductDocument; -import static org.opensearch.securityanalytics.TestHelpers.randomRule; -import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule; -import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; + +import static org.opensearch.securityanalytics.TestHelpers.*; public class DetectorRestApiIT extends SecurityAnalyticsRestTestCase { @@ -159,10 +148,10 @@ public void testCreateDetectorWithoutRules() throws IOException { Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response createMappingResponse = client().performRequest(createMappingRequest); @@ -177,11 +166,11 @@ public void testCreateDetectorWithoutRules() throws IOException { // Verify rules String request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); Assert.assertEquals(0, response.getHits().getTotalHits().value); @@ -344,10 +333,10 @@ public void testCreatingADetectorWithAggregationRules() throws IOException { Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"test_windows\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"test_windows\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -356,7 +345,7 @@ public void testCreatingADetectorWithAggregationRules() throws IOException { String customAvgRuleId = createRule(productIndexAvgAggRule()); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(customAvgRuleId)), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); Detector detector = randomDetectorWithInputs(List.of(input)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -374,12 +363,12 @@ public void testCreatingADetectorWithAggregationRules() throws IOException { Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -596,12 +585,12 @@ public void testDeletingADetector_single_ruleTopicIndex() throws IOException { Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), null); Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); // We deleted 1 detector, but 1 detector with same type exists, so we expect queryIndex to be present - Assert.assertTrue(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); + Assert.assertTrue(doesIndexExist(String.format(Locale.ROOT, ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId2, Collections.emptyMap(), null); Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); // We deleted all detectors of type windows, so we expect that queryIndex is deleted - Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); + Assert.assertFalse(doesIndexExist(String.format(Locale.ROOT, ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); request = "{\n" + " \"query\" : {\n" + @@ -702,4 +691,201 @@ public void testDeletingANonExistingDetector() throws IOException { Assert.assertEquals(404, ex.getResponse().getStatusLine().getStatusCode()); } } + + public void testCreatingADetectorWithTimestampFieldAliasMapping() throws IOException { + String index = createTestIndex(randomIndex(), productIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"test_windows\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + updateRequest.setJsonEntity(Strings.toString(XContentFactory.jsonBuilder().map(Map.of( + "index_name", index, + "field", "time", + "alias", "timestamp")))); + Response apiResponse = client().performRequest(updateRequest); + assertEquals(HttpStatus.SC_OK, apiResponse.getStatusLine().getStatusCode()); + + String customAvgRuleId = createRule(productIndexAvgAggRule()); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(customAvgRuleId)), + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, detectorId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, detectorId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + List monitorTypes = new ArrayList<>(); + + Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); + + String bucketLevelMonitorId = ""; + + // Verify that doc level monitor is created + List monitorIds = (List) (detectorAsMap).get("monitor_id"); + + String firstMonitorId = monitorIds.get(0); + String firstMonitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + firstMonitorId))).get("monitor")).get("monitor_type"); + + if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(firstMonitorType)){ + bucketLevelMonitorId = firstMonitorId; + } + monitorTypes.add(firstMonitorType); + + String secondMonitorId = monitorIds.get(1); + String secondMonitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + secondMonitorId))).get("monitor")).get("monitor_type"); + monitorTypes.add(secondMonitorType); + if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(secondMonitorType)){ + bucketLevelMonitorId = secondMonitorId; + } + Assert.assertTrue(Arrays.asList(MonitorType.BUCKET_LEVEL_MONITOR.getValue(), MonitorType.DOC_LEVEL_MONITOR.getValue()).containsAll(monitorTypes)); + + indexDoc(index, "1", randomProductDocumentWithTime(System.currentTimeMillis())); + + Response executeResponse = executeAlertingMonitor(bucketLevelMonitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + // verify bucket level monitor findings + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + assertNotNull(getFindingsBody); + Assert.assertEquals(1, getFindingsBody.get("total_findings")); + List findings = (List) getFindingsBody.get("findings"); + Assert.assertEquals(findings.size(), 1); + HashMap finding = (HashMap) findings.get(0); + Assert.assertTrue(finding.containsKey("queries")); + HashMap docLevelQuery = (HashMap) ((List) finding.get("queries")).get(0); + String ruleId = docLevelQuery.get("id").toString(); + // Verify if the rule id in bucket level finding is the same as rule used for bucket monitor creation + assertEquals(customAvgRuleId, ruleId); + Response getResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), null); + String getDetectorResponseString = new String(getResponse.getEntity().getContent().readAllBytes()); + Assert.assertTrue(getDetectorResponseString.contains(ruleId)); + } + + public void testCreatingADetectorWithTimestampFieldAliasMapping_verifyTimeRangeInBucketMonitor() throws IOException { + String index = createTestIndex(randomIndex(), productIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"test_windows\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + updateRequest.setJsonEntity(Strings.toString(XContentFactory.jsonBuilder().map(Map.of( + "index_name", index, + "field", "time", + "alias", "timestamp")))); + Response apiResponse = client().performRequest(updateRequest); + assertEquals(HttpStatus.SC_OK, apiResponse.getStatusLine().getStatusCode()); + + String customAvgRuleId = createRule(productIndexAvgAggRule()); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(customAvgRuleId)), + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, detectorId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, detectorId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + List monitorTypes = new ArrayList<>(); + + Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); + + String bucketLevelMonitorId = ""; + + // Verify that doc level monitor is created + List monitorIds = (List) (detectorAsMap).get("monitor_id"); + + String firstMonitorId = monitorIds.get(0); + String firstMonitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + firstMonitorId))).get("monitor")).get("monitor_type"); + + if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(firstMonitorType)){ + bucketLevelMonitorId = firstMonitorId; + } + monitorTypes.add(firstMonitorType); + + String secondMonitorId = monitorIds.get(1); + String secondMonitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + secondMonitorId))).get("monitor")).get("monitor_type"); + monitorTypes.add(secondMonitorType); + if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(secondMonitorType)){ + bucketLevelMonitorId = secondMonitorId; + } + Assert.assertTrue(Arrays.asList(MonitorType.BUCKET_LEVEL_MONITOR.getValue(), MonitorType.DOC_LEVEL_MONITOR.getValue()).containsAll(monitorTypes)); + + indexDoc(index, "1", randomProductDocumentWithTime(System.currentTimeMillis()-1000*60*70)); // doc's timestamp is older than 1 hr + + Response executeResponse = executeAlertingMonitor(bucketLevelMonitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + // verify bucket level monitor findings + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + assertNotNull(getFindingsBody); + Assert.assertEquals(0, getFindingsBody.get("total_findings")); + List findings = (List) getFindingsBody.get("findings"); + Assert.assertEquals(findings.size(), 0); //there should be no findings as doc is not in time range of current run + } } \ No newline at end of file