[Backport 2.x] Adds timestamp field alias and sets time range filter …

…in bucket level monitor (#262) (#279) (#282) Signed-off-by: Subhobrata Dey <[email protected]> Co-authored-by: Surya Sashank Nistala <[email protected]>
opensearch-project · Jan 11, 2023 · 0559053 · 0559053
1 parent 8f669cf
commit 0559053
Show file tree

Hide file tree

Showing 38 changed files with 702 additions and 674 deletions.
diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java
@@ -4,18 +4,6 @@
  */
 package org.opensearch.securityanalytics.transport;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -41,6 +29,8 @@
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.Client;
 import org.opensearch.client.node.NodeClient;
+import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
+import org.opensearch.cluster.metadata.MappingMetadata;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.common.io.stream.NamedWriteableRegistry;
@@ -68,9 +58,11 @@
 import org.opensearch.commons.alerting.model.SearchInput;
 import org.opensearch.commons.alerting.model.action.Action;
 import org.opensearch.commons.authuser.User;
+import org.opensearch.index.query.BoolQueryBuilder;
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.index.query.RangeQueryBuilder;
 import org.opensearch.index.reindex.BulkByScrollResponse;
 import org.opensearch.index.seqno.SequenceNumbers;
 import org.opensearch.rest.RestRequest;
@@ -80,11 +72,15 @@
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.SearchHits;
 import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.securityanalytics.action.GetIndexMappingsAction;
+import org.opensearch.securityanalytics.action.GetIndexMappingsRequest;
+import org.opensearch.securityanalytics.action.GetIndexMappingsResponse;
 import org.opensearch.securityanalytics.action.IndexDetectorAction;
 import org.opensearch.securityanalytics.action.IndexDetectorRequest;
 import org.opensearch.securityanalytics.action.IndexDetectorResponse;
 import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig;
 import org.opensearch.securityanalytics.mapper.MapperService;
+import org.opensearch.securityanalytics.mapper.MapperUtils;
 import org.opensearch.securityanalytics.model.Detector;
 import org.opensearch.securityanalytics.model.DetectorInput;
 import org.opensearch.securityanalytics.model.DetectorRule;
@@ -105,10 +101,24 @@
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
 public class TransportIndexDetectorAction extends HandledTransportAction<IndexDetectorRequest, IndexDetectorResponse> implements SecureTransportAction {
 
     public static final String PLUGIN_OWNER_FIELD = "security_analytics";
     private static final Logger log = LogManager.getLogger(TransportIndexDetectorAction.class);
+    public static final String TIMESTAMP_FIELD_ALIAS = "timestamp";
 
     private final Client client;
 
@@ -133,6 +143,8 @@ public class TransportIndexDetectorAction extends HandledTransportAction<IndexDe
 
     private final NamedWriteableRegistry namedWriteableRegistry;
 
+    private final IndexNameExpressionResolver indexNameExpressionResolver;
+
     private volatile TimeValue indexTimeout;
     @Inject
     public TransportIndexDetectorAction(TransportService transportService,
@@ -145,7 +157,8 @@ public TransportIndexDetectorAction(TransportService transportService,
                                         MapperService mapperService,
                                         ClusterService clusterService,
                                         Settings settings,
-                                        NamedWriteableRegistry namedWriteableRegistry) {
+                                        NamedWriteableRegistry namedWriteableRegistry,
+                                        IndexNameExpressionResolver indexNameExpressionResolver) {
         super(IndexDetectorAction.NAME, transportService, actionFilters, IndexDetectorRequest::new);
         this.client = client;
         this.xContentRegistry = xContentRegistry;
@@ -156,6 +169,7 @@ public TransportIndexDetectorAction(TransportService transportService,
         this.clusterService = clusterService;
         this.settings = settings;
         this.namedWriteableRegistry = namedWriteableRegistry;
+        this.indexNameExpressionResolver = indexNameExpressionResolver;
         this.threadPool = this.detectorIndices.getThreadPool();
         this.indexTimeout = SecurityAnalyticsSettings.INDEX_TIMEOUT.get(this.settings);
         this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings);
@@ -477,6 +491,39 @@ private IndexMonitorRequest createBucketLevelMonitorRequest(
             // Build query string filter
             .query(QueryBuilders.queryStringQuery(rule.getQueries().get(0).getValue()))
             .aggregation(aggregationQueries.getAggBuilder());
+        String concreteIndex = IndexUtils.getNewIndexByCreationDate( // index variable in method signature can also be an index pattern
+                clusterService.state(),
+                indexNameExpressionResolver,
+                index
+        );
+        try {
+            GetIndexMappingsResponse getIndexMappingsResponse = client.execute(
+                    GetIndexMappingsAction.INSTANCE,
+                    new GetIndexMappingsRequest(concreteIndex))
+                    .actionGet();
+            MappingMetadata mappingMetadata = getIndexMappingsResponse.mappings().get(concreteIndex);
+            List<Pair<String, String>> pairs = MapperUtils.getAllAliasPathPairs(mappingMetadata);
+            boolean timeStampAliasPresent = pairs.
+                    stream()
+                    .anyMatch(p ->
+                            TIMESTAMP_FIELD_ALIAS.equals(p.getLeft()) || TIMESTAMP_FIELD_ALIAS.equals(p.getRight()));
+            if(timeStampAliasPresent) {
+                BoolQueryBuilder boolQueryBuilder = searchSourceBuilder.query() == null
+                        ? new BoolQueryBuilder()
+                        : QueryBuilders.boolQuery().must(searchSourceBuilder.query());
+                RangeQueryBuilder timeRangeFilter = QueryBuilders.rangeQuery(TIMESTAMP_FIELD_ALIAS)
+                        .gt("{{period_end}}||-1h")
+                        .lte("{{period_end}}")
+                        .format("epoch_millis");
+                boolQueryBuilder.must(timeRangeFilter);
+                searchSourceBuilder.query(boolQueryBuilder);
+            }
+        } catch (Exception e) {
+            log.error(
+                    String.format(Locale.getDefault(),
+                            "Unable to verify presence of timestamp alias for index [%s] in detector [%s]. Not setting time range filter for bucket level monitor.",
+                    concreteIndex, detector.getName()), e);
+        }
 
         List<SearchInput> bucketLevelMonitorInputs = new ArrayList<>();
         bucketLevelMonitorInputs.add(new SearchInput(Arrays.asList(index), searchSourceBuilder));

diff --git a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java
@@ -7,11 +7,13 @@
 import java.util.SortedMap;
 import org.opensearch.action.ActionListener;
 import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.client.IndicesAdminClient;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.IndexAbstraction;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
 import org.opensearch.common.xcontent.NamedXContentRegistry;
 import org.opensearch.common.xcontent.XContentParser;
@@ -154,4 +156,10 @@ public static String getNewestIndexByCreationDate(String[] concreteIndices, Clus
         }
         return newestIndex;
     }
+
+    public static String getNewIndexByCreationDate(ClusterState state, IndexNameExpressionResolver i, String index) {
+        String[] strings = i.concreteIndexNames(state, IndicesOptions.LENIENT_EXPAND_OPEN, index);
+        return getNewestIndexByCreationDate(strings, state);
+    }
+
 }
diff --git a/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml b/src/main/resources/OSMapping/ad_ldap/fieldmappings.yml
@@ -1,7 +1,4 @@
 # this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under windows log group to their corresponding ECS Fields.
 fieldmappings:
-    EventID: event_uid
-    HiveName: unmapped.HiveName
-    fieldB: mappedB
-    fieldA1: mappedA
-    CommandLine: windows-event_data-CommandLine
+    TargetUserName: winlog-event_data-TargetUserName
+    creationTime: timestamp
diff --git a/src/main/resources/OSMapping/ad_ldap/mappings.json b/src/main/resources/OSMapping/ad_ldap/mappings.json
@@ -1,12 +1,12 @@
 {
   "properties": {
-    "windows-event_data-CommandLine": {
-      "type": "alias",
-      "path": "CommandLine"
+    "winlog-event_data-TargetUserName": {
+      "path": "winlog.event_data.TargetUserName",
+      "type": "alias"
     },
-    "event_uid": {
-      "type": "alias",
-      "path": "EventID"
+    "timestamp": {
+      "path": "creationTime",
+      "type": "alias"
     }
   }
 }
diff --git a/src/main/resources/OSMapping/apache_access/fieldmappings.yml b/src/main/resources/OSMapping/apache_access/fieldmappings.yml
@@ -5,3 +5,4 @@ fieldmappings:
     fieldB: mappedB
     fieldA1: mappedA
     CommandLine: windows-event_data-CommandLine
+    creationTime: timestamp
diff --git a/src/main/resources/OSMapping/apache_access/mappings.json b/src/main/resources/OSMapping/apache_access/mappings.json
@@ -7,6 +7,10 @@
     "event_uid": {
       "type": "alias",
       "path": "EventID"
+    },
+    "timestamp": {
+      "path": "creationTime",
+      "type": "alias"
     }
   }
 }
diff --git a/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml b/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml
@@ -7,4 +7,5 @@ fieldmappings:
     requestParameters.containerDefinitions.command: aws-cloudtrail-requestParameters-containerDefinitions-command
     userIdentity.sessionContext.sessionIssuer.type: userIdentity-sessionContext-sessionIssuer-type
     userIdentity.type: aws-cloudtrail-userIdentity-type
-    userIdentity.arn: aws-cloudtrail-userIdentity-arn
+    userIdentity.arn: aws-cloudtrail-userIdentity-arn
+    eventTime: timestamp
diff --git a/src/main/resources/OSMapping/cloudtrail/mappings.json b/src/main/resources/OSMapping/cloudtrail/mappings.json
@@ -35,6 +35,10 @@
     "userIdentity-sessionContext-sessionIssuer-type": {
       "type": "alias",
       "path": "aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.type"
+    },
+    "timestamp": {
+      "path": "aws.cloudtrail.eventTime",
+      "type": "alias"
     }
   }
 }
diff --git a/src/main/resources/OSMapping/dns/fieldmappings.yml b/src/main/resources/OSMapping/dns/fieldmappings.yml
@@ -1,4 +1,5 @@
 fieldmappings:
     record_type: dns-answers-type
     query: dns-question-name
-    parent_domain: dns-question-registered_domain
+    parent_domain: dns-question-registered_domain
+    creationTime: timestamp
diff --git a/src/main/resources/OSMapping/dns/mappings.json b/src/main/resources/OSMapping/dns/mappings.json
@@ -11,6 +11,10 @@
     "dns-question-registered_domain": {
       "type": "alias",
       "path": "dns.question.registered_domain"
+    },
+    "timestamp": {
+      "path": "creationTime",
+      "type": "alias"
     }
   }
 }
diff --git a/src/main/resources/OSMapping/linux/fieldmappings.yml b/src/main/resources/OSMapping/linux/fieldmappings.yml
@@ -12,4 +12,4 @@ fieldmappings:
   ParentImage: process-parent-executable
   CurrentDirectory: process-working_directory
   LogonId: process-real_user-id
-
+  creationTime: timestamp
diff --git a/src/main/resources/OSMapping/linux/mappings.json b/src/main/resources/OSMapping/linux/mappings.json
@@ -47,6 +47,10 @@
     "process-real_user-id": {
       "path": "process.real_user.id",
       "type": "alias"
+    },
+    "timestamp": {
+      "path": "creationTime",
+      "type": "alias"
     }
   }
 }
diff --git a/src/main/resources/OSMapping/network/NetFlowMapping.json b/src/main/resources/OSMapping/network/NetFlowMapping.json
@@ -23,6 +23,10 @@
     "http.response.status_code": {
       "type": "alias",
       "path": "netflow.http_status_code"
+    },
+    "timestamp": {
+      "path": "creationTime",
+      "type": "alias"
     }
   }
 }
diff --git a/src/main/resources/OSMapping/network/fieldmappings.yml b/src/main/resources/OSMapping/network/fieldmappings.yml
@@ -14,4 +14,5 @@ fieldmappings:
     client_header_names: zeek-http-client_header_names
     resp_mime_types: zeek-http-resp_mime_types
     cipher: zeek-kerberos-cipher
-    request_type: zeek-kerberos-request_type
+    request_type: zeek-kerberos-request_type
+    creationTime: timestamp
diff --git a/src/main/resources/OSMapping/network/mappings.json b/src/main/resources/OSMapping/network/mappings.json
@@ -59,6 +59,10 @@
     "zeek-kerberos-request_type": {
       "path": "zeek.kerberos.request_type",
       "type": "alias"
+    },
+    "timestamp": {
+      "path": "creationTime",
+      "type": "alias"
     }
   }
 }
diff --git a/src/main/resources/OSMapping/others_application/fieldmappings.yml b/src/main/resources/OSMapping/others_application/fieldmappings.yml
@@ -4,4 +4,4 @@ fieldmappings:
   HiveName: unmapped.HiveName
   fieldB: mappedB
   fieldA1: mappedA
-
+  creationTime: timestamp
diff --git a/src/main/resources/OSMapping/others_application/mappings.json b/src/main/resources/OSMapping/others_application/mappings.json
@@ -23,6 +23,10 @@
     "windows-servicename": {
       "type": "alias",
       "path": "ServiceName"
+    },
+    "timestamp": {
+      "path": "creationTime",
+      "type": "alias"
     }
   }
 }
diff --git a/src/main/resources/OSMapping/others_apt/fieldmappings.yml b/src/main/resources/OSMapping/others_apt/fieldmappings.yml
@@ -4,4 +4,4 @@ fieldmappings:
   HiveName: unmapped.HiveName
   fieldB: mappedB
   fieldA1: mappedA
-
+  creationTime: timestamp
diff --git a/src/main/resources/OSMapping/others_apt/mappings.json b/src/main/resources/OSMapping/others_apt/mappings.json
@@ -23,6 +23,10 @@
     "windows-servicename": {
       "type": "alias",
       "path": "ServiceName"
+    },
+    "timestamp": {
+      "path": "creationTime",
+      "type": "alias"
     }
   }
 }
diff --git a/src/main/resources/OSMapping/others_cloud/fieldmappings.yml b/src/main/resources/OSMapping/others_cloud/fieldmappings.yml
@@ -4,4 +4,4 @@ fieldmappings:
   HiveName: unmapped.HiveName
   fieldB: mappedB
   fieldA1: mappedA
-
+  creationTime: timestamp
diff --git a/src/main/resources/OSMapping/others_cloud/mappings.json b/src/main/resources/OSMapping/others_cloud/mappings.json
@@ -23,6 +23,10 @@
     "windows-servicename": {
       "type": "alias",
       "path": "ServiceName"
+    },
+    "creationTime": {
+      "path": "creationTime",
+      "type": "alias"
     }
   }
 }
diff --git a/src/main/resources/OSMapping/others_compliance/fieldmappings.yml b/src/main/resources/OSMapping/others_compliance/fieldmappings.yml
@@ -4,4 +4,4 @@ fieldmappings:
   HiveName: unmapped.HiveName
   fieldB: mappedB
   fieldA1: mappedA
-
+  creationTime: timestamp
diff --git a/src/main/resources/OSMapping/others_compliance/mappings.json b/src/main/resources/OSMapping/others_compliance/mappings.json
@@ -23,6 +23,10 @@
     "windows-servicename": {
       "type": "alias",
       "path": "ServiceName"
+    },
+    "timestamp": {
+      "path": "creationTime",
+      "type": "alias"
     }
   }
 }
diff --git a/src/main/resources/OSMapping/others_macos/fieldmappings.yml b/src/main/resources/OSMapping/others_macos/fieldmappings.yml
@@ -4,3 +4,4 @@ fieldmappings:
     HiveName: unmapped.HiveName
     fieldB: mappedB
     fieldA1: mappedA
+    creationTime: timestamp