Skip to content

Commit

Permalink
merge from main
Browse files Browse the repository at this point in the history 
Signed-off-by: Petar Dzepina <[email protected]>
  • Loading branch information
@petardz
petardz committed Jan 10, 2023
2 parents 1626aa8 + 8b7f1fd commit 5454203
Show file tree
Hide file tree
Showing 17 changed files with 704 additions and 151 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import org.opensearch.gradle.test.RestIntegTestTask

buildscript {
ext {
opensearch_version = System.getProperty("opensearch.version", "2.4.0-SNAPSHOT")
opensearch_version = System.getProperty("opensearch.version", "2.5.0-SNAPSHOT")
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
buildVersionQualifier = System.getProperty("build.version_qualifier", "")
version_tokens = opensearch_version.tokenize('-')
Expand Down
12 changes: 10 additions & 2 deletions src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ public class DetectorMonitorConfig {
public static final String OPENSEARCH_DEFAULT_ALL_FINDINGS_INDICES_PATTERN = ".opensearch-sap-findings-default*";
public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN = "<.opensearch-sap-findings-default-{now/d}-1>";

public static final String OPENSEARCH_SAP_RULE_INDEX_TEMPLATE = ".opensearch-sap-detectors-queries-index-template";

private static Map<String, MonitorConfig> detectorTypeToIndicesMapping;

static {
Expand Down Expand Up @@ -113,6 +115,13 @@ public static List<String> getAllFindingsIndicesPatternForAllTypes() {
.collect(Collectors.toList());
}

public static List<String> getAllRuleIndices() {
return detectorTypeToIndicesMapping.entrySet()
.stream()
.map(e -> e.getValue().getRuleIndex())
.collect(Collectors.toList());
}

public static String getFindingsIndexPattern(String detectorType) {
return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ?
detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getFindingsIndexPattern() :
Expand Down Expand Up @@ -145,8 +154,7 @@ private MonitorConfig(
String findingsIndex,
String findingsIndexPattern,
String allFindingsIndicesPattern,
String ruleIndex
) {
String ruleIndex) {
this.alertsIndex = alertsIndex;
this.alertsHistoryIndex = alertsHistoryIndex;
this.alertsHistoryIndexPattern = alertsHistoryIndexPattern;
Expand Down
48 changes: 5 additions & 43 deletions src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteDetectorAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ public void onResponse(GetResponse response) {

@Override
public void onFailure(Exception t) {
onFailures(t);
onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Detector with %s is not found", detectorId), RestStatus.NOT_FOUND));
}
});
}
Expand All @@ -152,48 +152,7 @@ public void onResponse(Collection<DeleteMonitorResponse> responses) {
}).count() > 0) {
onFailures(new OpenSearchStatusException("Monitor associated with detected could not be deleted", errorStatusSupplier.get()));
}
ruleTopicIndices.countQueries(ruleIndex, new ActionListener<>() {
@Override
public void onResponse(SearchResponse response) {
if (response.isTimedOut()) {
log.info("Count response timed out");
deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy());
} else {
long count = response.getHits().getTotalHits().value;

if (count == 0) {
try {
ruleTopicIndices.deleteRuleTopicIndex(ruleIndex,
new ActionListener<>() {
@Override
public void onResponse(AcknowledgedResponse response) {
deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy());
}

@Override
public void onFailure(Exception e) {
// error is suppressed as it is not a critical deletion
log.info(e.getMessage());
deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy());
}
});
} catch (IOException e) {
deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy());
}
} else {
deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy());
}
}
}

@Override
public void onFailure(Exception e) {
// error is suppressed as it is not a critical deletion
log.info(e.getMessage());


}
});
deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy());
}

@Override
Expand Down Expand Up @@ -240,6 +199,9 @@ private void onFailures(Exception t) {
private void finishHim(String detectorId, Exception t) {
threadPool.executor(ThreadPool.Names.GENERIC).execute(ActionRunnable.supply(listener, () -> {
if (t != null) {
if (t instanceof OpenSearchStatusException) {
throw t;
}
throw SecurityAnalyticsException.wrap(t);
} else {
return new DeleteDetectorResponse(detectorId, NO_VERSION, RestStatus.NO_CONTENT);
Expand Down
5 changes: 4 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteRuleAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ public void onResponse(GetResponse response) {

@Override
public void onFailure(Exception e) {
onFailures(e);
onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Rule with %s is not found", ruleId), RestStatus.NOT_FOUND));
}
});
}
Expand Down Expand Up @@ -273,6 +273,9 @@ private void onFailures(Exception t) {
private void finishHim(String ruleId, Exception t) {
threadPool.executor(ThreadPool.Names.GENERIC).execute(ActionRunnable.supply(listener, () -> {
if (t != null) {
if (t instanceof OpenSearchStatusException) {
throw t;
}
throw SecurityAnalyticsException.wrap(t);
} else {
return new DeleteRuleResponse(ruleId, NO_VERSION, RestStatus.NO_CONTENT);
Expand Down
23 changes: 16 additions & 7 deletions src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
import org.opensearch.commons.alerting.model.SearchInput;
import org.opensearch.commons.alerting.model.action.Action;
import org.opensearch.commons.authuser.User;
import org.opensearch.index.IndexNotFoundException;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.index.reindex.BulkByScrollResponse;
Expand Down Expand Up @@ -197,8 +198,13 @@ public void onFailure(Exception e) {
listener.onFailure(SecurityAnalyticsException.wrap(
new OpenSearchStatusException(String.format(Locale.getDefault(), "User doesn't have read permissions for one or more configured index %s", detectorIndices), RestStatus.FORBIDDEN)
));
} else {
listener.onFailure(e);
} else if (e instanceof IndexNotFoundException) {
listener.onFailure(SecurityAnalyticsException.wrap(
new OpenSearchStatusException(String.format(Locale.getDefault(), "Indices not found %s", String.join(", ", detectorIndices)), RestStatus.NOT_FOUND)
));
}
else {
listener.onFailure(SecurityAnalyticsException.wrap(e));
}
}
});
Expand Down Expand Up @@ -696,9 +702,9 @@ void createDetector() {

if (!detector.getInputs().isEmpty()) {
try {
ruleTopicIndices.initRuleTopicIndex(detector.getRuleIndex(), new ActionListener<>() {
ruleTopicIndices.initRuleTopicIndexTemplate(new ActionListener<>() {
@Override
public void onResponse(CreateIndexResponse createIndexResponse) {
public void onResponse(AcknowledgedResponse acknowledgedResponse) {

initRuleIndexAndImportRules(request, new ActionListener<>() {
@Override
Expand Down Expand Up @@ -802,9 +808,9 @@ void onGetResponse(Detector currentDetector, User user) {

if (!detector.getInputs().isEmpty()) {
try {
ruleTopicIndices.initRuleTopicIndex(detector.getRuleIndex(), new ActionListener<>() {
ruleTopicIndices.initRuleTopicIndexTemplate(new ActionListener<>() {
@Override
public void onResponse(CreateIndexResponse createIndexResponse) {
public void onResponse(AcknowledgedResponse acknowledgedResponse) {
initRuleIndexAndImportRules(request, new ActionListener<>() {
@Override
public void onResponse(List<IndexMonitorResponse> monitorResponses) {
Expand Down Expand Up @@ -991,7 +997,7 @@ public void onResponse(SearchResponse response) {
if (ruleIndices.ruleIndexExists(false)) {
importCustomRules(detector, detectorInput, queries, listener);
} else if (detectorInput.getCustomRules().size() > 0) {
onFailures(new OpenSearchStatusException("Custom Rule Index not found", RestStatus.BAD_REQUEST));
onFailures(new OpenSearchStatusException("Custom Rule Index not found", RestStatus.NOT_FOUND));
} else {
if (request.getMethod() == RestRequest.Method.POST) {
createMonitorFromQueries(logIndex, queries, detector, listener, request.getRefreshPolicy());
Expand Down Expand Up @@ -1109,6 +1115,9 @@ private void onFailures(Exception t) {
private void finishHim(Detector detector, Exception t) {
threadPool.executor(ThreadPool.Names.GENERIC).execute(ActionRunnable.supply(listener, () -> {
if (t != null) {
if (t instanceof OpenSearchStatusException) {
throw t;
}
throw SecurityAnalyticsException.wrap(t);
} else {
return new IndexDetectorResponse(detector.getId(), detector.getVersion(), request.getMethod() == RestRequest.Method.POST? RestStatus.CREATED: RestStatus.OK, detector);
Expand Down
58 changes: 22 additions & 36 deletions src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,22 @@
*/
package org.opensearch.securityanalytics.util;

import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.action.ActionListener;
import org.opensearch.action.admin.indices.create.CreateIndexRequest;
import org.opensearch.action.admin.indices.create.CreateIndexResponse;
import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.opensearch.action.support.master.AcknowledgedResponse;
import org.opensearch.client.AdminClient;
import org.opensearch.client.Client;
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.search.builder.SearchSourceBuilder;

import java.io.IOException;
import java.nio.charset.Charset;
import java.util.Objects;
import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig;

public class RuleTopicIndices {
private static final Logger log = LogManager.getLogger(DetectorIndices.class);
Expand All @@ -37,40 +33,30 @@ public RuleTopicIndices(Client client, ClusterService clusterService) {
this.clusterService = clusterService;
}

public static String ruleTopicIndexMappings() throws IOException {
return new String(Objects.requireNonNull(DetectorIndices.class.getClassLoader().getResourceAsStream("mappings/detector-queries.json")).readAllBytes(), Charset.defaultCharset());
}

public static String ruleTopicIndexSettings() throws IOException {
return new String(Objects.requireNonNull(DetectorIndices.class.getClassLoader().getResourceAsStream("mappings/detector-settings.json")).readAllBytes(), Charset.defaultCharset());
}

public void initRuleTopicIndex(String ruleTopicIndex, ActionListener<CreateIndexResponse> actionListener) throws IOException {
if (!ruleTopicIndexExists(ruleTopicIndex)) {
CreateIndexRequest indexRequest = new CreateIndexRequest(ruleTopicIndex)
.mapping(ruleTopicIndexMappings())
public void initRuleTopicIndexTemplate(ActionListener<AcknowledgedResponse> actionListener) throws IOException {
if (!ruleTopicIndexTemplateExists()) {
// Compose list of all patterns to cover all query indices
List<String> indexPatterns = new ArrayList<>();
for(String ruleIndex : DetectorMonitorConfig.getAllRuleIndices()) {
indexPatterns.add(ruleIndex + "*");
}
PutIndexTemplateRequest indexRequest =
new PutIndexTemplateRequest(DetectorMonitorConfig.OPENSEARCH_SAP_RULE_INDEX_TEMPLATE)
.patterns(indexPatterns)
.settings(Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build());
client.admin().indices().create(indexRequest, actionListener);
client.admin().indices().putTemplate(indexRequest, actionListener);
} else {
actionListener.onResponse(new CreateIndexResponse(true, true, ruleTopicIndex));
actionListener.onResponse(new AcknowledgedResponse(true));
}
}

public void deleteRuleTopicIndex(String ruleTopicIndex, ActionListener<AcknowledgedResponse> actionListener) throws IOException {
if (ruleTopicIndexExists(ruleTopicIndex)) {
DeleteIndexRequest request = new DeleteIndexRequest(ruleTopicIndex);
client.admin().indices().delete(request, actionListener);
}
}

public void countQueries(String ruleTopicIndex, ActionListener<SearchResponse> listener) {
SearchRequest request = new SearchRequest(ruleTopicIndex)
.source(new SearchSourceBuilder().size(0));
client.search(request, listener);
}

public boolean ruleTopicIndexExists(String ruleTopicIndex) {
public boolean ruleTopicIndexTemplateExists() {
ClusterState clusterState = clusterService.state();
return clusterState.getRoutingTable().hasIndex(ruleTopicIndex);
return clusterState.metadata().templates()
.get(DetectorMonitorConfig.OPENSEARCH_SAP_RULE_INDEX_TEMPLATE) != null;
}
}
72 changes: 63 additions & 9 deletions src/main/resources/OSMapping/windows/fieldmappings.yml
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,65 @@
# this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under windows log group to their corresponding ECS Fields.
fieldmappings:
EventID: event_uid
HiveName: unmapped.HiveName
fieldB: mappedB
fieldA1: mappedA
CommandLine: windows-event_data-CommandLine
HostName: windows-hostname
Message: windows-message
Provider_Name: windows-provider-name
ServiceName: windows-servicename
AccountName: winlog-computerObject-name
AuthenticationPackageName: winlog-event_data-AuthenticationPackageName
Channel: winlog-channel
Company: winlog-event_data-Company
ComputerName: winlog-computer_name
Description: winlog-event_data-Description
Details: winlog-event_data-Detail
Device: winlog-event_data-DeviceName
DeviceName: winlog-event_data-DeviceName
FileName: winlog-event_data-OriginalFileName
FileVersion: winlog-event_data-FileVersion
IntegrityLevel: winlog-event_data-IntegrityLevel
IpAddress: winlog-event_data-IpAddress
KeyLength: winlog-event_data-KeyLength
Keywords: winlog-keywords
LogonId: winlog-event_data-LogonId
LogonProcessName: winlog-event_data-LogonProcessName
LogonType: winlog-event_data-LogonType
OriginalFileName: winlog-event_data-OriginalFileName
OriginalFilename: winlog-event_data-OriginalFileName
Path: winlog-event_data-Path
PrivilegeList: winlog-event_data-PrivilegeList
ProcessId: winlog-event_data-ProcessId
Product: winlog-event_data-Product
Provider: winlog-provider_name
ProviderName: winlog-provider_name
ScriptBlockText: winlog-event_data-ScriptBlockText
ServerName: winlog-event_data-TargetServerName
Service: winlog-event_data-ServiceName
Signed: winlog-event_data-Signed
State: winlog-event_data-State
Status: winlog-event_data-Status
SubjectDomainName: winlog-event_data-SubjectDomainName
SubjectLogonId: winlog-event_data-SubjectLogonId
SubjectUserName: winlog-event_data-SubjectUserName
SubjectUserSid: winlog-event_data-SubjectUserSid
TargetLogonId: winlog-event_data-TargetLogonId
TargetName: winlog-event_data-TargetUserName
TargetServerName: winlog-event_data-TargetServerName
TargetUserName: winlog-event_data-TargetUserName
TargetUserSid: winlog-event_data-TargetUserSid
TaskName: winlog-task
Type: winlog-user-type
User: winlog-user-name
UserName: winlog-user-name
Workstation: winlog-event_data-Workstation
WorkstationName: winlog-event_data-Workstation
event_uid: winlog-event_id
CommandLine: server-user-hash
hostname: host-hostname
message: windows-message
Provider_Name: winlog-provider_name
EventId: winlog-event_id
processPath: winlog-event_data-ProcessPath
ProcessName: winlog-event_data-ProcessName
ObjectName: winlog-computerObject-name
param1: winlog-event_data-param1
param2: winlog-event_data-param2
windows-hostname: winlog-computer_name
windows-provider-name: winlog-provider_name
windows-servicename: winlog-event_data-ServiceName


Loading

0 comments on commit 5454203

Please sign in to comment.