Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* changed windows sample rule and query construction
Browse files Browse the repository at this point in the history
Signed-off-by: Joanne Wang <[email protected]>

* remove wildcard

Signed-off-by: Joanne Wang <[email protected]>

* changed wildcardtest

Signed-off-by: Joanne Wang <[email protected]>

* fixed wildcards

Signed-off-by: Joanne Wang <[email protected]>

* fixed wildcard query test

Signed-off-by: Joanne Wang <[email protected]>

* fixed correlation engine tests

Signed-off-by: Joanne Wang <[email protected]>

* fixed query backend tests

Signed-off-by: Joanne Wang <[email protected]>

* clean up

Signed-off-by: Joanne Wang <[email protected]>

* added two integration tests

Signed-off-by: Joanne Wang <[email protected]>

---------

Signed-off-by: Joanne Wang <[email protected]>
jowg-amazon committed Mar 11, 2024
1 parent 830a3b9 commit 8de7fa0
Showing 5 changed files with 539 additions and 23 deletions.
Original file line number Diff line number Diff line change
@@ -132,10 +132,10 @@ public OSQueryBackend(String ruleCategory, boolean collectErrors, boolean enable
this.reExpression = "%s: /%s/";
this.cidrExpression = "%s: \"%s\"";
this.fieldNullExpression = "%s: null";
this.unboundValueStrExpression = "%s: \"%s\"";
this.unboundValueNumExpression = "%s: %s";
this.unboundWildcardExpression = "%s: %s";
this.unboundReExpression = "%s: /%s/";
this.unboundValueStrExpression = "\"%s\"";
this.unboundValueNumExpression = "\"%s\"";
this.unboundWildcardExpression = "%s";
this.unboundReExpression = "/%s/";
this.compareOpExpression = "\"%s\" \"%s\" %s";
this.valExpCount = 0;
this.aggQuery = "{\"%s\":{\"terms\":{\"field\":\"%s\"},\"aggs\":{\"%s\":{\"%s\":{\"field\":\"%s\"}}}}}";
@@ -332,28 +332,18 @@ public Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpre
@Override
public Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError {
SigmaString value = (SigmaString) condition.getValue();

String field = getFinalValueField();
ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer"));
boolean containsWildcard = value.containsWildcard();
return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), field, this.convertValueStr((SigmaString) condition.getValue()));
return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), this.convertValueStr((SigmaString) condition.getValue()));
}

@Override
public Object convertConditionValNum(ConditionValueExpression condition) {
String field = getFinalValueField();

SigmaNumber number = (SigmaNumber) condition.getValue();
ruleQueryFields.put(field, number.getNumOpt().isLeft()? Collections.singletonMap("type", "integer"): Collections.singletonMap("type", "float"));

return String.format(Locale.getDefault(), this.unboundValueNumExpression, field, condition.getValue().toString());
return String.format(Locale.getDefault(), this.unboundValueNumExpression, condition.getValue().toString());
}

@Override
public Object convertConditionValRe(ConditionValueExpression condition) {
String field = getFinalValueField();
ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer"));
return String.format(Locale.getDefault(), this.unboundReExpression, field, convertValueRe((SigmaRegularExpression) condition.getValue()));
return String.format(Locale.getDefault(), this.unboundReExpression, convertValueRe((SigmaRegularExpression) condition.getValue()));
}

// TODO: below methods will be supported when Sigma Expand Modifier is supported.
4 changes: 3 additions & 1 deletion src/main/resources/rules/test_windows/win_sample_rule.yml
Original file line number Diff line number Diff line change
@@ -19,6 +19,8 @@ detection:
EventID: 22
Message|contains: 'C:\\Program Files\\nxlog\\nxlog.exe'
HostName|startswith: 'EC2AMAZ'
condition: selection
keywords:
- "NT AUTHORITY"
condition: selection or keywords
falsepositives:
- Unknown
248 changes: 247 additions & 1 deletion src/test/java/org/opensearch/securityanalytics/TestHelpers.java
Original file line number Diff line number Diff line change
@@ -186,6 +186,188 @@ public static String randomRule() {
"level: high";
}

public static String randomRuleForMappingView(String field) {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection:\n" +
" "+ field + ": 'ACL'\n" +
" condition: selection\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomRuleForCustomLogType() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection:\n" +
" EventID: 22\n" +
" Author: 'Hello'\n" +
" condition: selection\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomRuleWithAlias() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection:\n" +
" event_uid: 22\n" +
" condition: selection\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomRuleWithKeywords() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection:\n" +
" EventID: 21\n" +
" keywords:\n" +
" - 1996\n" +
" - EC2AMAZ*\n" +
" condition: selection or keywords\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomRuleWithStringKeywords() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection:\n" +
" EventID: 21\n" +
" keywords:\n" +
" - \"INFO\"\n" +
" condition: selection or keywords\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomRuleWithDateKeywords() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection:\n" +
" EventID: 21\n" +
" keywords:\n" +
" - \"2020-02-04T14:59:39.343541+00:00\"\n" +
" condition: selection or keywords\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String countAggregationTestRule() {
return " title: Test\n" +
" id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" +
@@ -1175,6 +1357,48 @@ public static String windowsIndexMapping() {
" }";
}

public static String windowsIndexMappingOnlyNumericAndDate() {
return "\"properties\": {\n" +
" \"@timestamp\": {\"type\":\"date\"},\n" +
" \"EventTime\": {\n" +
" \"type\": \"date\"\n" +
" },\n" +
" \"ExecutionProcessID\": {\n" +
" \"type\": \"long\"\n" +
" },\n" +
" \"ExecutionThreadID\": {\n" +
" \"type\": \"integer\"\n" +
" },\n" +
" \"EventID\": {\n" +
" \"type\": \"integer\"\n" +
" },\n" +
" \"TaskValue\": {\n" +
" \"type\": \"integer\"\n" +
" }\n" +
" }";
}

public static String windowsIndexMappingOnlyNumericAndText() {
return "\"properties\": {\n" +
" \"TaskName\": {\n" +
" \"type\": \"text\"\n" +
" },\n" +
" \"ExecutionProcessID\": {\n" +
" \"type\": \"long\"\n" +
" },\n" +
" \"ExecutionThreadID\": {\n" +
" \"type\": \"integer\"\n" +
" },\n" +
" \"EventID\": {\n" +
" \"type\": \"integer\"\n" +
" },\n" +
" \"TaskValue\": {\n" +
" \"type\": \"integer\"\n" +
" }\n" +
" }";
}


public static String randomDoc(int severity, int version, String opCode) {
String doc = "{\n" +
"\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
@@ -1214,6 +1438,28 @@ public static String randomDoc(int severity, int version, String opCode) {

}

public static String randomDocOnlyNumericAndDate(int severity, int version, String opCode) {
String doc = "{\n" +
"\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
"\"ExecutionProcessID\":2001,\n" +
"\"ExecutionThreadID\":2616,\n" +
"\"EventID\": 1234,\n" +
"\"TaskValue\":22\n" +
"}";
return String.format(Locale.ROOT, doc, severity, version, opCode);
}

public static String randomDocOnlyNumericAndText(int severity, int version, String opCode) {
String doc = "{\n" +
"\"TaskName\":\"SYSTEM\",\n" +
"\"ExecutionProcessID\":2001,\n" +
"\"ExecutionThreadID\":2616,\n" +
"\"EventID\": 1234,\n" +
"\"TaskValue\":22\n" +
"}";
return String.format(Locale.ROOT, doc, severity, version, opCode);
}

public static String randomDoc() {
return "{\n" +
"\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
@@ -1278,7 +1524,7 @@ public static String randomAppLogDoc() {
return "{\n" +
" \"endpoint\": \"/customer_records.txt\",\n" +
" \"http_method\": \"POST\",\n" +
" \"keywords\": \"PermissionDenied\"\n" +
" \"keywords\": \"INVALID\"\n" +
"}";
}

Original file line number Diff line number Diff line change
@@ -12,6 +12,13 @@
import static org.opensearch.securityanalytics.TestHelpers.randomIndex;
import static org.opensearch.securityanalytics.TestHelpers.randomRule;
import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping;
import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndText;
import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithDateKeywords;
import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithKeywords;
import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithStringKeywords;
import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndDate;
import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndDate;
import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndText;

import java.io.IOException;
import java.util.ArrayList;
@@ -974,6 +981,252 @@ else if (ruleId == minRuleId) {
assertTrue(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8").containsAll(docLevelFinding));
}

public void testCreateDetectorWithKeywordsRule_verifyFindings_success() throws IOException {
String index = createTestIndex(randomIndex(), windowsIndexMapping());

// Execute CreateMappingsAction to add alias mapping for index
Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
// both req params and req body are supported
createMappingRequest.setJsonEntity(
"{ \"index_name\":\"" + index + "\"," +
" \"rule_topic\":\"" + randomDetectorType() + "\", " +
" \"partial\":true" +
"}"
);

Response createMappingResponse = client().performRequest(createMappingRequest);

assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode());

// Create random doc rule
String randomDocRuleId = createRule(randomRuleWithKeywords());
List<String> prepackagedRules = getRandomPrePackagedRules();
DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)),
prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList()));
Detector detector = randomDetectorWithInputs(List.of(input));

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));

assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

Map updateResponseBody = asMap(createResponse);
String detectorId = updateResponseBody.get("_id").toString();
String request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + detectorId + "\"\n" +
" }\n" +
" }\n" +
"}";

// Verify newly created doc level monitor
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);
Map<String, Object> detectorAsMap = (Map<String, Object>) hit.getSourceAsMap().get("detector");
List<String> monitorIds = ((List<String>) (detectorAsMap).get("monitor_id"));

assertEquals(1, monitorIds.size());

String monitorId = monitorIds.get(0);
String monitorType = ((Map<String, String>) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type");

assertEquals(MonitorType.DOC_LEVEL_MONITOR.getValue(), monitorType);

// Verify rules
request = "{\n" +
" \"query\" : {\n" +
" \"match_all\":{\n" +
" }\n" +
" }\n" +
"}";
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);

assertEquals(6, response.getHits().getTotalHits().value);

// Verify findings
indexDoc(index, "1", randomDoc(2, 5, "Test"));
indexDoc(index, "2", randomDoc(3, 5, "Test"));


Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap());
Map<String, Object> executeResults = entityAsMap(executeResponse);
int noOfSigmaRuleMatches = ((List<Map<String, Object>>) ((Map<String, Object>) executeResults.get("input_results")).get("results")).get(0).size();
// Verify 5 prepackaged rules and 1 custom rule
assertEquals(6, noOfSigmaRuleMatches);

Map<String, String> params = new HashMap<>();
params.put("detector_id", detectorId);
Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null);
Map<String, Object> getFindingsBody = entityAsMap(getFindingsResponse);

assertNotNull(getFindingsBody);
// When doc level monitor is being applied one finding is generated per document
assertEquals(2, getFindingsBody.get("total_findings"));

Set<String> docRuleIds = new HashSet<>(prepackagedRules);
docRuleIds.add(randomDocRuleId);

List<Map<String, Object>> findings = (List) getFindingsBody.get("findings");
List<String> foundDocIds = new ArrayList<>();
for (Map<String, Object> finding : findings) {
Set<String> aggRulesFinding = ((List<Map<String, Object>>) finding.get("queries")).stream().map(it -> it.get("id").toString()).collect(
Collectors.toSet());

assertTrue(docRuleIds.containsAll(aggRulesFinding));

List<String> findingDocs = (List<String>) finding.get("related_doc_ids");
Assert.assertEquals(1, findingDocs.size());
foundDocIds.addAll(findingDocs);
}
assertTrue(Arrays.asList("1", "2").containsAll(foundDocIds));
}

public void testCreateDetectorWithKeywordsRule_ensureNoFindingsWithoutTextMapping_success() throws IOException {
String index = createTestIndex(randomIndex(), windowsIndexMappingOnlyNumericAndDate());

// Execute CreateMappingsAction to add alias mapping for index
Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
// both req params and req body are supported
createMappingRequest.setJsonEntity(
"{ \"index_name\":\"" + index + "\"," +
" \"rule_topic\":\"" + randomDetectorType() + "\", " +
" \"partial\":true" +
"}"
);

Response createMappingResponse = client().performRequest(createMappingRequest);

assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode());

// Create random doc rule
String randomDocRuleId = createRule(randomRuleWithStringKeywords());
DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)),
Collections.emptyList());
Detector detector = randomDetectorWithInputs(List.of(input));

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));

assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

Map updateResponseBody = asMap(createResponse);
String detectorId = updateResponseBody.get("_id").toString();
String request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + detectorId + "\"\n" +
" }\n" +
" }\n" +
"}";

// Verify newly created doc level monitor
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);
Map<String, Object> detectorAsMap = (Map<String, Object>) hit.getSourceAsMap().get("detector");
List<String> monitorIds = ((List<String>) (detectorAsMap).get("monitor_id"));

assertEquals(1, monitorIds.size());

String monitorId = monitorIds.get(0);
String monitorType = ((Map<String, String>) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type");

assertEquals(MonitorType.DOC_LEVEL_MONITOR.getValue(), monitorType);

// Verify rules created
request = "{\n" +
" \"query\" : {\n" +
" \"match_all\":{\n" +
" }\n" +
" }\n" +
"}";
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);

assertEquals(1, response.getHits().getTotalHits().value);

// Insert test document
indexDoc(index, "1", randomDocOnlyNumericAndDate(2, 5, "Test"));
indexDoc(index, "2", randomDocOnlyNumericAndDate(3, 5, "Test"));


Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap());
Map<String, Object> executeResults = entityAsMap(executeResponse);
int noOfSigmaRuleMatches = ((List<Map<String, Object>>) ((Map<String, Object>) executeResults.get("input_results")).get("results")).get(0).size();
// Verify no rules match test document
assertEquals(0, noOfSigmaRuleMatches);
}

public void testCreateDetectorWithKeywordsRule_ensureNoFindingsWithoutDateMapping_success() throws IOException {
String index = createTestIndex(randomIndex(), windowsIndexMappingOnlyNumericAndText());

// Execute CreateMappingsAction to add alias mapping for index
Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
// both req params and req body are supported
createMappingRequest.setJsonEntity(
"{ \"index_name\":\"" + index + "\"," +
" \"rule_topic\":\"" + randomDetectorType() + "\", " +
" \"partial\":true" +
"}"
);

Response createMappingResponse = client().performRequest(createMappingRequest);

assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode());

// Create random doc rule
String randomDocRuleId = createRule(randomRuleWithDateKeywords());
DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)),
Collections.emptyList());
Detector detector = randomDetectorWithInputs(List.of(input));

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));

assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

Map updateResponseBody = asMap(createResponse);
String detectorId = updateResponseBody.get("_id").toString();
String request = "{\n" +
" \"query\" : {\n" +
" \"match\":{\n" +
" \"_id\": \"" + detectorId + "\"\n" +
" }\n" +
" }\n" +
"}";

// Verify newly created doc level monitor
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);
Map<String, Object> detectorAsMap = (Map<String, Object>) hit.getSourceAsMap().get("detector");
List<String> monitorIds = ((List<String>) (detectorAsMap).get("monitor_id"));

assertEquals(1, monitorIds.size());

String monitorId = monitorIds.get(0);
String monitorType = ((Map<String, String>) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type");

assertEquals(MonitorType.DOC_LEVEL_MONITOR.getValue(), monitorType);

// Verify rules created
request = "{\n" +
" \"query\" : {\n" +
" \"match_all\":{\n" +
" }\n" +
" }\n" +
"}";
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true);

assertEquals(1, response.getHits().getTotalHits().value);

// Insert test document
indexDoc(index, "1", randomDocOnlyNumericAndText(2, 5, "Test"));
indexDoc(index, "2", randomDocOnlyNumericAndText(3, 5, "Test"));


Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap());
Map<String, Object> executeResults = entityAsMap(executeResponse);
int noOfSigmaRuleMatches = ((List<Map<String, Object>>) ((Map<String, Object>) executeResults.get("input_results")).get("results")).get(0).size();
// Verify no rules match test document
assertEquals(0, noOfSigmaRuleMatches);
}

private static void assertRuleMonitorFinding(Map<String, Object> executeResults, String ruleId, int expectedDocCount, List<String> expectedTriggerResult) {
List<Map<String, Object>> buckets = ((List<Map<String, Object>>)(((Map<String, Object>)((Map<String, Object>)((Map<String, Object>)((List<Object>)((Map<String, Object>) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets")));
Integer docCount = buckets.stream().mapToInt(it -> (Integer)it.get("doc_count")).sum();
Original file line number Diff line number Diff line change
@@ -322,7 +322,7 @@ public void testConvertValueRegexUnbound() throws IOException, SigmaError {
" sel:\n" +
" \"|re\": pat.*tern\"foo\"bar\n" +
" condition: sel", false));
Assert.assertEquals("_0: /pat.*tern\\\"foo\\\"bar/", queries.get(0).toString());
Assert.assertEquals("/pat.*tern\\\"foo\\\"bar/", queries.get(0).toString());
}

public void testConvertValueCidrWildcardNone() throws IOException, SigmaError {
@@ -478,7 +478,7 @@ public void testConvertOrInMixedKeywordField() throws IOException, SigmaError {
" fieldB: value2\n" +
" sel3: value3\n" +
" condition: sel1 or sel2 or sel3", false));
Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) OR (_0: \"value3\")", queries.get(0).toString());
Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) OR (\"value3\")", queries.get(0).toString());
}

public void testConvertOrInMixedFields() throws IOException, SigmaError {
@@ -591,9 +591,9 @@ public void testConvertUnboundValues() throws IOException, SigmaError {
" sel:\n" +
" - value1\n" +
" - value2\n" +
" - 4\n" +
" - 123\n" +
" condition: sel", false));
Assert.assertEquals("(_0: \"value1\") OR (_1: \"value2\") OR (_2: 4)", queries.get(0).toString());
Assert.assertEquals("(\"value1\") OR (\"value2\") OR (\"123\")", queries.get(0).toString());
}

public void testConvertInvalidUnboundBool() throws IOException {
@@ -876,6 +876,31 @@ public void testConvertProxyRule() throws IOException, SigmaError {
Assert.assertEquals(true, true);
}

public void testConvertUnboundValuesAsWildcard() throws IOException, SigmaError {
OSQueryBackend queryBackend = testBackend();
List<Object> queries = queryBackend.convertRule(SigmaRule.fromYaml(
" title: Test\n" +
" id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" +
" status: test\n" +
" level: critical\n" +
" description: Detects QuarksPwDump clearing access history in hive\n" +
" author: Florian Roth\n" +
" date: 2017/05/15\n" +
" logsource:\n" +
" category: test_category\n" +
" product: test_product\n" +
" detection:\n" +
" sel:\n" +
" fieldA1: \n" +
" - value1\n" +
" - value2\n" +
" - value3\n" +
" keywords:\n" +
" - test*\n" +
" condition: sel or keywords", false));
Assert.assertEquals("((mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: \"value3\")) OR (test*)", queries.get(0).toString());
}

private OSQueryBackend testBackend() throws IOException {
return new OSQueryBackend("others_proxy", true, true);
}

0 comments on commit 8de7fa0

Please sign in to comment.