From 438b20b2986046339deec32ded5924a0d2a6a7fe Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Tue, 21 Nov 2023 13:25:00 -0500 Subject: [PATCH 1/9] changed windows sample rule and query construction Signed-off-by: Joanne Wang --- .../rules/backend/OSQueryBackend.java | 28 ++--- .../rules/test_windows/win_sample_rule.yml | 4 +- .../securityanalytics/TestHelpers.java | 33 ++++++ .../resthandler/DetectorMonitorRestApiIT.java | 109 ++++++++++++++++-- .../rules/backend/QueryBackendTests.java | 105 +++++++++++++++++ 5 files changed, 255 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index 7e0be9ddc..39100aae5 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -132,10 +132,10 @@ public OSQueryBackend(Map fieldMappings, boolean collectErrors, this.reExpression = "%s: /%s/"; this.cidrExpression = "%s: \"%s\""; this.fieldNullExpression = "%s: null"; - this.unboundValueStrExpression = "%s: \"%s\""; - this.unboundValueNumExpression = "%s: %s"; - this.unboundWildcardExpression = "%s: %s"; - this.unboundReExpression = "%s: /%s/"; + this.unboundValueStrExpression = "\"%s\""; + this.unboundValueNumExpression = "\"%s\""; + this.unboundWildcardExpression = "\"%s\""; + this.unboundReExpression = "\"/%s/\""; // TODO this.compareOpExpression = "\"%s\" \"%s\" %s"; this.valExpCount = 0; this.aggQuery = "{\"%s\":{\"terms\":{\"field\":\"%s\"},\"aggs\":{\"%s\":{\"%s\":{\"field\":\"%s\"}}}}}"; @@ -333,27 +333,27 @@ public Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpre public Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError { SigmaString value = (SigmaString) condition.getValue(); - String field = getFinalValueField(); - ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); +// String field = getFinalValueField(); +// ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); boolean containsWildcard = value.containsWildcard(); - return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), field, this.convertValueStr((SigmaString) condition.getValue())); + return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), this.convertValueStr((SigmaString) condition.getValue())); } @Override public Object convertConditionValNum(ConditionValueExpression condition) { - String field = getFinalValueField(); +// String field = getFinalValueField(); - SigmaNumber number = (SigmaNumber) condition.getValue(); - ruleQueryFields.put(field, number.getNumOpt().isLeft()? Collections.singletonMap("type", "integer"): Collections.singletonMap("type", "float")); +// SigmaNumber number = (SigmaNumber) condition.getValue(); +// ruleQueryFields.put(field, number.getNumOpt().isLeft()? Collections.singletonMap("type", "integer"): Collections.singletonMap("type", "float")); - return String.format(Locale.getDefault(), this.unboundValueNumExpression, field, condition.getValue().toString()); + return String.format(Locale.getDefault(), this.unboundValueNumExpression, condition.getValue().toString()); } @Override public Object convertConditionValRe(ConditionValueExpression condition) { - String field = getFinalValueField(); - ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); - return String.format(Locale.getDefault(), this.unboundReExpression, field, convertValueRe((SigmaRegularExpression) condition.getValue())); +// String field = getFinalValueField(); +// ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); + return String.format(Locale.getDefault(), this.unboundReExpression, convertValueRe((SigmaRegularExpression) condition.getValue())); } // TODO: below methods will be supported when Sigma Expand Modifier is supported. diff --git a/src/main/resources/rules/test_windows/win_sample_rule.yml b/src/main/resources/rules/test_windows/win_sample_rule.yml index b55e9c9b7..b4e25a2cd 100644 --- a/src/main/resources/rules/test_windows/win_sample_rule.yml +++ b/src/main/resources/rules/test_windows/win_sample_rule.yml @@ -19,6 +19,8 @@ detection: EventID: 22 Message|contains: 'C:\\Program Files\\nxlog\\nxlog.exe' HostName|startswith: 'EC2AMAZ' - condition: selection + keywords: + - "NT AUTHORITY" + condition: selection and keywords falsepositives: - Unknown diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 9a61ec957..7c1a3938c 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -347,6 +347,39 @@ public static String randomRuleWithAlias() { "level: high"; } + public static String randomRuleWithKeywords() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " EventID: 21\n" + + " keywords:\n" + + " - 1996\n" + + " - 'windows '\n" + + " - AccessDeniedException\n" + + " condition: selection or keywords\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + public static String countAggregationTestRule() { return " title: Test\n" + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index dfea4bac8..2415425a7 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -33,15 +33,7 @@ import java.util.stream.Collectors; import static java.util.Collections.emptyList; -import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule; -import static org.opensearch.securityanalytics.TestHelpers.randomDetector; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; -import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers; -import static org.opensearch.securityanalytics.TestHelpers.randomDoc; -import static org.opensearch.securityanalytics.TestHelpers.randomIndex; -import static org.opensearch.securityanalytics.TestHelpers.randomRule; -import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.TestHelpers.*; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; public class DetectorMonitorRestApiIT extends SecurityAnalyticsRestTestCase { @@ -1648,6 +1640,105 @@ public void testCreateDetector_verifyWorkflowExecutionMultipleBucketLevelDocLeve assertEquals(19, getFindingsBody.get("total_findings")); } + public void testCreateDetectorWithKeywordsRule_verifyFindings_success() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + // Create random doc rule + String randomDocRuleId = createRule(randomRuleWithKeywords()); + List prepackagedRules = getRandomPrePackagedRules(); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), + prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList())); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map updateResponseBody = asMap(createResponse); + String detectorId = updateResponseBody.get("_id").toString(); + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + + // Verify newly created doc level monitor + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); + List monitorIds = ((List) (detectorAsMap).get("monitor_id")); + + assertEquals(1, monitorIds.size()); + + String monitorId = monitorIds.get(0); + String monitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type"); + + assertEquals(MonitorType.DOC_LEVEL_MONITOR.getValue(), monitorType); + + // Verify rules + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + + assertEquals(6, response.getHits().getTotalHits().value); + + // Verify findings + indexDoc(index, "1", randomDoc(2, 5, "Test")); + indexDoc(index, "2", randomDoc(3, 5, "Test")); + + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + // Verify 5 prepackaged rules and 1 custom rule + assertEquals(6, noOfSigmaRuleMatches); + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + assertNotNull(getFindingsBody); + // When doc level monitor is being applied one finding is generated per document + assertEquals(2, getFindingsBody.get("total_findings")); + + Set docRuleIds = new HashSet<>(prepackagedRules); + docRuleIds.add(randomDocRuleId); + + List> findings = (List) getFindingsBody.get("findings"); + List foundDocIds = new ArrayList<>(); + for (Map finding : findings) { + Set aggRulesFinding = ((List>) finding.get("queries")).stream().map(it -> it.get("id").toString()).collect( + Collectors.toSet()); + + assertTrue(docRuleIds.containsAll(aggRulesFinding)); + + List findingDocs = (List) finding.get("related_doc_ids"); + Assert.assertEquals(1, findingDocs.size()); + foundDocIds.addAll(findingDocs); + } + assertTrue(Arrays.asList("1", "2").containsAll(foundDocIds)); + } private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { List> buckets = ((List>) (((Map) ((Map) ((Map) ((List) ((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); diff --git a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java index 5dc4c7a9b..99b611c03 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java @@ -7,6 +7,9 @@ import java.io.IOException; import java.util.List; import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.junit.Assert; import org.opensearch.securityanalytics.rules.exceptions.SigmaError; import org.opensearch.securityanalytics.rules.exceptions.SigmaTypeError; @@ -15,6 +18,7 @@ import org.opensearch.test.OpenSearchTestCase; public class QueryBackendTests extends OpenSearchTestCase { + private static final Logger log = LogManager.getLogger(QueryBackendTests.class); private static Map testFieldMapping = Map.of( "EventID", "event_uid", @@ -882,6 +886,107 @@ public void testConvertProxyRule() throws IOException, SigmaError { Assert.assertEquals(true, true); } + public void testKeywordsFieldAsString() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1: \n" + + " - value1\n" + + " - value2\n" + + " - value3\n" + + " keywords:\n" + + " - valueA\n" + + " - valueB\n" + + " condition: sel or keywords", false)); + Assert.assertEquals("((mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: \"value3\")) OR ((\"valueA\") OR (\"valueB\"))", queries.get(0).toString()); + } + + public void testKeywordsFieldAsNumber() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1: \n" + + " - value1\n" + + " - value2\n" + + " - value3\n" + + " keywords:\n" + + " - 1\n" + + " - 2\n" + + " condition: sel and keywords", false)); + Assert.assertEquals("((mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: \"value3\")) AND ((\"1\") OR (\"2\"))", queries.get(0).toString()); + } + +// public void testKeywordsFieldAsRegexAndWildcard() throws IOException, SigmaError { +// OSQueryBackend queryBackend = testBackend(); +// List queries = queryBackend.convertRule(SigmaRule.fromYaml( +// " title: Test\n" + +// " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + +// " status: test\n" + +// " level: critical\n" + +// " description: Detects QuarksPwDump clearing access history in hive\n" + +// " author: Florian Roth\n" + +// " date: 2017/05/15\n" + +// " logsource:\n" + +// " category: test_category\n" + +// " product: test_product\n" + +// " detection:\n" + +// " sel:\n" + +// " fieldA1: \n" + +// " - value1\n" + +// " - value2\n" + +// " - value3\n" + +// " keywords:\n" + +// " - ^[0-9]+$\n" + +// " - test*\n" + +// " condition: sel or keywords", false)); +// log.info(queries.get(0).toString()); +// Assert.assertEquals("((mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: \"value3\")) OR ((/^[0-9]+$/) OR (\"test*\"))", queries.get(0).toString()); +// } + + public void test() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " selection:\n" + + " eventID: 21\n" + + " keywords:\n" + + " - \"22\"\n" + + " condition: selection or keywords", false)); + log.info(queries.get(0).toString()); + } + private OSQueryBackend testBackend() throws IOException { return new OSQueryBackend(testFieldMapping, false, true); } From c84cb71d1eb9c825d6f5dce622ec5e3c1952c04d Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Tue, 21 Nov 2023 13:37:07 -0500 Subject: [PATCH 2/9] remove wildcard Signed-off-by: Joanne Wang --- .../resthandler/DetectorMonitorRestApiIT.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 2415425a7..1a35eed44 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -33,7 +33,16 @@ import java.util.stream.Collectors; import static java.util.Collections.emptyList; -import static org.opensearch.securityanalytics.TestHelpers.*; +import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule; +import static org.opensearch.securityanalytics.TestHelpers.randomDetector; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers; +import static org.opensearch.securityanalytics.TestHelpers.randomDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.randomRule; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithKeywords; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; public class DetectorMonitorRestApiIT extends SecurityAnalyticsRestTestCase { From 5ce7573c75a148c58543d7646aeefc5b00c188df Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Tue, 21 Nov 2023 16:36:04 -0500 Subject: [PATCH 3/9] changed wildcardtest Signed-off-by: Joanne Wang --- .../rules/backend/QueryBackendTests.java | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java index 99b611c03..e42fbd0e3 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java @@ -938,32 +938,31 @@ public void testKeywordsFieldAsNumber() throws IOException, SigmaError { Assert.assertEquals("((mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: \"value3\")) AND ((\"1\") OR (\"2\"))", queries.get(0).toString()); } -// public void testKeywordsFieldAsRegexAndWildcard() throws IOException, SigmaError { -// OSQueryBackend queryBackend = testBackend(); -// List queries = queryBackend.convertRule(SigmaRule.fromYaml( -// " title: Test\n" + -// " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + -// " status: test\n" + -// " level: critical\n" + -// " description: Detects QuarksPwDump clearing access history in hive\n" + -// " author: Florian Roth\n" + -// " date: 2017/05/15\n" + -// " logsource:\n" + -// " category: test_category\n" + -// " product: test_product\n" + -// " detection:\n" + -// " sel:\n" + -// " fieldA1: \n" + -// " - value1\n" + -// " - value2\n" + -// " - value3\n" + -// " keywords:\n" + -// " - ^[0-9]+$\n" + -// " - test*\n" + -// " condition: sel or keywords", false)); -// log.info(queries.get(0).toString()); -// Assert.assertEquals("((mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: \"value3\")) OR ((/^[0-9]+$/) OR (\"test*\"))", queries.get(0).toString()); -// } + public void testKeywordsFieldAsWildcard() throws IOException, SigmaError { + OSQueryBackend queryBackend = testBackend(); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + " title: Test\n" + + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + + " status: test\n" + + " level: critical\n" + + " description: Detects QuarksPwDump clearing access history in hive\n" + + " author: Florian Roth\n" + + " date: 2017/05/15\n" + + " logsource:\n" + + " category: test_category\n" + + " product: test_product\n" + + " detection:\n" + + " sel:\n" + + " fieldA1: \n" + + " - value1\n" + + " - value2\n" + + " - value3\n" + + " keywords:\n" + + " - test*\n" + + " condition: sel or keywords", false)); + log.info(queries.get(0).toString()); + Assert.assertEquals("((mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: \"value3\")) OR (\"test*\")", queries.get(0).toString()); + } public void test() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); From d37be0be39bcb0295e5698e59e0425604f6336f8 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Tue, 21 Nov 2023 17:19:09 -0500 Subject: [PATCH 4/9] fixed wildcards Signed-off-by: Joanne Wang --- .../securityanalytics/rules/backend/OSQueryBackend.java | 4 ++-- .../java/org/opensearch/securityanalytics/TestHelpers.java | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index 39100aae5..bc6e4fbbf 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -134,8 +134,8 @@ public OSQueryBackend(Map fieldMappings, boolean collectErrors, this.fieldNullExpression = "%s: null"; this.unboundValueStrExpression = "\"%s\""; this.unboundValueNumExpression = "\"%s\""; - this.unboundWildcardExpression = "\"%s\""; - this.unboundReExpression = "\"/%s/\""; // TODO + this.unboundWildcardExpression = "%s"; + this.unboundReExpression = "\"/%s/\""; this.compareOpExpression = "\"%s\" \"%s\" %s"; this.valExpCount = 0; this.aggQuery = "{\"%s\":{\"terms\":{\"field\":\"%s\"},\"aggs\":{\"%s\":{\"%s\":{\"field\":\"%s\"}}}}}"; diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 7c1a3938c..2614f3d8e 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -372,8 +372,7 @@ public static String randomRuleWithKeywords() { " EventID: 21\n" + " keywords:\n" + " - 1996\n" + - " - 'windows '\n" + - " - AccessDeniedException\n" + + " - EC2AMAZ*\n" + " condition: selection or keywords\n" + "falsepositives:\n" + " - Legitimate usage of remote file encryption\n" + From 380b8371f7b651a01ea110b2c3abd922a58acf74 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Tue, 21 Nov 2023 17:37:46 -0500 Subject: [PATCH 5/9] fixed wildcard query test Signed-off-by: Joanne Wang --- .../securityanalytics/rules/backend/QueryBackendTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java index e42fbd0e3..d287db3ed 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java @@ -961,7 +961,7 @@ public void testKeywordsFieldAsWildcard() throws IOException, SigmaError { " - test*\n" + " condition: sel or keywords", false)); log.info(queries.get(0).toString()); - Assert.assertEquals("((mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: \"value3\")) OR (\"test*\")", queries.get(0).toString()); + Assert.assertEquals("((mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: \"value3\")) OR (test*)", queries.get(0).toString()); } public void test() throws IOException, SigmaError { From 97560bc0bfb818812d73aa40cac683f7d6bb779a Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Wed, 22 Nov 2023 12:00:54 -0500 Subject: [PATCH 6/9] fixed correlation engine tests Signed-off-by: Joanne Wang --- src/main/resources/rules/test_windows/win_sample_rule.yml | 2 +- src/test/java/org/opensearch/securityanalytics/TestHelpers.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/rules/test_windows/win_sample_rule.yml b/src/main/resources/rules/test_windows/win_sample_rule.yml index b4e25a2cd..8720614a0 100644 --- a/src/main/resources/rules/test_windows/win_sample_rule.yml +++ b/src/main/resources/rules/test_windows/win_sample_rule.yml @@ -21,6 +21,6 @@ detection: HostName|startswith: 'EC2AMAZ' keywords: - "NT AUTHORITY" - condition: selection and keywords + condition: selection or keywords falsepositives: - Unknown diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 2614f3d8e..092643fd3 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -1542,7 +1542,7 @@ public static String randomAppLogDoc() { return "{\n" + " \"endpoint\": \"/customer_records.txt\",\n" + " \"http_method\": \"POST\",\n" + - " \"keywords\": \"PermissionDenied\"\n" + + " \"keywords\": \"INVALID\"\n" + "}"; } From 1d681cb6460c9c4880005b917c0bd90f72140ecf Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Wed, 22 Nov 2023 15:01:55 -0500 Subject: [PATCH 7/9] fixed query backend tests Signed-off-by: Joanne Wang --- .../rules/backend/OSQueryBackend.java | 2 +- .../rules/backend/QueryBackendTests.java | 89 ++----------------- 2 files changed, 6 insertions(+), 85 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index bc6e4fbbf..6a7dadc38 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -135,7 +135,7 @@ public OSQueryBackend(Map fieldMappings, boolean collectErrors, this.unboundValueStrExpression = "\"%s\""; this.unboundValueNumExpression = "\"%s\""; this.unboundWildcardExpression = "%s"; - this.unboundReExpression = "\"/%s/\""; + this.unboundReExpression = "/%s/"; this.compareOpExpression = "\"%s\" \"%s\" %s"; this.valExpCount = 0; this.aggQuery = "{\"%s\":{\"terms\":{\"field\":\"%s\"},\"aggs\":{\"%s\":{\"%s\":{\"field\":\"%s\"}}}}}"; diff --git a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java index d287db3ed..81a149311 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java @@ -7,9 +7,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.junit.Assert; import org.opensearch.securityanalytics.rules.exceptions.SigmaError; import org.opensearch.securityanalytics.rules.exceptions.SigmaTypeError; @@ -18,7 +15,6 @@ import org.opensearch.test.OpenSearchTestCase; public class QueryBackendTests extends OpenSearchTestCase { - private static final Logger log = LogManager.getLogger(QueryBackendTests.class); private static Map testFieldMapping = Map.of( "EventID", "event_uid", @@ -332,7 +328,7 @@ public void testConvertValueRegexUnbound() throws IOException, SigmaError { " sel:\n" + " \"|re\": pat.*tern\"foo\"bar\n" + " condition: sel", false)); - Assert.assertEquals("_0: /pat.*tern\\\"foo\\\"bar/", queries.get(0).toString()); + Assert.assertEquals("/pat.*tern\\\"foo\\\"bar/", queries.get(0).toString()); } public void testConvertValueCidrWildcardNone() throws IOException, SigmaError { @@ -488,7 +484,7 @@ public void testConvertOrInMixedKeywordField() throws IOException, SigmaError { " fieldB: value2\n" + " sel3: value3\n" + " condition: sel1 or sel2 or sel3", false)); - Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) OR (_0: \"value3\")", queries.get(0).toString()); + Assert.assertEquals("((fieldA: \"value1\") OR (mappedB: \"value2\")) OR (\"value3\")", queries.get(0).toString()); } public void testConvertOrInMixedFields() throws IOException, SigmaError { @@ -601,9 +597,9 @@ public void testConvertUnboundValues() throws IOException, SigmaError { " sel:\n" + " - value1\n" + " - value2\n" + - " - 4\n" + + " - 123\n" + " condition: sel", false)); - Assert.assertEquals("(_0: \"value1\") OR (_1: \"value2\") OR (_2: 4)", queries.get(0).toString()); + Assert.assertEquals("(\"value1\") OR (\"value2\") OR (\"123\")", queries.get(0).toString()); } public void testConvertInvalidUnboundBool() throws IOException { @@ -886,59 +882,7 @@ public void testConvertProxyRule() throws IOException, SigmaError { Assert.assertEquals(true, true); } - public void testKeywordsFieldAsString() throws IOException, SigmaError { - OSQueryBackend queryBackend = testBackend(); - List queries = queryBackend.convertRule(SigmaRule.fromYaml( - " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1: \n" + - " - value1\n" + - " - value2\n" + - " - value3\n" + - " keywords:\n" + - " - valueA\n" + - " - valueB\n" + - " condition: sel or keywords", false)); - Assert.assertEquals("((mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: \"value3\")) OR ((\"valueA\") OR (\"valueB\"))", queries.get(0).toString()); - } - - public void testKeywordsFieldAsNumber() throws IOException, SigmaError { - OSQueryBackend queryBackend = testBackend(); - List queries = queryBackend.convertRule(SigmaRule.fromYaml( - " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " sel:\n" + - " fieldA1: \n" + - " - value1\n" + - " - value2\n" + - " - value3\n" + - " keywords:\n" + - " - 1\n" + - " - 2\n" + - " condition: sel and keywords", false)); - Assert.assertEquals("((mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: \"value3\")) AND ((\"1\") OR (\"2\"))", queries.get(0).toString()); - } - - public void testKeywordsFieldAsWildcard() throws IOException, SigmaError { + public void testConvertUnboundValuesAsWildcard() throws IOException, SigmaError { OSQueryBackend queryBackend = testBackend(); List queries = queryBackend.convertRule(SigmaRule.fromYaml( " title: Test\n" + @@ -960,32 +904,9 @@ public void testKeywordsFieldAsWildcard() throws IOException, SigmaError { " keywords:\n" + " - test*\n" + " condition: sel or keywords", false)); - log.info(queries.get(0).toString()); Assert.assertEquals("((mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: \"value3\")) OR (test*)", queries.get(0).toString()); } - public void test() throws IOException, SigmaError { - OSQueryBackend queryBackend = testBackend(); - List queries = queryBackend.convertRule(SigmaRule.fromYaml( - " title: Test\n" + - " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + - " status: test\n" + - " level: critical\n" + - " description: Detects QuarksPwDump clearing access history in hive\n" + - " author: Florian Roth\n" + - " date: 2017/05/15\n" + - " logsource:\n" + - " category: test_category\n" + - " product: test_product\n" + - " detection:\n" + - " selection:\n" + - " eventID: 21\n" + - " keywords:\n" + - " - \"22\"\n" + - " condition: selection or keywords", false)); - log.info(queries.get(0).toString()); - } - private OSQueryBackend testBackend() throws IOException { return new OSQueryBackend(testFieldMapping, false, true); } From 29060aac8a12b4c5fd2b8a773218fdf938d70f24 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Wed, 22 Nov 2023 15:34:02 -0500 Subject: [PATCH 8/9] clean up Signed-off-by: Joanne Wang --- .../rules/backend/OSQueryBackend.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index 6a7dadc38..05f0bd9f3 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -332,27 +332,17 @@ public Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpre @Override public Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError { SigmaString value = (SigmaString) condition.getValue(); - -// String field = getFinalValueField(); -// ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); boolean containsWildcard = value.containsWildcard(); return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), this.convertValueStr((SigmaString) condition.getValue())); } @Override public Object convertConditionValNum(ConditionValueExpression condition) { -// String field = getFinalValueField(); - -// SigmaNumber number = (SigmaNumber) condition.getValue(); -// ruleQueryFields.put(field, number.getNumOpt().isLeft()? Collections.singletonMap("type", "integer"): Collections.singletonMap("type", "float")); - return String.format(Locale.getDefault(), this.unboundValueNumExpression, condition.getValue().toString()); } @Override public Object convertConditionValRe(ConditionValueExpression condition) { -// String field = getFinalValueField(); -// ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); return String.format(Locale.getDefault(), this.unboundReExpression, convertValueRe((SigmaRegularExpression) condition.getValue())); } From 30027ea311c2bc45d4fcaa164edc2d08bc621f7c Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Mon, 27 Nov 2023 16:22:29 -0500 Subject: [PATCH 9/9] added two integration tests Signed-off-by: Joanne Wang --- .../securityanalytics/TestHelpers.java | 126 +++++++++++++++ .../resthandler/DetectorMonitorRestApiIT.java | 153 ++++++++++++++++++ 2 files changed, 279 insertions(+) diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 092643fd3..774143aba 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -379,6 +379,68 @@ public static String randomRuleWithKeywords() { "level: high"; } + public static String randomRuleWithStringKeywords() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " EventID: 21\n" + + " keywords:\n" + + " - \"INFO\"\n" + + " condition: selection or keywords\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + + public static String randomRuleWithDateKeywords() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " EventID: 21\n" + + " keywords:\n" + + " - \"2020-02-04T14:59:39.343541+00:00\"\n" + + " condition: selection or keywords\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + public static String countAggregationTestRule() { return " title: Test\n" + " id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" + @@ -1399,6 +1461,48 @@ public static String windowsIndexMapping() { " }"; } + public static String windowsIndexMappingOnlyNumericAndDate() { + return "\"properties\": {\n" + + " \"@timestamp\": {\"type\":\"date\"},\n" + + " \"EventTime\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"ExecutionProcessID\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"ExecutionThreadID\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"EventID\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"TaskValue\": {\n" + + " \"type\": \"integer\"\n" + + " }\n" + + " }"; + } + + public static String windowsIndexMappingOnlyNumericAndText() { + return "\"properties\": {\n" + + " \"TaskName\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"ExecutionProcessID\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"ExecutionThreadID\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"EventID\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"TaskValue\": {\n" + + " \"type\": \"integer\"\n" + + " }\n" + + " }"; + } + + public static String randomDoc(int severity, int version, String opCode) { String doc = "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + @@ -1438,6 +1542,28 @@ public static String randomDoc(int severity, int version, String opCode) { } + public static String randomDocOnlyNumericAndDate(int severity, int version, String opCode) { + String doc = "{\n" + + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"ExecutionProcessID\":2001,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"EventID\": 1234,\n" + + "\"TaskValue\":22\n" + + "}"; + return String.format(Locale.ROOT, doc, severity, version, opCode); + } + + public static String randomDocOnlyNumericAndText(int severity, int version, String opCode) { + String doc = "{\n" + + "\"TaskName\":\"SYSTEM\",\n" + + "\"ExecutionProcessID\":2001,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"EventID\": 1234,\n" + + "\"TaskValue\":22\n" + + "}"; + return String.format(Locale.ROOT, doc, severity, version, opCode); + } + //Add IPs in HostName field. public static String randomDocWithIpIoc(int severity, int version, String ioc) { String doc = "{\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 1a35eed44..95a9c3707 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -43,6 +43,13 @@ import static org.opensearch.securityanalytics.TestHelpers.randomRule; import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithKeywords; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithStringKeywords; +import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndDate; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndDate; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndText; +import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithDateKeywords; +import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndText; + import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; public class DetectorMonitorRestApiIT extends SecurityAnalyticsRestTestCase { @@ -1749,6 +1756,152 @@ public void testCreateDetectorWithKeywordsRule_verifyFindings_success() throws I assertTrue(Arrays.asList("1", "2").containsAll(foundDocIds)); } + public void testCreateDetectorWithKeywordsRule_ensureNoFindingsWithoutTextMapping_success() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMappingOnlyNumericAndDate()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + // Create random doc rule + String randomDocRuleId = createRule(randomRuleWithStringKeywords()); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), + emptyList()); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map updateResponseBody = asMap(createResponse); + String detectorId = updateResponseBody.get("_id").toString(); + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + + // Verify newly created doc level monitor + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); + List monitorIds = ((List) (detectorAsMap).get("monitor_id")); + + assertEquals(1, monitorIds.size()); + + String monitorId = monitorIds.get(0); + String monitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type"); + + assertEquals(MonitorType.DOC_LEVEL_MONITOR.getValue(), monitorType); + + // Verify rules created + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + + assertEquals(1, response.getHits().getTotalHits().value); + + // Insert test document + indexDoc(index, "1", randomDocOnlyNumericAndDate(2, 5, "Test")); + indexDoc(index, "2", randomDocOnlyNumericAndDate(3, 5, "Test")); + + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + // Verify no rules match test document + assertEquals(0, noOfSigmaRuleMatches); + } + + public void testCreateDetectorWithKeywordsRule_ensureNoFindingsWithoutDateMapping_success() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMappingOnlyNumericAndText()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + // Create random doc rule + String randomDocRuleId = createRule(randomRuleWithDateKeywords()); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), + emptyList()); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map updateResponseBody = asMap(createResponse); + String detectorId = updateResponseBody.get("_id").toString(); + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + + // Verify newly created doc level monitor + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); + List monitorIds = ((List) (detectorAsMap).get("monitor_id")); + + assertEquals(1, monitorIds.size()); + + String monitorId = monitorIds.get(0); + String monitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type"); + + assertEquals(MonitorType.DOC_LEVEL_MONITOR.getValue(), monitorType); + + // Verify rules created + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + + assertEquals(1, response.getHits().getTotalHits().value); + + // Insert test document + indexDoc(index, "1", randomDocOnlyNumericAndText(2, 5, "Test")); + indexDoc(index, "2", randomDocOnlyNumericAndText(3, 5, "Test")); + + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + // Verify no rules match test document + assertEquals(0, noOfSigmaRuleMatches); + } + private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { List> buckets = ((List>) (((Map) ((Map) ((Map) ((List) ((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); Integer docCount = buckets.stream().mapToInt(it -> (Integer) it.get("doc_count")).sum();