From a59a014467bf1dde7cb40f9d8012545a315cddff Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Wed, 29 Nov 2023 15:59:47 -0800 Subject: [PATCH] fix null query filter conversion from sigma to query string query (#722) * fix null query filter conversion from sigma to query string query Signed-off-by: Surya Sashank Nistala * fix rule to query conversion tests for null filter Signed-off-by: Surya Sashank Nistala * enhance test to verify non null doc doesnt match null query Signed-off-by: Surya Sashank Nistala --------- Signed-off-by: Surya Sashank Nistala --- .../rules/backend/OSQueryBackend.java | 2 +- .../DetectorThreatIntelIT.java | 99 ++++++++++++++++++- .../securityanalytics/TestHelpers.java | 70 +++++++++++++ .../rules/backend/QueryBackendTests.java | 4 +- 4 files changed, 168 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index 05f0bd9f3..2a08cad89 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -131,7 +131,7 @@ public OSQueryBackend(Map fieldMappings, boolean collectErrors, this.reEscapeChar = "\\"; this.reExpression = "%s: /%s/"; this.cidrExpression = "%s: \"%s\""; - this.fieldNullExpression = "%s: null"; + this.fieldNullExpression = "%s: (NOT [* TO *])"; this.unboundValueStrExpression = "\"%s\""; this.unboundValueNumExpression = "\"%s\""; this.unboundWildcardExpression = "%s"; diff --git a/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java b/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java index 9d83b3ed3..144b5d2c0 100644 --- a/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java +++ b/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java @@ -26,11 +26,14 @@ import static java.util.Collections.emptyList; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndThreatIntel; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndThreatIntelAndTriggers; import static org.opensearch.securityanalytics.TestHelpers.randomDoc; import static org.opensearch.securityanalytics.TestHelpers.randomDocWithIpIoc; +import static org.opensearch.securityanalytics.TestHelpers.randomDocWithNullField; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.randomNullRule; import static org.opensearch.securityanalytics.TestHelpers.randomRule; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; @@ -150,6 +153,94 @@ public void testCreateDetectorWithThreatIntelEnabled_updateDetectorWithThreatInt assertEquals(1, noOfSigmaRuleMatches); } + public void testCreateDetectorForSigmaRuleWithNullCondition() throws IOException { + + updateClusterSetting(ENABLE_WORKFLOW_USAGE.getKey(), "true"); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + String testOpCode = "Test"; + + String randomDocRuleId = createRule(randomNullRule()); + List detectorRules = List.of(new DetectorRule(randomDocRuleId)); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, + emptyList()); + DetectorTrigger trigger = new DetectorTrigger("all", "all", "high", List.of(randomDetectorType()), emptyList(), emptyList(), List.of(), emptyList(), List.of(DetectorTrigger.RULES_DETECTION_TYPE, DetectorTrigger.THREAT_INTEL_DETECTION_TYPE)); + Detector detector = randomDetectorWithInputs(List.of(input)); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorMap = (HashMap) (hit.getSourceAsMap().get("detector")); + List inputArr = (List) detectorMap.get("inputs"); + + + List monitorIds = ((List) (detectorMap).get("monitor_id")); + assertEquals(1, monitorIds.size()); + + Response getMonitorResponse = getAlertingMonitor(client(), monitorIds.get(0)); + Map alertingMonitor = asMap(getMonitorResponse); + assertNotNull(alertingMonitor); + String workflowId = ((List) detectorMap.get("workflow_ids")).get(0); + + indexDoc(index, "1", randomDocWithNullField()); + indexDoc(index, "2", randomDoc()); + + Response executeResponse = executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + List> monitorRunResults = (List>) entityAsMap(executeResponse).get("monitor_run_results"); + assertEquals(1, monitorRunResults.size()); + + Map docLevelQueryResults = ((List>) ((Map) monitorRunResults.get(0).get("input_results")).get("results")).get(0); + int noOfSigmaRuleMatches = docLevelQueryResults.size(); + assertEquals(1, noOfSigmaRuleMatches); + String queryId = docLevelQueryResults.keySet().stream().findAny().get(); + ArrayList docs = (ArrayList) docLevelQueryResults.get(queryId); + assertEquals(docs.size(), 1); + + indexDoc(index, "3", randomDoc()); + Response executeResponse1 = executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + List> monitorRunResults1 = (List>) entityAsMap(executeResponse1).get("monitor_run_results"); + assertEquals(1, monitorRunResults1.size()); + + Map docLevelQueryResults1 = ((List>) ((Map) monitorRunResults1.get(0).get("input_results")).get("results")).get(0); + int noOfSigmaRuleMatches1 = docLevelQueryResults1.size(); + assertEquals(0, noOfSigmaRuleMatches1); + + } + public void testCreateDetectorWithThreatIntelDisabled_updateDetectorWithThreatIntelEnabled() throws IOException { updateClusterSetting(ENABLE_WORKFLOW_USAGE.getKey(), "true"); @@ -594,8 +685,8 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithThreatIntelDete verifyWorkflow(detectorMap, monitorIds, 1); int i = 1; - while (i<4) { - indexDoc(index, i + "", randomDocWithIpIoc(5, 3, i+"")); + while (i < 4) { + indexDoc(index, i + "", randomDocWithIpIoc(5, 3, i + "")); i++; } String workflowId = ((List) detectorMap.get("workflow_ids")).get(0); @@ -686,8 +777,8 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT verifyWorkflow(detectorMap, monitorIds, 1); int i = 1; - while (i<4) { - indexDoc(index, i + "", randomDocWithIpIoc(5, 3, i+"")); + while (i < 4) { + indexDoc(index, i + "", randomDocWithIpIoc(5, 3, i + "")); i++; } String workflowId = ((List) detectorMap.get("workflow_ids")).get(0); diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 774143aba..ec48cdcf5 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -259,6 +259,38 @@ public static String randomRule() { "level: high"; } + + + public static String randomNullRule() { + return "title: null field\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firew all to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " EventID: 22\n" + + " RecordNumber: null\n" + + " condition: selection\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + public static String randomRuleForMappingView(String field) { return "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + @@ -1604,6 +1636,44 @@ public static String randomDocWithIpIoc(int severity, int version, String ioc) } + public static String randomDocWithNullField() { + return "{\n" + + "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + + "\"Keywords\":\"9223372036854775808\",\n" + + "\"SeverityValue\":2,\n" + + "\"Severity\":\"INFO\",\n" + + "\"EventID\":22,\n" + + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + "\"Version\":5,\n" + + "\"TaskValue\":22,\n" + + "\"OpcodeValue\":0,\n" + + "\"RecordNumber\":null,\n" + + "\"ExecutionProcessID\":1996,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + + "\"Domain\":\"NTAUTHORITY\",\n" + + "\"AccountName\":\"SYSTEM\",\n" + + "\"UserID\":\"S-1-5-18\",\n" + + "\"AccountType\":\"User\",\n" + + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + + "\"Opcode\":\"Info\",\n" + + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + + "\"QueryResults\":\"172.31.46.38;\",\n" + + "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + + "\"SourceModuleName\":\"in\",\n" + + "\"SourceModuleType\":\"im_msvistalog\",\n" + + "\"CommandLine\": \"eachtest\",\n" + + "\"Initiated\": \"true\"\n" + + "}"; + } + public static String randomDoc() { return "{\n" + "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java index 81a149311..aff11d913 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java @@ -288,7 +288,7 @@ public void testConvertValueNull() throws IOException, SigmaError { " sel:\n" + " fieldA1: null\n" + " condition: sel", false)); - Assert.assertEquals("mappedA: null", queries.get(0).toString()); + Assert.assertEquals("mappedA: (NOT [* TO *])", queries.get(0).toString()); } public void testConvertValueRegex() throws IOException, SigmaError { @@ -531,7 +531,7 @@ public void testConvertOrInUnallowedValueType() throws IOException, SigmaError { " - value2\n" + " - null\n" + " condition: sel", false)); - Assert.assertEquals("(mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: null)", queries.get(0).toString()); + Assert.assertEquals("(mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: (NOT [* TO *]))", queries.get(0).toString()); } public void testConvertOrInListNumbers() throws IOException, SigmaError {