[FR] Adding Support for missing_field_strategy Field in Alert Suppression

[terrancedejesus]

Issues

• [FR] Add suppression for missing fields option #3191

Summary

This PR adds missing_field_strategy to AlertSuppressionMapping dataclass. This allows it to be a viable field in rules when loaded via schema.

Additionally, the missing_fields_strategy has been min-stacked to 8.8 where it was merged as shown with elastic/kibana#155055.

Specific field values have been limited to suppress and doNotSuppress via definiitions.AlertSuppressionMissing. Validation has been included to ensure the strings are bound to these two options.

Note that we created a rule on an 8.10.3 stack, created a rule with alert suppression and both options toggled, then exported to understand what was required.

Example Rule JSON Data

{
    "actions": [
    ],
    "alert_suppression": {
        "duration": {
            "unit": "m",
            "value": 5
        },
        "group_by": [
            "process.name"
        ],
        "missing_fields_strategy": "suppress"
    },
    "author": [
    ],
    "created_at": "2023-10-17T17:44:22.796Z",
    "created_by": "4220331459",
    "description": "missing suppression test",
    "enabled": false,
    "exceptions_list": [
    ],
    "false_positives": [
    ],
    "filters": [
    ],
    "from": "now-360s",
    "id": "ce970000-6d14-11ee-a7f9-598400f1831b",
    "immutable": false,
    "index": [
        "apm-*-transaction*",
        "auditbeat-*",
        "endgame-*",
        "filebeat-*",
        "logs-*",
        "packetbeat-*",
        "traces-apm*",
        "winlogbeat-*",
        "-*elastic-cloud-logs-*"
    ],
    "interval": "5m",
    "investigation_fields": [
    ],
    "language": "kuery",
    "license": "",
    "max_signals": 100,
    "meta": {
        "from": "1m",
        "kibana_siem_app_url": "https://dejesus-threat-detection-dev.kb.us-central1.gcp.cloud.es.io:9243/app/security"
    },
    "name": "missing suppression test",
    "output_index": "",
    "query": "process.name: \"outlook.exe"",
    "references": [
    ],
    "related_integrations": [
    ],
    "required_fields": [
    ],
    "revision": 0,
    "risk_score": 21,
    "risk_score_mapping": [
    ],
    "rule_id": "c1a665ad-44a5-4dd1-ae8e-49c9f8b3f240",
    "setup": "",
    "severity": "low",
    "severity_mapping": [
    ],
    "tags": [
    ],
    "threat": [
    ],
    "to": "now",
    "type": "query",
    "updated_at": "2023-10-17T17:44:22.796Z",
    "updated_by": "4220331459",
    "version": 1
}

Example Good TOML Rule for Testing

[metadata]
creation_date = "2022/05/16"
integration = ["endpoint"]
maturity = "production"
min_stack_comments = "New fields added: required_fields, related_integrations, setup"
min_stack_version = "8.3.0"
updated_date = "2023/02/22"

[rule]
author = ["Elastic"]
description = """
Identifies an outbound network connection attempt followed by a session id change as the root user by the same process
entity. This particular instantiation of a network connection is abnormal and should be investigated as it may indicate
a potential reverse shell activity via a privileged process.
"""
false_positives = [
    """
    False-Positives (FP) can appear if another remote terminal service is being used to connect to it's listener but
    typically SSH is used in these scenarios.
    """,
]
from = "now-9m"
index = ["logs-endpoint.events.*"]
language = "kuery"
license = "Elastic License v2"
name = "Suspicious Network Connection Attempt by Root"
note = """## Triage and analysis
### Investigating Connection Attempt by Non-SSH Root Session
Detection alerts from this rule indicate a strange or abnormal outbound connection attempt by a privileged process.  Here are some possible avenues of investigation:
- Examine unusual and active sessions using commands such as 'last -a', 'netstat -a', and 'w -a'.
- Analyze processes and command line arguments to detect anomalous process execution that may be acting as a listener.
- Analyze anomalies in the use of files that do not normally initiate connections.
- Examine processes utilizing the network that do not normally have network communication.
"""
references = [
    "https://www.sandflysecurity.com/blog/linux-file-masquerading-and-malicious-pids-sandfly-1-2-6-update/",
    "https://twitter.com/GossiTheDog/status/1522964028284411907",
    "https://exatrack.com/public/Tricephalic_Hellkeeper.pdf",
]
risk_score = 47
rule_id = "eb6a3790-d52d-11ec-8ce9-f661ea17fbce"
severity = "medium"
tags = ["Elastic", "Host", "Linux", "Threat Detection", "Command and Control"]
type = "query"

query = '''process : "outlooke.exe"'''


[[rule.threat]]
framework = "MITRE ATT&CK"
[[rule.threat.technique]]
id = "T1095"
name = "Non-Application Layer Protocol"
reference = "https://attack.mitre.org/techniques/T1095/"


[rule.threat.tactic]
id = "TA0011"
name = "Command and Control"
reference = "https://attack.mitre.org/tactics/TA0011/"
[[rule.threat]]
framework = "MITRE ATT&CK"
[[rule.threat.technique]]
id = "T1548"
name = "Abuse Elevation Control Mechanism"
reference = "https://attack.mitre.org/techniques/T1548/"
[[rule.threat.technique.subtechnique]]
id = "T1548.003"
name = "Sudo and Sudo Caching"
reference = "https://attack.mitre.org/techniques/T1548/003/"



[rule.threat.tactic]
id = "TA0004"
name = "Privilege Escalation"
reference = "https://attack.mitre.org/tactics/TA0004/"

[rule.alert_suppression]
group_by = ["process.name"]
missing_fields_strategy = "suppress"

[rule.alert_suppression.duration]
unit = "m"
value = 10

Example Bad TOML Rule for Testing

[metadata]
creation_date = "2022/05/16"
integration = ["endpoint"]
maturity = "production"
min_stack_comments = "New fields added: required_fields, related_integrations, setup"
min_stack_version = "8.8.0"
updated_date = "2023/02/22"

[rule]
author = ["Elastic"]
description = """
Identifies an outbound network connection attempt followed by a session id change as the root user by the same process
entity. This particular instantiation of a network connection is abnormal and should be investigated as it may indicate
a potential reverse shell activity via a privileged process.
"""
false_positives = [
    """
    False-Positives (FP) can appear if another remote terminal service is being used to connect to it's listener but
    typically SSH is used in these scenarios.
    """,
]
from = "now-9m"
index = ["logs-endpoint.events.*"]
language = "eql"
license = "Elastic License v2"
name = "Suspicious Network Connection Attempt by Root"
note = """## Triage and analysis
### Investigating Connection Attempt by Non-SSH Root Session
Detection alerts from this rule indicate a strange or abnormal outbound connection attempt by a privileged process.  Here are some possible avenues of investigation:
- Examine unusual and active sessions using commands such as 'last -a', 'netstat -a', and 'w -a'.
- Analyze processes and command line arguments to detect anomalous process execution that may be acting as a listener.
- Analyze anomalies in the use of files that do not normally initiate connections.
- Examine processes utilizing the network that do not normally have network communication.
"""
references = [
    "https://www.sandflysecurity.com/blog/linux-file-masquerading-and-malicious-pids-sandfly-1-2-6-update/",
    "https://twitter.com/GossiTheDog/status/1522964028284411907",
    "https://exatrack.com/public/Tricephalic_Hellkeeper.pdf",
]
risk_score = 47
rule_id = "eb6a3790-d52d-11ec-8ce9-f661ea17fbce"
severity = "medium"
tags = ["Elastic", "Host", "Linux", "Threat Detection", "Command and Control"]
type = "eql"

query = '''process where process.name == "outlooke.exe"'''


[[rule.threat]]
framework = "MITRE ATT&CK"
[[rule.threat.technique]]
id = "T1095"
name = "Non-Application Layer Protocol"
reference = "https://attack.mitre.org/techniques/T1095/"


[rule.threat.tactic]
id = "TA0011"
name = "Command and Control"
reference = "https://attack.mitre.org/tactics/TA0011/"
[[rule.threat]]
framework = "MITRE ATT&CK"
[[rule.threat.technique]]
id = "T1548"
name = "Abuse Elevation Control Mechanism"
reference = "https://attack.mitre.org/techniques/T1548/"
[[rule.threat.technique.subtechnique]]
id = "T1548.003"
name = "Sudo and Sudo Caching"
reference = "https://attack.mitre.org/techniques/T1548/003/"



[rule.threat.tactic]
id = "TA0004"
name = "Privilege Escalation"
reference = "https://attack.mitre.org/tactics/TA0004/"

[rule.alert_suppression]
group_by = ["process.name"]
missing_fields_strategy = "suppress"

[rule.alert_suppression.duration]
unit = "m"
value = 10

Testing

1. Checkout branch
2. Add test_rule.toml in /rules directory. Contents are shared above. Use both good and bad rule.
3. Run python -m detection_rules view-rule PATH_TO_RULE_FILE

With the good rule, testing output should be a JSON object, which means the rule has been loaded through the dataclasses defined, including the alert suppression dataclass.

{'_schema': ["Suspicious Network Connection Attempt by Root rule's 'language' must be 'kuery' when using alert suppression."]}

[terrancedejesus]

As exampled by in @brokensound77 's draft PR: https://github.com/elastic/detection-rules/pull/3207/files

We can use validates_schema decorator to add additional validation on QueryRuleData objects. Therefore if alert suppression exists and the type is not query raise a validation issue. Since we are using validates_schema the error is stored on the _schema key which is contextual for the end user, should they arrive at this issue.

[brokensound77]

one suggestion then LGTM

[Mikaayenson]

Much better! LGTM