Expand timestamp override tests (#1907)

* Expand timestamp_override tests
* removed timestamp_override from eql sequence rules
* add config entry for eql rules with beats index and t_o
* add timestamp_override to missing fields

detection_rules/rule.py

@@ -294,10 +294,15 @@ def convert_relative_delta(self, lookback: str) -> int:
         else:
             return self.convert_time_span(lookback)
 
+    @cached_property
+    def is_sequence(self) -> bool:
+        """Checks if the current rule is a sequence-based rule."""
+        return eql.utils.get_query_type(self.ast) == 'sequence'
+
     @cached_property
     def max_span(self) -> Optional[int]:
         """Maxspan value for sequence rules if defined."""
-        if eql.utils.get_query_type(self.ast) == 'sequence' and hasattr(self.ast.first, 'max_span'):
+        if self.is_sequence and hasattr(self.ast.first, 'max_span'):
             return self.ast.first.max_span.as_milliseconds() if self.ast.first.max_span else None
 
     @cached_property

rules/cross-platform/credential_access_cookies_chromium_browsers_debugging.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/12/21"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -17,6 +17,10 @@ language = "eql"
 license = "Elastic License v2"
 max_signals = 33
 name = "Potential Cookies Theft via Browser Debugging"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://github.com/defaultnamehere/cookie_crimes",
     "https://embracethered.com/blog/posts/2020/cookie-crimes-on-mirosoft-edge/",

rules/cross-platform/defense_evasion_deleting_websvr_access_logs.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/11/03"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -14,6 +14,10 @@ index = ["auditbeat-*", "winlogbeat-*", "logs-endpoint.events.*", "logs-windows.
 language = "eql"
 license = "Elastic License v2"
 name = "WebServer Access Logs Deleted"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 risk_score = 47
 rule_id = "665e7a4f-c58e-4fc6-bc83-87a7572670ac"
 severity = "medium"

rules/cross-platform/defense_evasion_timestomp_touch.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/11/03"
 maturity = "production"
-updated_date = "2021/03/09"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -15,6 +15,10 @@ language = "eql"
 license = "Elastic License v2"
 max_signals = 33
 name = "Timestomping using Touch Command"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 risk_score = 47
 rule_id = "b0046934-486e-462f-9487-0d4cf9e429c6"
 severity = "medium"

rules/cross-platform/discovery_security_software_grep.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/12/20"
 maturity = "production"
-updated_date = "2021/03/08"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -15,6 +15,10 @@ index = ["logs-endpoint.events.*", "auditbeat-*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Security Software Discovery via Grep"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 risk_score = 47
 rule_id = "870aecc0-cea4-4110-af3f-e02e9b373655"
 severity = "medium"
@@ -78,3 +82,4 @@ reference = "https://attack.mitre.org/techniques/T1518/001/"
 id = "TA0007"
 name = "Discovery"
 reference = "https://attack.mitre.org/tactics/TA0007/"
+

rules/cross-platform/discovery_virtual_machine_fingerprinting_grep.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2021/09/29"
 maturity = "production"
-updated_date = "2021/09/29"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -21,6 +21,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Virtual Machine Fingerprinting via Grep"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = ["https://objective-see.com/blog/blog_0x4F.html"]
 risk_score = 47
 rule_id = "c85eb82c-d2c8-485c-a36f-534f914b7663"
@@ -49,3 +53,4 @@ reference = "https://attack.mitre.org/techniques/T1082/"
 id = "TA0007"
 name = "Discovery"
 reference = "https://attack.mitre.org/tactics/TA0007/"
+

rules/cross-platform/execution_python_script_in_cmdline.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2021/01/13"
 maturity = "development"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -15,6 +15,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Python Script Execution via Command Line"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 risk_score = 47
 rule_id = "ee9f08dc-cf80-4124-94ae-08c405f059ae"
 severity = "medium"

rules/cross-platform/execution_revershell_via_shell_cmd.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/01/07"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -11,6 +11,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Potential Reverse Shell Activity via Terminal"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://github.com/swisskyrepo/PayloadsAllTheThings/blob/master/Methodology%20and%20Resources/Reverse%20Shell%20Cheatsheet.md",
     "https://github.com/WangYihang/Reverse-Shell-Manager",

rules/cross-platform/execution_suspicious_jar_child_process.toml

@@ -1,23 +1,27 @@
 [metadata]
 creation_date = "2021/01/19"
 maturity = "production"
-updated_date = "2021/12/10"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
 description = """
-Identifies suspicious child processes of the Java interpreter process. This may indicate an attempt to execute a malicious
-JAR file or an exploitation attempt via a JAVA specific vulnerability.
+Identifies suspicious child processes of the Java interpreter process. This may indicate an attempt to execute a
+malicious JAR file or an exploitation attempt via a JAVA specific vulnerability.
 """
 from = "now-9m"
 index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Suspicious JAVA Child Process"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
-"https://www.lunasec.io/docs/blog/log4j-zero-day/",
-"https://github.com/christophetd/log4shell-vulnerable-app",
-"https://www.blackhat.com/docs/us-16/materials/us-16-Munoz-A-Journey-From-JNDI-LDAP-Manipulation-To-RCE.pdf",
+    "https://www.lunasec.io/docs/blog/log4j-zero-day/",
+    "https://github.com/christophetd/log4shell-vulnerable-app",
+    "https://www.blackhat.com/docs/us-16/materials/us-16-Munoz-A-Journey-From-JNDI-LDAP-Manipulation-To-RCE.pdf",
 ]
 risk_score = 47
 rule_id = "8acb7614-1d92-4359-bfcf-478b6d9de150"

rules/cross-platform/execution_suspicious_java_netcon_childproc.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2021/12/10"
 maturity = "production"
-updated_date = "2021/12/10"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -23,7 +23,6 @@ risk_score = 73
 rule_id = "c3f5e1d8-910e-43b4-8d44-d748e498ca86"
 severity = "high"
 tags = ["Elastic", "Host", "Linux", "macOS", "Threat Detection", "Execution"]
-timestamp_override = "event.ingested"
 type = "eql"
 
 query = '''

rules/cross-platform/impact_hosts_file_modified.toml

@@ -20,7 +20,10 @@ license = "Elastic License v2"
 name = "Hosts File Modified"
 note = """## Config
 
-For Windows systems using Auditbeat, this rule requires adding `C:/Windows/System32/drivers/etc` as an additional path in the 'file_integrity' module of auditbeat.yml."""
+For Windows systems using Auditbeat, this rule requires adding `C:/Windows/System32/drivers/etc` as an additional path in the 'file_integrity' module of auditbeat.yml.
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = ["https://www.elastic.co/guide/en/beats/auditbeat/current/auditbeat-reference-yml.html"]
 risk_score = 47
 rule_id = "9c260313-c811-4ec8-ab89-8f6530e0246c"

rules/integrations/okta/credential_access_mfa_push_brute_force.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2022/01/05"
 maturity = "production"
-updated_date = "2022/02/28"
+updated_date = "2022/03/31"
 integration = "okta"
 
 [rule]
@@ -23,7 +23,6 @@ risk_score = 73
 rule_id = "97a8e584-fd3b-421f-9b9d-9c9d9e57e9d7"
 severity = "high"
 tags = ["Elastic", "Identity", "Okta", "Continuous Monitoring", "SecOps", "Identity and Access"]
-timestamp_override = "event.ingested"
 type = "eql"
 
 query = '''

rules/linux/command_and_control_tunneling_via_earthworm.toml

@@ -1,22 +1,27 @@
 [metadata]
 creation_date = "2021/04/12"
 maturity = "production"
-updated_date = "2021/04/12"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
 description = """
 Identifies the execution of the EarthWorm tunneler. Adversaries may tunnel network communications to and from a victim
-system within a separate protocol to avoid detection and network filtering, or to enable access to otherwise unreachable systems.
+system within a separate protocol to avoid detection and network filtering, or to enable access to otherwise unreachable
+systems.
 """
 from = "now-9m"
 index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Potential Protocol Tunneling via EarthWorm"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "http://rootkiter.com/EarthWorm/",
-    "https://decoded.avast.io/luigicamastra/apt-group-targeting-governmental-agencies-in-east-asia/"
+    "https://decoded.avast.io/luigicamastra/apt-group-targeting-governmental-agencies-in-east-asia/",
 ]
 risk_score = 47
 rule_id = "9f1c4ca3-44b5-481d-ba42-32dc215a2769"
@@ -38,7 +43,9 @@ id = "T1572"
 name = "Protocol Tunneling"
 reference = "https://attack.mitre.org/techniques/T1572/"
 
+
 [rule.threat.tactic]
 id = "TA0011"
 name = "Command and Control"
 reference = "https://attack.mitre.org/tactics/TA0011/"
+

rules/linux/credential_access_ssh_backdoor_log.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/12/21"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -16,6 +16,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Potential OpenSSH Backdoor Logging Activity"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://github.com/eset/malware-ioc/tree/master/sshdoor",
     "https://www.welivesecurity.com/wp-content/uploads/2021/01/ESET_Kobalos.pdf",

rules/linux/defense_evasion_deletion_of_bash_command_line_history.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/05/04"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -14,6 +14,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "Tampering of Bash Command-Line History"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 risk_score = 47
 rule_id = "7bcbb3ac-e533-41ad-a612-d6c3bf666aba"
 severity = "medium"

rules/linux/defense_evasion_hidden_file_dir_tmp.toml

@@ -1,9 +1,9 @@
 [metadata]
 creation_date = "2020/04/29"
 maturity = "production"
-updated_date = "2021/03/03"
 min_stack_comments = "EQL regex syntax introduced in 7.12"
 min_stack_version = "7.12.0"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -24,6 +24,10 @@ language = "eql"
 license = "Elastic License v2"
 max_signals = 33
 name = "Creation of Hidden Files and Directories"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 risk_score = 47
 rule_id = "b9666521-4742-49ce-9ddc-b8e84c35acae"
 severity = "medium"

rules/linux/defense_evasion_log_files_deleted.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2020/11/03"
 maturity = "production"
-updated_date = "2021/03/03"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -14,6 +14,10 @@ index = ["auditbeat-*", "logs-endpoint.events.*"]
 language = "eql"
 license = "Elastic License v2"
 name = "System Log File Deletion"
+note = """## Config
+
+If enabling an EQL rule on a non-elastic-agent index (such as beats) for versions <8.2, events will not define `event.ingested` and default fallback for EQL rules was not added until 8.2, so you will need to add a custom pipeline to populate `event.ingested` to @timestamp for this rule to work.
+"""
 references = [
     "https://www.fireeye.com/blog/threat-research/2020/11/live-off-the-land-an-overview-of-unc1945.html",
 ]

rules/linux/execution_apt_binary.toml

@@ -1,7 +1,7 @@
 [metadata]
 creation_date = "2022/02/24"
 maturity = "production"
-updated_date = "2022/03/28"
+updated_date = "2022/03/31"
 
 [rule]
 author = ["Elastic"]
@@ -26,8 +26,8 @@ timestamp_override = "event.ingested"
 type = "eql"
 
 query = '''
-process where event.type == "start" and process.name == "sensible-pager" and 
-  process.args in ("/bin/sh", "/bin/bash", "/bin/dash", "sh", "bash", "dash") and 
+process where event.type == "start" and process.name == "sensible-pager" and
+  process.args in ("/bin/sh", "/bin/bash", "/bin/dash", "sh", "bash", "dash") and
   process.parent.name in ("apt", "apt-get") and process.parent.args == "changelog"
 '''