From 5ddda3c0f4ef9e3d382500b226bb6684c670274a Mon Sep 17 00:00:00 2001
From: Ross Wolf <31489089+rw-access@users.noreply.github.com>
Date: Thu, 24 Sep 2020 16:27:53 -0600
Subject: [PATCH 1/7] Add ATT&CK subtechniques to the schema

---
 detection_rules/schemas/v7_10.py | 39 ++++++++++++++
 detection_rules/schemas/v7_8.py  |  8 +--
 tests/test_schemas.py            | 90 ++++++++++++++++++--------------
 3 files changed, 94 insertions(+), 43 deletions(-)

diff --git a/detection_rules/schemas/v7_10.py b/detection_rules/schemas/v7_10.py
index c2bc2c1372f..d2b131ec89f 100644
--- a/detection_rules/schemas/v7_10.py
+++ b/detection_rules/schemas/v7_10.py
@@ -5,6 +5,7 @@
 """Definitions for rule metadata and schemas."""
 
 import jsl
+from .v7_8 import Threat as Threat78, MITRE_URL_PATTERN
 from .v7_9 import ApiSchema79
 
 
@@ -12,6 +13,23 @@
 EQL = "eql"
 
 
+class Threat710(Threat78):
+    """Threat framework mapping such as MITRE ATT&CK."""
+
+    class ThreatTechnique(Threat78.ThreatTechnique):
+        """Patched threat.technique to add threat.technique.subtechnique."""
+
+        class ThreatSubTechnique(jsl.Document):
+            id = jsl.StringField(required=True)
+            name = jsl.StringField(required=True)
+            reference = jsl.StringField(MITRE_URL_PATTERN.format(type='techniques') + r"[0-9]+/")
+
+        subtechnique = jsl.ArrayField(jsl.DocumentField(ThreatSubTechnique))
+
+    # override the `technique` field definition
+    technique = jsl.ArrayField(jsl.DocumentField(ThreatTechnique), required=True)
+
+
 class ApiSchema710(ApiSchema79):
     """Schema for siem rule in API format."""
 
@@ -26,6 +44,8 @@ class ApiSchema710(ApiSchema79):
     ml_scope = ApiSchema79.ml_scope
     threshold_scope = ApiSchema79.threshold_scope
 
+    threat = jsl.ArrayField(jsl.DocumentField(Threat710))
+
     with jsl.Scope(EQL) as eql_scope:
         eql_scope.index = jsl.ArrayField(jsl.StringField(), required=False)
         eql_scope.query = jsl.StringField(required=True)
@@ -34,3 +54,22 @@ class ApiSchema710(ApiSchema79):
 
     with jsl.Scope(jsl.DEFAULT_ROLE) as default_scope:
         default_scope.type = type
+
+    @classmethod
+    def downgrade(cls, target_cls, document, role=None):
+        """Remove 7.10 additions from the rule."""
+        # ignore when this method is inherited by subclasses
+        if cls == ApiSchema710 and "threat" in document:
+            threat_field = list(document["threat"])
+            for threat in threat_field:
+                if "technique" in threat:
+                    threat["technique"] = [t.copy() for t in threat["technique"]]
+
+                    for technique in threat["technique"]:
+                        technique.pop("subtechnique", None)
+
+            document = document.copy()
+            document["threat"] = threat_field
+
+        # now strip any any unrecognized properties
+        return target_cls.strip_additional_properties(document, role)
diff --git a/detection_rules/schemas/v7_8.py b/detection_rules/schemas/v7_8.py
index d467d9a127c..fa955243ae4 100644
--- a/detection_rules/schemas/v7_8.py
+++ b/detection_rules/schemas/v7_8.py
@@ -65,13 +65,13 @@ class Threat(jsl.Document):
     """Threat framework mapping such as MITRE ATT&CK."""
 
     class ThreatTactic(jsl.Document):
-        id = jsl.StringField(enum=tactics_map.values())
-        name = jsl.StringField(enum=tactics)
+        id = jsl.StringField(enum=tactics_map.values(), required=True)
+        name = jsl.StringField(enum=tactics, required=True)
         reference = jsl.StringField(MITRE_URL_PATTERN.format(type='tactics'))
 
     class ThreatTechnique(jsl.Document):
-        id = jsl.StringField(enum=list(technique_lookup))
-        name = jsl.StringField()
+        id = jsl.StringField(enum=list(technique_lookup), required=True)
+        name = jsl.StringField(required=True)
         reference = jsl.StringField(MITRE_URL_PATTERN.format(type='techniques'))
 
     framework = jsl.StringField(default='MITRE ATT&CK', required=True)
diff --git a/tests/test_schemas.py b/tests/test_schemas.py
index 08966862c63..2c05c9fe2c1 100644
--- a/tests/test_schemas.py
+++ b/tests/test_schemas.py
@@ -6,6 +6,7 @@
 import unittest
 import uuid
 import eql
+import copy
 
 from detection_rules.rule import Rule
 from detection_rules.schemas import downgrade, CurrentSchema
@@ -16,21 +17,46 @@ class TestSchemas(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        cls.compatible_rule = Rule("test.toml", {
-            "author": ["Elastic"],
+        # expected contents for a downgraded rule
+        cls.v78_kql = {
             "description": "test description",
             "index": ["filebeat-*"],
             "language": "kuery",
-            "license": "Elastic License",
             "name": "test rule",
             "query": "process.name:test.query",
             "risk_score": 21,
             "rule_id": str(uuid.uuid4()),
             "severity": "low",
-            "type": "query"
-        })
-        cls.versioned_rule = cls.compatible_rule.copy()
+            "type": "query",
+            "threat": [
+                {
+                    "framework": "MITRE ATT&CK",
+                    "tactic": {
+                        "id": "TA0001",
+                        "name": "Execution",
+                        "reference": "https://attack.mitre.org/tactics/TA0001/"
+                    },
+                    "technique": [
+                        {
+                            "id": "T1059",
+                            "name": "Command and Scripting Interpreter",
+                            "reference": "https://attack.mitre.org/techniques/T1059/",
+                        }
+                    ],
+                }
+            ]
+        }
+        cls.v79_kql = dict(cls.v78_kql, author=["Elastic"], license="Elastic License")
+        cls.v710_kql = copy.deepcopy(cls.v79_kql)
+        cls.v710_kql["threat"][0]["technique"][0]["subtechnique"] = [{
+            "id": "T1059.001",
+            "name": "PowerShell",
+            "reference": "https://attack.mitre.org/techniques/T1059/001/"
+        }]
+
+        cls.versioned_rule = Rule("test.toml", copy.deepcopy(cls.v79_kql))
         cls.versioned_rule.contents["version"] = 10
+
         cls.threshold_rule = Rule("test.toml", {
             "author": ["Elastic"],
             "description": "test description",
@@ -50,47 +76,33 @@ def setUpClass(cls):
 
     def test_query_downgrade(self):
         """Downgrade a standard KQL rule."""
-        api_contents = self.compatible_rule.contents
-        self.assertDictEqual(downgrade(api_contents, CurrentSchema.STACK_VERSION), api_contents)
-        self.assertDictEqual(downgrade(api_contents, "7.9"), api_contents)
-        self.assertDictEqual(downgrade(api_contents, "7.9.2"), api_contents)
-        self.assertDictEqual(downgrade(api_contents, "7.8"), {
-            # "author": ["Elastic"],
-            "description": "test description",
-            "index": ["filebeat-*"],
-            "language": "kuery",
-            # "license": "Elastic License",
-            "name": "test rule",
-            "query": "process.name:test.query",
-            "risk_score": 21,
-            "rule_id": self.compatible_rule.id,
-            "severity": "low",
-            "type": "query"
-        })
+        self.assertDictEqual(downgrade(self.v710_kql, "7.10"), self.v710_kql)
+        self.assertDictEqual(downgrade(self.v710_kql, "7.9"), self.v79_kql)
+        self.assertDictEqual(downgrade(self.v710_kql, "7.9.2"), self.v79_kql)
+        self.assertDictEqual(downgrade(self.v710_kql, "7.8.1"), self.v78_kql)
+        self.assertDictEqual(downgrade(self.v79_kql, "7.8"), self.v78_kql)
+        self.assertDictEqual(downgrade(self.v79_kql, "7.8"), self.v78_kql)
 
         with self.assertRaises(ValueError):
-            downgrade(api_contents, "7.7")
+            downgrade(self.v710_kql, "7.7")
+
+        with self.assertRaises(ValueError):
+            downgrade(self.v79_kql, "7.7")
+
+        with self.assertRaises(ValueError):
+            downgrade(self.v78_kql, "7.7")
 
     def test_versioned_downgrade(self):
         """Downgrade a KQL rule with version information"""
         api_contents = self.versioned_rule.contents
-        self.assertDictEqual(downgrade(api_contents, CurrentSchema.STACK_VERSION), api_contents)
         self.assertDictEqual(downgrade(api_contents, "7.9"), api_contents)
         self.assertDictEqual(downgrade(api_contents, "7.9.2"), api_contents)
-        self.assertDictEqual(downgrade(api_contents, "7.8"), {
-            # "author": ["Elastic"],
-            "description": "test description",
-            "index": ["filebeat-*"],
-            "language": "kuery",
-            # "license": "Elastic License",
-            "name": "test rule",
-            "query": "process.name:test.query",
-            "risk_score": 21,
-            "rule_id": self.versioned_rule.id,
-            "severity": "low",
-            "type": "query",
-            "version": 10,
-        })
+
+        api_contents78 = api_contents.copy()
+        api_contents78.pop("author")
+        api_contents78.pop("license")
+
+        self.assertDictEqual(downgrade(api_contents, "7.8"), api_contents78)
 
         with self.assertRaises(ValueError):
             downgrade(api_contents, "7.7")

From 45eed20f1fdedbf9c5fd48cbf9bd43533d0b832d Mon Sep 17 00:00:00 2001
From: Ross Wolf <31489089+rw-access@users.noreply.github.com>
Date: Tue, 8 Dec 2020 14:33:57 -0700
Subject: [PATCH 2/7] Switch subtechniques to the 7.11 schema

---
 detection_rules/schemas/__init__.py |  2 +
 detection_rules/schemas/v7_11.py    | 58 +++++++++++++++++++++++++++++
 tests/test_schemas.py               | 14 +++----
 3 files changed, 67 insertions(+), 7 deletions(-)
 create mode 100644 detection_rules/schemas/v7_11.py

diff --git a/detection_rules/schemas/__init__.py b/detection_rules/schemas/__init__.py
index d29352d6a60..15c859c7aa2 100644
--- a/detection_rules/schemas/__init__.py
+++ b/detection_rules/schemas/__init__.py
@@ -10,6 +10,7 @@
 from .v7_8 import ApiSchema78
 from .v7_9 import ApiSchema79
 from .v7_10 import ApiSchema710
+from .v7_11 import ApiSchema711
 
 __all__ = (
     "all_schemas",
@@ -23,6 +24,7 @@
     ApiSchema78,
     ApiSchema79,
     ApiSchema710,
+    ApiSchema711,
 ]
 
 CurrentSchema = all_schemas[-1]
diff --git a/detection_rules/schemas/v7_11.py b/detection_rules/schemas/v7_11.py
new file mode 100644
index 00000000000..045c1564a11
--- /dev/null
+++ b/detection_rules/schemas/v7_11.py
@@ -0,0 +1,58 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Definitions for rule metadata and schemas."""
+
+import jsl
+from .v7_8 import Threat as Threat78, MITRE_URL_PATTERN
+from .v7_10 import ApiSchema710
+
+
+# rule types
+EQL = "eql"
+
+
+class Threat711(Threat78):
+    """Threat framework mapping such as MITRE ATT&CK."""
+
+    class ThreatTechnique(Threat78.ThreatTechnique):
+        """Patched threat.technique to add threat.technique.subtechnique."""
+
+        class ThreatSubTechnique(jsl.Document):
+            id = jsl.StringField(required=True)
+            name = jsl.StringField(required=True)
+            reference = jsl.StringField(MITRE_URL_PATTERN.format(type='techniques') + r"[0-9]+/")
+
+        subtechnique = jsl.ArrayField(jsl.DocumentField(ThreatSubTechnique), required=False)
+
+    # override the `technique` field definition
+    technique = jsl.ArrayField(jsl.DocumentField(ThreatTechnique), required=False)
+
+
+class ApiSchema711(ApiSchema710):
+    """Schema for siem rule in API format."""
+
+    STACK_VERSION = "7.11"
+    RULE_TYPES = ApiSchema710.RULE_TYPES
+
+    threat = jsl.ArrayField(jsl.DocumentField(Threat711))
+
+    @classmethod
+    def downgrade(cls, target_cls, document, role=None):
+        """Remove 7.11 additions from the rule."""
+        # ignore when this method is inherited by subclasses
+        if cls == ApiSchema711 and "threat" in document:
+            threat_field = list(document["threat"])
+            for threat in threat_field:
+                if "technique" in threat:
+                    threat["technique"] = [t.copy() for t in threat["technique"]]
+
+                    for technique in threat["technique"]:
+                        technique.pop("subtechnique", None)
+
+            document = document.copy()
+            document["threat"] = threat_field
+
+        # now strip any any unrecognized properties
+        return target_cls.strip_additional_properties(document, role)
diff --git a/tests/test_schemas.py b/tests/test_schemas.py
index 2c05c9fe2c1..64f0d52247e 100644
--- a/tests/test_schemas.py
+++ b/tests/test_schemas.py
@@ -47,8 +47,8 @@ def setUpClass(cls):
             ]
         }
         cls.v79_kql = dict(cls.v78_kql, author=["Elastic"], license="Elastic License")
-        cls.v710_kql = copy.deepcopy(cls.v79_kql)
-        cls.v710_kql["threat"][0]["technique"][0]["subtechnique"] = [{
+        cls.v711_kql = copy.deepcopy(cls.v79_kql)
+        cls.v711_kql["threat"][0]["technique"][0]["subtechnique"] = [{
             "id": "T1059.001",
             "name": "PowerShell",
             "reference": "https://attack.mitre.org/techniques/T1059/001/"
@@ -76,15 +76,15 @@ def setUpClass(cls):
 
     def test_query_downgrade(self):
         """Downgrade a standard KQL rule."""
-        self.assertDictEqual(downgrade(self.v710_kql, "7.10"), self.v710_kql)
-        self.assertDictEqual(downgrade(self.v710_kql, "7.9"), self.v79_kql)
-        self.assertDictEqual(downgrade(self.v710_kql, "7.9.2"), self.v79_kql)
-        self.assertDictEqual(downgrade(self.v710_kql, "7.8.1"), self.v78_kql)
+        self.assertDictEqual(downgrade(self.v711_kql, "7.11"), self.v711_kql)
+        self.assertDictEqual(downgrade(self.v711_kql, "7.9"), self.v79_kql)
+        self.assertDictEqual(downgrade(self.v711_kql, "7.9.2"), self.v79_kql)
+        self.assertDictEqual(downgrade(self.v711_kql, "7.8.1"), self.v78_kql)
         self.assertDictEqual(downgrade(self.v79_kql, "7.8"), self.v78_kql)
         self.assertDictEqual(downgrade(self.v79_kql, "7.8"), self.v78_kql)
 
         with self.assertRaises(ValueError):
-            downgrade(self.v710_kql, "7.7")
+            downgrade(self.v711_kql, "7.7")
 
         with self.assertRaises(ValueError):
             downgrade(self.v79_kql, "7.7")

From a4dc27ce9d97f4959a83488ea4df96fda34ecee4 Mon Sep 17 00:00:00 2001
From: Ross Wolf <31489089+rw-access@users.noreply.github.com>
Date: Tue, 8 Dec 2020 14:39:59 -0700
Subject: [PATCH 3/7] Make technique still required

---
 detection_rules/schemas/v7_10.py | 38 --------------------------------
 detection_rules/schemas/v7_11.py |  2 +-
 2 files changed, 1 insertion(+), 39 deletions(-)

diff --git a/detection_rules/schemas/v7_10.py b/detection_rules/schemas/v7_10.py
index d2b131ec89f..41bbcb66d2a 100644
--- a/detection_rules/schemas/v7_10.py
+++ b/detection_rules/schemas/v7_10.py
@@ -13,23 +13,6 @@
 EQL = "eql"
 
 
-class Threat710(Threat78):
-    """Threat framework mapping such as MITRE ATT&CK."""
-
-    class ThreatTechnique(Threat78.ThreatTechnique):
-        """Patched threat.technique to add threat.technique.subtechnique."""
-
-        class ThreatSubTechnique(jsl.Document):
-            id = jsl.StringField(required=True)
-            name = jsl.StringField(required=True)
-            reference = jsl.StringField(MITRE_URL_PATTERN.format(type='techniques') + r"[0-9]+/")
-
-        subtechnique = jsl.ArrayField(jsl.DocumentField(ThreatSubTechnique))
-
-    # override the `technique` field definition
-    technique = jsl.ArrayField(jsl.DocumentField(ThreatTechnique), required=True)
-
-
 class ApiSchema710(ApiSchema79):
     """Schema for siem rule in API format."""
 
@@ -44,8 +27,6 @@ class ApiSchema710(ApiSchema79):
     ml_scope = ApiSchema79.ml_scope
     threshold_scope = ApiSchema79.threshold_scope
 
-    threat = jsl.ArrayField(jsl.DocumentField(Threat710))
-
     with jsl.Scope(EQL) as eql_scope:
         eql_scope.index = jsl.ArrayField(jsl.StringField(), required=False)
         eql_scope.query = jsl.StringField(required=True)
@@ -54,22 +35,3 @@ class ApiSchema710(ApiSchema79):
 
     with jsl.Scope(jsl.DEFAULT_ROLE) as default_scope:
         default_scope.type = type
-
-    @classmethod
-    def downgrade(cls, target_cls, document, role=None):
-        """Remove 7.10 additions from the rule."""
-        # ignore when this method is inherited by subclasses
-        if cls == ApiSchema710 and "threat" in document:
-            threat_field = list(document["threat"])
-            for threat in threat_field:
-                if "technique" in threat:
-                    threat["technique"] = [t.copy() for t in threat["technique"]]
-
-                    for technique in threat["technique"]:
-                        technique.pop("subtechnique", None)
-
-            document = document.copy()
-            document["threat"] = threat_field
-
-        # now strip any any unrecognized properties
-        return target_cls.strip_additional_properties(document, role)
diff --git a/detection_rules/schemas/v7_11.py b/detection_rules/schemas/v7_11.py
index 045c1564a11..cbd09dee82b 100644
--- a/detection_rules/schemas/v7_11.py
+++ b/detection_rules/schemas/v7_11.py
@@ -27,7 +27,7 @@ class ThreatSubTechnique(jsl.Document):
         subtechnique = jsl.ArrayField(jsl.DocumentField(ThreatSubTechnique), required=False)
 
     # override the `technique` field definition
-    technique = jsl.ArrayField(jsl.DocumentField(ThreatTechnique), required=False)
+    technique = jsl.ArrayField(jsl.DocumentField(ThreatTechnique), required=True)
 
 
 class ApiSchema711(ApiSchema710):

From b55d9b1aae1695246ec91613170182a688565711 Mon Sep 17 00:00:00 2001
From: Ross Wolf <31489089+rw-access@users.noreply.github.com>
Date: Tue, 8 Dec 2020 14:40:50 -0700
Subject: [PATCH 4/7] Lint fixes

---
 detection_rules/schemas/v7_10.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/detection_rules/schemas/v7_10.py b/detection_rules/schemas/v7_10.py
index 41bbcb66d2a..c2bc2c1372f 100644
--- a/detection_rules/schemas/v7_10.py
+++ b/detection_rules/schemas/v7_10.py
@@ -5,7 +5,6 @@
 """Definitions for rule metadata and schemas."""
 
 import jsl
-from .v7_8 import Threat as Threat78, MITRE_URL_PATTERN
 from .v7_9 import ApiSchema79
 
 

From 76090b7f23bcbf50f6a446bc1d05e11fa9e0f05f Mon Sep 17 00:00:00 2001
From: Ross Wolf <31489089+rw-access@users.noreply.github.com>
Date: Tue, 8 Dec 2020 14:41:34 -0700
Subject: [PATCH 5/7] Cleanup EQL constant

---
 detection_rules/schemas/v7_10.py | 4 ----
 detection_rules/schemas/v7_11.py | 4 ----
 2 files changed, 8 deletions(-)

diff --git a/detection_rules/schemas/v7_10.py b/detection_rules/schemas/v7_10.py
index c2bc2c1372f..9aa50018e8a 100644
--- a/detection_rules/schemas/v7_10.py
+++ b/detection_rules/schemas/v7_10.py
@@ -8,10 +8,6 @@
 from .v7_9 import ApiSchema79
 
 
-# rule types
-EQL = "eql"
-
-
 class ApiSchema710(ApiSchema79):
     """Schema for siem rule in API format."""
 
diff --git a/detection_rules/schemas/v7_11.py b/detection_rules/schemas/v7_11.py
index cbd09dee82b..11eefa7baa8 100644
--- a/detection_rules/schemas/v7_11.py
+++ b/detection_rules/schemas/v7_11.py
@@ -9,10 +9,6 @@
 from .v7_10 import ApiSchema710
 
 
-# rule types
-EQL = "eql"
-
-
 class Threat711(Threat78):
     """Threat framework mapping such as MITRE ATT&CK."""
 

From 9ff067be0d8feff8c815224d3f71894a46827e41 Mon Sep 17 00:00:00 2001
From: Ross Wolf <31489089+rw-access@users.noreply.github.com>
Date: Tue, 8 Dec 2020 14:41:55 -0700
Subject: [PATCH 6/7] Trim more cruft

---
 detection_rules/schemas/v7_11.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/detection_rules/schemas/v7_11.py b/detection_rules/schemas/v7_11.py
index 11eefa7baa8..7263bc08ae1 100644
--- a/detection_rules/schemas/v7_11.py
+++ b/detection_rules/schemas/v7_11.py
@@ -30,7 +30,6 @@ class ApiSchema711(ApiSchema710):
     """Schema for siem rule in API format."""
 
     STACK_VERSION = "7.11"
-    RULE_TYPES = ApiSchema710.RULE_TYPES
 
     threat = jsl.ArrayField(jsl.DocumentField(Threat711))
 

From 437954d2243526851878d11799738ff963ddf046 Mon Sep 17 00:00:00 2001
From: Ross Wolf <31489089+rw-access@users.noreply.github.com>
Date: Tue, 8 Dec 2020 14:44:25 -0700
Subject: [PATCH 7/7] Restore EQL for 710

---
 detection_rules/schemas/v7_10.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/detection_rules/schemas/v7_10.py b/detection_rules/schemas/v7_10.py
index 9aa50018e8a..c2bc2c1372f 100644
--- a/detection_rules/schemas/v7_10.py
+++ b/detection_rules/schemas/v7_10.py
@@ -8,6 +8,10 @@
 from .v7_9 import ApiSchema79
 
 
+# rule types
+EQL = "eql"
+
+
 class ApiSchema710(ApiSchema79):
     """Schema for siem rule in API format."""