Add ctibulter api key (#13)

* add support for CTIBUTLER API Key #12

* add deprecated folder #11

* replace CTIBUTLER_HOST with CTIBUTLER_BASE_URL

* updating docs for new sigma version

* updating docs

---------

Co-authored-by: David G <[email protected]>

.env.example

@@ -1,4 +1,6 @@
 ## CTIBUTLER
-CTIBUTLER_HOST=
+CTIBUTLER_BASE_URL=
+CTIBUTLER_API_KEY=
 ## VULMATCH
-VULMATCH_HOST=
+VULMATCH_BASE_URL=
+VULMATCH_API_KEY=

.env.markdown

@@ -8,12 +8,16 @@ However, if you just want to experiment, set the following values
 
 sigma2stix requires [ctibutler](https://github.com/muchdogesec/ctibutler) to lookup ATT&CK references
 
-* `CTIBUTLER_HOST`: `'http://host.docker.internal:8006'`
-	* If you are running CTI Butler locally, be sure to set `'http://host.docker.internal:8006'` in the `.env` file otherwise you will run into networking errors.
+* `CTIBUTLER_BASE_URL`: `'http://api.ctibutler.com'` (recommended)
+	* If you are running CTI Butler locally, be sure to set `'http://host.docker.internal:8006/api/'` in the `.env` file otherwise you will run into networking errors.
+* `CTIBUTLER_API_KEY`:
+	* If using `'http://api.ctibutler.com'`, [get your API key here](http://app.ctibutler.com). Not required if running locally.
 
 ## VULMATCH
 
 sigma2stix requires [vulmatch](https://github.com/muchdogesec/vulmatch) to lookup CVEs references
 
-* `VULMATCH_HOST`: `'http://host.docker.internal:8005'`
-	* If you are running Vulmatch locally, be sure to set `'http://host.docker.internal:8005'` in the `.env` file otherwise you will run into networking errors.
+* `VULMATCH_BASE_URL`: `'http://api.vulmatch.com'` (recommended)
+	* If you are running Vulmatch locally, be sure to set `'http://host.docker.internal:8005/api/'` in the `.env` file otherwise you will run into networking errors.
+* `VULMATCH_API_KEY`:
+	* If using `'http://api.vulmatch.com'`, [get your API key here](http://app.vulmatch.com). Can be left blank if running locally.

README.md

@@ -76,7 +76,7 @@ python3 sigma2stix.py \
 Where;
 
 * `mode` (required): should always be `sigmahq` if you want to download the latest rules from the [SigmaHQ/sigma repository](https://github.com/SigmaHQ/sigma)
-* `sigma_version_tag` (optional): is the name of the tag in the SigmaHQ/sigma repository ([tags listed here](https://github.com/SigmaHQ/sigma/releases)), e.g. `r2023-08-24`. If no value passed, the master branch will be cloned.
+* `sigma_version_tag` (optional): is the name of the tag in the SigmaHQ/sigma repository ([tags listed here](https://github.com/SigmaHQ/sigma/releases)), e.g. `r2024-12-19`. If no value passed, the master branch will be cloned.
 
 Note this script only supports Sigma Rule version tags in the format `rYYYY-MM-DD`.
 
@@ -94,7 +94,7 @@ python3 sigma2stix.py \
 ```shell
 python3 sigma2stix.py \
 	--mode sigmahq \
-	--sigma_version_tag r2023-08-24
+	--sigma_version_tag r2024-12-19
 ```
 
 ### Mode 2: SigmaHQ YAML file -> STIX

src/parser.py

@@ -37,6 +37,7 @@ def parse_indicator(cls, data:dict, path:str, url: str) -> list:
                 pattern=data,
                 pattern_type="sigma",
                 valid_from=as_date(data.get('date')),
+                revoked=data.get('status') == 'deprecated',
                 external_references=[
                     {
                         "source_name": "sigma-rule",

src/retriever.py

@@ -10,20 +10,20 @@
 class STIXObjectRetriever:
     def __init__(self, host="ctibutler") -> None:
         if host == "ctibutler":
-            self.api_root = os.environ['CTIBUTLER_HOST']
-            self.api_key = os.environ.get('CTIBUTLER_APIKEY')
+            self.api_root = os.environ['CTIBUTLER_BASE_URL'] + '/'
+            self.api_key = os.environ.get('CTIBUTLER_API_KEY')
         elif host == "vulmatch":
-            self.api_root = os.environ['VULMATCH_HOST']
-            self.api_key = os.environ.get('VULMATCH_APIKEY')
+            self.api_root = os.environ['VULMATCH_BASE_URL'] + '/'
+            self.api_key = os.environ.get('VULMATCH_API_KEY')
         else:
             raise NotImplementedError("The type `%s` is not supported", host)
 
     def get_attack_objects(self, matrix, attack_id):
-        endpoint = urljoin(self.api_root, f"/api/v1/attack-{matrix}/objects/{attack_id}/")
+        endpoint = urljoin(self.api_root, f"v1/attack-{matrix}/objects/{attack_id}/")
         return self._retrieve_objects(endpoint)
 
     def get_objects_by_external_ids(self, ids, type, key='objects', query_filter='id'):
-        objects = self._retrieve_objects(urljoin(self.api_root, f"/api/v1/{type}/objects/?{query_filter}={','.join(ids)}"), key)
+        objects = self._retrieve_objects(urljoin(self.api_root, f"v1/{type}/objects/?{query_filter}={','.join(ids)}"), key)
         objects_map : dict[str, list[dict]] = {}
         for obj in objects:
             object_id = obj['external_references'][0]['external_id']
@@ -32,20 +32,20 @@ def get_objects_by_external_ids(self, ids, type, key='objects', query_filter='id
         return objects_map
 
     def get_attack_tactics(self, matrix):
-        objects = self._retrieve_objects(urljoin(self.api_root, f"/api/v1/attack-{matrix}/objects/?type=x-mitre-tactic"), 'objects')
+        objects = self._retrieve_objects(urljoin(self.api_root, f"v1/attack-{matrix}/objects/?type=x-mitre-tactic"), 'objects')
         objects_map : dict[str, list[dict]] = {}
         for obj in objects:
             name = obj['x_mitre_shortname']
             objects_map.setdefault(name, []).append(obj)
         return objects_map
 
     def get_vulnerabilities(self, cve_ids):
-        return self.get_objects_by_external_ids(cve_ids, 'cve', 'vulnerabilities', 'cve_id')
+        return self.get_objects_by_external_ids(cve_ids, 'cve', 'objects', 'cve_id')
 
     def _retrieve_objects(self, endpoint, key='objects'):
         s = requests.Session()
         s.headers.update({
-            "Authority": f"Bearer {self.api_key}"
+            "API-KEY": self.api_key
         })
         data = []
         page = 1

src/utils.py

@@ -25,21 +25,21 @@ def check_dir(dir:str):
 
 
 
-def delete_files_and_folders_except_rules(prefix='rules', keep_count=2):
+def delete_files_and_folders_except_rules(prefixes=['rules', 'deprecated'], keep_count=20):
     directory_path = 'data'
     all_items = os.listdir(directory_path)
-    rules_folders = [item for item in all_items if item.startswith(prefix)]
-    rules_folders.sort()
-    folders_to_keep = rules_folders[:keep_count]
+
+    rules_folders = []
+
     for item in all_items:
-        item_path = os.path.join(directory_path, item)
-        if os.path.isdir(item_path) and not item.startswith(prefix):
-            if item not in folders_to_keep:
-                shutil.rmtree(item_path)
-                print(f"Deleted: {item_path}")
-        elif os.path.isfile(item_path):
-            if not item.startswith(prefix):
-                os.remove(item_path)
+        for prefix in prefixes:
+            if item.startswith(prefix):
+                rules_folders.append(item)
+                break
+        else:
+            item_path = os.path.join(directory_path, item)
+            shutil.rmtree(item_path, ignore_errors=True)
+            print(f"Deleted: {item_path}")
 
 
 def get_all_yaml_files(folder="data"):

tests/demo_rule.yml

@@ -11,6 +11,7 @@ tags:
     - attack.privilege_escalation
     - attack.initial_access
     - attack.t1114
+    - attack.T1190
     - detection.emerging_threats
     - cve.2022.21722
 logsource:

utilities/backfill_all.sh

@@ -15,4 +15,5 @@ python3 sigma2stix.py --mode sigmahq --sigma_version_tag r2024-04-29 && mv stix2
 python3 sigma2stix.py --mode sigmahq --sigma_version_tag r2024-05-13 && mv stix2_objects/sigma-rule-bundle.json bundle_store/sigma-rule-bundle-r2024-05-13.json && \
 python3 sigma2stix.py --mode sigmahq --sigma_version_tag r2024-07-17 && mv stix2_objects/sigma-rule-bundle.json bundle_store/sigma-rule-bundle-r2024-07-17.json && \
 python3 sigma2stix.py --mode sigmahq --sigma_version_tag r2024-09-02 && mv stix2_objects/sigma-rule-bundle.json bundle_store/sigma-rule-bundle-r2024-09-02.json && \
-python3 sigma2stix.py --mode sigmahq --sigma_version_tag r2024-11-10 && mv stix2_objects/sigma-rule-bundle.json bundle_store/sigma-rule-bundle-r2024-11-10.json
+python3 sigma2stix.py --mode sigmahq --sigma_version_tag r2024-11-10 && mv stix2_objects/sigma-rule-bundle.json bundle_store/sigma-rule-bundle-r2024-11-10.json && \
+python3 sigma2stix.py --mode sigmahq --sigma_version_tag r2024-12-19 && mv stix2_objects/sigma-rule-bundle.json bundle_store/sigma-rule-bundle-r2024-12-19.json