MISP improvements: (#23070)

- Add id based on body uuid - Add pagination and cursor - Migrate to httpjson v2
elastic · Dec 14, 2020 · e0d654a · e0d654a
1 parent 9db963e
commit e0d654a
Show file tree

Hide file tree

Showing 11 changed files with 70 additions and 64 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -768,6 +768,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add the ability to override `network.direction` based on interfaces in Fortinet/firewall fileset. {pull}23072[23072]
 - Add `network.direction` override by specifying `internal_networks` in gcp module. {pull}23081[23081]
 - Migrate okta to httpjson v2 config {pull}23059[23059]
+- Misp improvements: Migration to httpjson v2 config, pagination and deduplication ID {pull}23070[23070]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/modules/misp.asciidoc b/filebeat/docs/modules/misp.asciidoc
@@ -17,9 +17,9 @@ This is a filebeat module for reading threat intel information from the MISP pla
 The configuration in the config.yml file uses the following format:
 
  * var.api_key: specifies the API key to access MISP.
- * var.json_objects_array: specifies the array object in MISP response, e.g., "response.Attribute".
+ * var.http_request_body: an object containing any parameter that needs to be sent to the search API. Default: `limit: 1000`
  * var.url: URL of the MISP REST API, e.g., "http://x.x.x.x/attributes/restSearch"
- 
+
 include::../include/gs-link.asciidoc[]
 
 [float]

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -1340,16 +1340,13 @@ filebeat.modules:
     #var.api_key
 
     # Array object in MISP response
-    #var.json_objects_array
+    #var.http_request_body.limit: 1000
 
     # URL of the MISP REST API
     #var.url
-   
+
     # You can also pass SSL options. For example:
-    #var.ssl: |-
-    #  {
-    #    verification_mode: none
-    #  }
+    #var.ssl.verification_mode: none
 
 #------------------------------- Mongodb Module -------------------------------
 #- module: mongodb

diff --git a/x-pack/filebeat/module/misp/_meta/config.yml b/x-pack/filebeat/module/misp/_meta/config.yml
@@ -5,13 +5,10 @@
     #var.api_key
 
     # Array object in MISP response
-    #var.json_objects_array
+    #var.http_request_body.limit: 1000
 
     # URL of the MISP REST API
     #var.url
-   
+
     # You can also pass SSL options. For example:
-    #var.ssl: |-
-    #  {
-    #    verification_mode: none
-    #  }
+    #var.ssl.verification_mode: none
diff --git a/x-pack/filebeat/module/misp/_meta/docs.asciidoc b/x-pack/filebeat/module/misp/_meta/docs.asciidoc
@@ -12,9 +12,9 @@ This is a filebeat module for reading threat intel information from the MISP pla
 The configuration in the config.yml file uses the following format:
 
  * var.api_key: specifies the API key to access MISP.
- * var.json_objects_array: specifies the array object in MISP response, e.g., "response.Attribute".
+ * var.http_request_body: an object containing any parameter that needs to be sent to the search API. Default: `limit: 1000`
  * var.url: URL of the MISP REST API, e.g., "http://x.x.x.x/attributes/restSearch"
- 
+
 include::../include/gs-link.asciidoc[]
 
 [float]

diff --git a/x-pack/filebeat/module/misp/threat/config/input.yml b/x-pack/filebeat/module/misp/threat/config/input.yml
@@ -1,16 +1,35 @@
 {{ if eq .input "httpjson" }}
 
 type: httpjson
-api_key: {{ .api_key }}
-http_client_timeout: {{ .http_client_timeout }}
-http_method: {{ .http_method }}
-http_headers: {{ .http_headers }}
-http_request_body: {{ .http_request_body }}
+config_version: "2"
 interval: {{ .interval }}
-json_objects_array: {{ .json_objects_array }}
-pagination: {{ .pagination }}
-url: {{ .url }}
-ssl: {{ .ssl }}
+
+request.method: POST
+request.ssl: {{ .ssl | tojson }}
+request.url: {{ .url }}
+request.timeout: {{ .http_client_timeout }}
+request.body: {{ .http_request_body | tojson }}
+request.body.returnFormat: json
+request.transforms:
+  - set:
+      target: header.Authorization
+      value: {{.api_key}}
+  - set:
+      target: body.timestamp
+      value: "[[.cursor.timestamp]]"
+
+response.split:
+  target: body.response.Attribute
+
+response.request_body_on_pagination: true
+response.pagination:
+  - set:
+      target: body.page
+      value: "[[add .last_response.page 1]]"
+
+cursor:
+  timestamp:
+    value: "[[.last_event.timestamp]]"
 
 {{ else if eq .input "file" }}
 

diff --git a/x-pack/filebeat/module/misp/threat/config/pipeline.js b/x-pack/filebeat/module/misp/threat/config/pipeline.js
@@ -5,11 +5,19 @@
 var threat = (function () {
     var processor = require("processor");
 
+    var copyToOriginal = function (evt) {
+        evt.Put("event.original", evt.Get("message"));
+    };
+
     var decodeJson = new processor.DecodeJSONFields({
         fields: ["message"],
         target: "json",
     });
 
+    var setID = function (evt) {
+        evt.Put("@metadata._id", evt.Get("event.id"));
+    };
+
     var categorizeEvent = new processor.AddFields({
         target: "event",
         fields: {
@@ -73,11 +81,11 @@ var threat = (function () {
                 break;
             case 'btc':
                 attackPattern = '[' + 'bitcoin:address = ' + '\'' + v + '\'' + ']';
-                attackPatternKQL = 'bitcoin.address: ' + '"' + v + '"'; 
+                attackPatternKQL = 'bitcoin.address: ' + '"' + v + '"';
                 break;
             case "domain":
                 attackPattern = '[' + 'dns:question:name = ' + '\'' + v + '\'' + ' OR url:domain = ' + '\'' + v + '\'' + ' OR source:domain = ' + '\'' + v + '\'' + ' OR destination:domain = ' + '\'' + v + '\'' + ']';
-                attackPatternKQL = 'dns.question.name: ' + '"' + v + '"' + ' OR url.domain: ' + '"' + v + '"' + ' OR source.domain: ' + '"' + v + '"' + ' OR destination.domain: ' + '"' + v + '"'; 
+                attackPatternKQL = 'dns.question.name: ' + '"' + v + '"' + ' OR url.domain: ' + '"' + v + '"' + ' OR source.domain: ' + '"' + v + '"' + ' OR destination.domain: ' + '"' + v + '"';
                 break;
             case "domain|ip":
                 arr = v.split("|");
@@ -186,7 +194,7 @@ var threat = (function () {
                 attackPattern = '[' + 'file:hash:sha256 = ' + '\'' + v + '\'' + ']';
                 attackPatternKQL = 'file.hash.sha256: ' + '"' + v + '"';
                 evt.Put("file.hash.sha256", v);
-                break;            
+                break;
             case "sha512":
                 attackPattern = '[' + 'file:hash:sha512 = ' + '\'' + v + '\'' + ']';
                 attackPatternKQL = 'file.hash.sha512: ' + '"' + v + '"';
@@ -200,7 +208,7 @@ var threat = (function () {
             case 'yara':
                 attackPattern = '[' + 'yara:rule = ' + '\'' + v + '\'' + ']';
                 attackPatternKQL = 'yara.rule: ' + '"' + v + '"';
-                break; 
+                break;
         }
         if (attackPattern == undefined || attackPatternKQL == undefined) {
             evt.Put("error.message", 'Unsupported type: ' + indicator_type);
@@ -210,10 +218,12 @@ var threat = (function () {
     };
 
     var pipeline = new processor.Chain()
+        .Add(copyToOriginal)
         .Add(decodeJson)
         .Add(categorizeEvent)
         .Add(setThreatFeedField)
         .Add(convertFields)
+        .Add(setID)
         .Add(setAttackPattern)
         .Add(copyTags)
         .Build();

diff --git a/x-pack/filebeat/module/misp/threat/manifest.yml b/x-pack/filebeat/module/misp/threat/manifest.yml
@@ -4,35 +4,15 @@ var:
   - name: input
     default: httpjson
   - name: api_key
-    default: ""
-  - name: http_client_timeout
-    default: 60
-  - name: http_method
-    default: GET
-  - name: http_headers
-    default: |-
-      {}
   - name: http_request_body
-    default: |-
-      {}
+    default:
+      limit: 1000
+  - name: http_client_timeout
+    default: "60s"
   - name: interval
-    default: 0
-  - name: json_objects_array
-    default: "response.Attribute"
-  - name: pagination
-    default: |-
-      {
-        "enabled": false,
-        "extra_body_content": {},
-        "id_field": "",
-        "req_field": "",
-        "url": ""
-      }
+    default: "60s"
   - name: url
-    default: ""
   - name: ssl
-    default: |-
-      {}
 
 input: config/input.yml
 ingest_pipeline: ingest/pipeline.json

diff --git a/x-pack/filebeat/module/misp/threat/test/misp-test.json.log b/x-pack/filebeat/module/misp/threat/test/misp-test.json.log
@@ -1,5 +1,5 @@
 {"id":"1","event_id":"1","object_id":"0","object_relation":null,"category":"Network activity","type":"ip-dst","to_ids":false,"uuid":"5d2cb906-eff4-40f0-9f1d-10eb7d6a0c26","timestamp":"1490878466","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"value":"98.235.162.24","Event":{"org_id":"1","distribution":"3","id":"1","info":"Tor exit nodes feed","orgc_id":"2","uuid":"58dcfe62-ed84-4e5e-b293-4991950d210f"}}
 {"id":"2","event_id":"2","object_id":"0","object_relation":null,"category":"Payload delivery","type":"md5","to_ids":true,"uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b81","timestamp":"1490878466","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"value":"89357a1b2e32f2b9bddff94b8136810b","Event":{"org_id":"1","distribution":"3","id":"1","info":"OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass","orgc_id":"2","uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b81"}}
-{"id":"3","event_id":"3","object_id":"0","object_relation":null,"category":"Payload delivery","type":"filename","to_ids":true,"uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b81","timestamp":"1490878466","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"value":"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de","Event":{"org_id":"1","distribution":"3","id":"1","info":"OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass","orgc_id":"2","uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b81"}}
+{"id":"3","event_id":"3","object_id":"0","object_relation":null,"category":"Payload delivery","type":"filename","to_ids":true,"uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b82","timestamp":"1490878466","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"value":"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de","Event":{"org_id":"1","distribution":"3","id":"1","info":"OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass","orgc_id":"2","uuid":"5d159be2-d4b4-4d97-9e14-406a02de0b82"}}
 {"id":"4","event_id":"4","object_id":"0","object_relation":null,"category":"Bad Domain","type":"domain","to_ids":true,"uuid":"563b3ea6-b26c-401f-a68b-4d84950d210b","timestamp":"1490878466","distribution":"5","sharing_group_id":"0","comment":"","deleted":false,"disable_correlation":false,"value":"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de","Event":{"org_id":"4","distribution":"3","id":"4","info":"OSINT Expansion on Systematic cyber attacks against Israeli and Palestinian targets going on for a year by Norman","orgc_id":"2","uuid":"563b3ea6-b26c-401f-a68b-4d84950d210b"}}
 {"Galaxy":[],"ShadowAttribute":[],"Tag":[{"colour":"#eb2323","exportable":true,"hide_tag":false,"id":"966","local":0,"name":"critical-ioc-quarantine","numerical_value":null,"user_id":"1"}],"category":"Network activity","comment":"","deleted":false,"disable_correlation":false,"distribution":"5","event_id":"693","first_seen":null,"id":"170814","last_seen":null,"object_id":"0","object_relation":null,"sharing_group_id":"0","timestamp":"1602078120","to_ids":false,"type":"url","uuid":"83078f8c-3d38-4fec-87a1-8be22a0be0a6","value":"endgame.hungmnguyen.us"}
diff --git a/x-pack/filebeat/module/misp/threat/test/misp-test.json.log-expected.json b/x-pack/filebeat/module/misp/threat/test/misp-test.json.log-expected.json
@@ -15,6 +15,7 @@
         "event.id": "5d2cb906-eff4-40f0-9f1d-10eb7d6a0c26",
         "event.kind": "event",
         "event.module": "misp",
+        "event.original": "{\"id\":\"1\",\"event_id\":\"1\",\"object_id\":\"0\",\"object_relation\":null,\"category\":\"Network activity\",\"type\":\"ip-dst\",\"to_ids\":false,\"uuid\":\"5d2cb906-eff4-40f0-9f1d-10eb7d6a0c26\",\"timestamp\":\"1490878466\",\"distribution\":\"5\",\"sharing_group_id\":\"0\",\"comment\":\"\",\"deleted\":false,\"disable_correlation\":false,\"value\":\"98.235.162.24\",\"Event\":{\"org_id\":\"1\",\"distribution\":\"3\",\"id\":\"1\",\"info\":\"Tor exit nodes feed\",\"orgc_id\":\"2\",\"uuid\":\"58dcfe62-ed84-4e5e-b293-4991950d210f\"}}",
         "event.type": "indicator",
         "fileset.name": "threat",
         "input.type": "log",
@@ -39,6 +40,7 @@
         "event.id": "5d159be2-d4b4-4d97-9e14-406a02de0b81",
         "event.kind": "event",
         "event.module": "misp",
+        "event.original": "{\"id\":\"2\",\"event_id\":\"2\",\"object_id\":\"0\",\"object_relation\":null,\"category\":\"Payload delivery\",\"type\":\"md5\",\"to_ids\":true,\"uuid\":\"5d159be2-d4b4-4d97-9e14-406a02de0b81\",\"timestamp\":\"1490878466\",\"distribution\":\"5\",\"sharing_group_id\":\"0\",\"comment\":\"\",\"deleted\":false,\"disable_correlation\":false,\"value\":\"89357a1b2e32f2b9bddff94b8136810b\",\"Event\":{\"org_id\":\"1\",\"distribution\":\"3\",\"id\":\"1\",\"info\":\"OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass\",\"orgc_id\":\"2\",\"uuid\":\"5d159be2-d4b4-4d97-9e14-406a02de0b81\"}}",
         "event.type": "indicator",
         "file.hash.md5": "89357a1b2e32f2b9bddff94b8136810b",
         "fileset.name": "threat",
@@ -61,9 +63,10 @@
         "@timestamp": "2017-03-30T12:54:26.000Z",
         "event.category": "threat-intel",
         "event.dataset": "misp.threat",
-        "event.id": "5d159be2-d4b4-4d97-9e14-406a02de0b81",
+        "event.id": "5d159be2-d4b4-4d97-9e14-406a02de0b82",
         "event.kind": "event",
         "event.module": "misp",
+        "event.original": "{\"id\":\"3\",\"event_id\":\"3\",\"object_id\":\"0\",\"object_relation\":null,\"category\":\"Payload delivery\",\"type\":\"filename\",\"to_ids\":true,\"uuid\":\"5d159be2-d4b4-4d97-9e14-406a02de0b82\",\"timestamp\":\"1490878466\",\"distribution\":\"5\",\"sharing_group_id\":\"0\",\"comment\":\"\",\"deleted\":false,\"disable_correlation\":false,\"value\":\"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de\",\"Event\":{\"org_id\":\"1\",\"distribution\":\"3\",\"id\":\"1\",\"info\":\"OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass\",\"orgc_id\":\"2\",\"uuid\":\"5d159be2-d4b4-4d97-9e14-406a02de0b82\"}}",
         "event.type": "indicator",
         "file.path": "f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de",
         "fileset.name": "threat",
@@ -74,12 +77,12 @@
         "misp.threat_indicator.attack_pattern_kql": "file.path: \"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de\"",
         "misp.threat_indicator.description": "OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass",
         "misp.threat_indicator.feed": "misp",
-        "misp.threat_indicator.id": "5d159be2-d4b4-4d97-9e14-406a02de0b81",
+        "misp.threat_indicator.id": "5d159be2-d4b4-4d97-9e14-406a02de0b82",
         "misp.threat_indicator.type": "filename",
         "rule.category": "Payload delivery",
         "rule.description": "OSINT - OSX/Linker: New Mac malware attempts zero-day Gatekeeper bypass",
         "rule.id": "1",
-        "rule.uuid": "5d159be2-d4b4-4d97-9e14-406a02de0b81",
+        "rule.uuid": "5d159be2-d4b4-4d97-9e14-406a02de0b82",
         "service.type": "misp"
     },
     {
@@ -89,6 +92,7 @@
         "event.id": "563b3ea6-b26c-401f-a68b-4d84950d210b",
         "event.kind": "event",
         "event.module": "misp",
+        "event.original": "{\"id\":\"4\",\"event_id\":\"4\",\"object_id\":\"0\",\"object_relation\":null,\"category\":\"Bad Domain\",\"type\":\"domain\",\"to_ids\":true,\"uuid\":\"563b3ea6-b26c-401f-a68b-4d84950d210b\",\"timestamp\":\"1490878466\",\"distribution\":\"5\",\"sharing_group_id\":\"0\",\"comment\":\"\",\"deleted\":false,\"disable_correlation\":false,\"value\":\"f6bf5b8bb2400aad4ac844f2b94a4e556907f35b44c5ff462fb4e70c0208c9de\",\"Event\":{\"org_id\":\"4\",\"distribution\":\"3\",\"id\":\"4\",\"info\":\"OSINT Expansion on Systematic cyber attacks against Israeli and Palestinian targets going on for a year by Norman\",\"orgc_id\":\"2\",\"uuid\":\"563b3ea6-b26c-401f-a68b-4d84950d210b\"}}",
         "event.type": "indicator",
         "fileset.name": "threat",
         "input.type": "log",
@@ -113,6 +117,7 @@
         "event.id": "83078f8c-3d38-4fec-87a1-8be22a0be0a6",
         "event.kind": "event",
         "event.module": "misp",
+        "event.original": "{\"Galaxy\":[],\"ShadowAttribute\":[],\"Tag\":[{\"colour\":\"#eb2323\",\"exportable\":true,\"hide_tag\":false,\"id\":\"966\",\"local\":0,\"name\":\"critical-ioc-quarantine\",\"numerical_value\":null,\"user_id\":\"1\"}],\"category\":\"Network activity\",\"comment\":\"\",\"deleted\":false,\"disable_correlation\":false,\"distribution\":\"5\",\"event_id\":\"693\",\"first_seen\":null,\"id\":\"170814\",\"last_seen\":null,\"object_id\":\"0\",\"object_relation\":null,\"sharing_group_id\":\"0\",\"timestamp\":\"1602078120\",\"to_ids\":false,\"type\":\"url\",\"uuid\":\"83078f8c-3d38-4fec-87a1-8be22a0be0a6\",\"value\":\"endgame.hungmnguyen.us\"}",
         "event.type": "indicator",
         "fileset.name": "threat",
         "input.type": "log",

diff --git a/x-pack/filebeat/modules.d/misp.yml.disabled b/x-pack/filebeat/modules.d/misp.yml.disabled
@@ -8,13 +8,10 @@
     #var.api_key
 
     # Array object in MISP response
-    #var.json_objects_array
+    #var.http_request_body.limit: 1000
 
     # URL of the MISP REST API
     #var.url
-   
+
     # You can also pass SSL options. For example:
-    #var.ssl: |-
-    #  {
-    #    verification_mode: none
-    #  }
+    #var.ssl.verification_mode: none