x-pack/filebeat/module/threatintel/misp: add support for secondary ob…

…ject attribute handling (elastic#28124)

Co-authored-by: Marius Iversen <[email protected]>

CHANGELOG.next.asciidoc

@@ -744,6 +744,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Make aws-cloudwatch input GA. {pull}28161[28161]
 - Move processing to ingest node for AWS vpcflow fileset. {pull}28168[28168]
 - Release zoom module as GA. {pull}28106[28106]
+- Add support for secondary object attribute handling in ThreatIntel MISP module {pull}28124[28124]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -151364,6 +151364,156 @@ type: keyword
 The value of the attribute, depending on the type like "url, sha1, email-src".
 
 
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.id`*::
++
+--
+The ID of the secondary attribute related to the event object.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.type`*::
++
+--
+The type of the secondary attribute related to the event object. For example email, ipv4, sha1 and such.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.category`*::
++
+--
+The category of the secondary attribute related to the event object. For example "Network Activity".
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.to_ids`*::
++
+--
+If the secondary attribute should be automatically synced with an IDS.
+
+
+type: boolean
+
+--
+
+*`threatintel.misp.context.attribute.uuid`*::
++
+--
+The UUID of the secondary attribute related to the event.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.event_id`*::
++
+--
+The local event ID of the secondary attribute related to the event.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.distribution`*::
++
+--
+How the secondary attribute has been distributed, represented by integer numbers.
+
+
+type: long
+
+--
+
+*`threatintel.misp.context.attribute.timestamp`*::
++
+--
+The timestamp in which the secondary attribute was attached to the event object.
+
+
+type: date
+
+--
+
+*`threatintel.misp.context.attribute.comment`*::
++
+--
+Comments made to the secondary attribute itself.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.sharing_group_id`*::
++
+--
+The group ID of the sharing group related to the specific secondary attribute.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.deleted`*::
++
+--
+If the secondary attribute has been removed from the event object.
+
+
+type: boolean
+
+--
+
+*`threatintel.misp.context.attribute.disable_correlation`*::
++
+--
+If correlation has been enabled on the secondary attribute related to the event object.
+
+
+type: boolean
+
+--
+
+*`threatintel.misp.context.attribute.object_id`*::
++
+--
+The ID of the Object in which the secondary attribute is attached.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.object_relation`*::
++
+--
+The type of relation the secondary attribute has with the event object itself.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.value`*::
++
+--
+The value of the attribute, depending on the type like "url, sha1, email-src".
+
+
 type: keyword
 
 --

x-pack/filebeat/module/threatintel/misp/_meta/fields.yml

@@ -163,3 +163,63 @@
     type: keyword
     description: >
       The value of the attribute, depending on the type like "url, sha1, email-src".
+  - name: context.attribute.id
+    type: keyword
+    description: >
+      The ID of the secondary attribute related to the event object.
+  - name: context.attribute.type
+    type: keyword
+    description: >
+      The type of the secondary attribute related to the event object. For example email, ipv4, sha1 and such.
+  - name: context.attribute.category
+    type: keyword
+    description: >
+      The category of the secondary attribute related to the event object. For example "Network Activity".
+  - name: context.attribute.to_ids
+    type: boolean
+    description: >
+      If the secondary attribute should be automatically synced with an IDS.
+  - name: context.attribute.uuid
+    type: keyword
+    description: >
+      The UUID of the secondary attribute related to the event.
+  - name: context.attribute.event_id
+    type: keyword
+    description: >
+      The local event ID of the secondary attribute related to the event.
+  - name: context.attribute.distribution
+    type: long
+    description: >
+      How the secondary attribute has been distributed, represented by integer numbers.
+  - name: context.attribute.timestamp
+    type: date
+    description: >
+      The timestamp in which the secondary attribute was attached to the event object.
+  - name: context.attribute.comment
+    type: keyword
+    description: >
+      Comments made to the secondary attribute itself.
+  - name: context.attribute.sharing_group_id
+    type: keyword
+    description: >
+      The group ID of the sharing group related to the specific secondary attribute.
+  - name: context.attribute.deleted
+    type: boolean
+    description: >
+      If the secondary attribute has been removed from the event object.
+  - name: context.attribute.disable_correlation
+    type: boolean
+    description: >
+      If correlation has been enabled on the secondary attribute related to the event object.
+  - name: context.attribute.object_id
+    type: keyword
+    description: >
+      The ID of the Object in which the secondary attribute is attached.
+  - name: context.attribute.object_relation
+    type: keyword
+    description: >
+      The type of relation the secondary attribute has with the event object itself.
+  - name: context.attribute.value
+    type: keyword
+    description: >
+      The value of the attribute, depending on the type like "url, sha1, email-src".

x-pack/filebeat/module/threatintel/misp/config/config.yml

@@ -33,12 +33,20 @@ response.split:
   target: body.response
   split:
     target: body.Event.Attribute
+    ignore_empty_value: true
     keep_parent: true
+    split:
+      target: body.Event.Object
+      keep_parent: true
+      split:
+        target: body.Event.Object.Attribute
+        keep_parent: true
 response.request_body_on_pagination: true
 response.pagination:
 - set:
     target: body.page
     value: '[[add .last_response.page 1]]'
+    fail_on_template_error: true
 cursor:
   timestamp:
     value: '[[.last_event.Event.timestamp]]'
@@ -61,8 +69,14 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 processors:
   - decode_json_fields:
       fields: [message]
-      document_id: Event.Attribute.uuid
       target: json
+  - fingerprint:
+      fields:
+        - json.Event.Attribute.uuid
+        - json.Event.Object.Attribute.uuid
+      ignore_missing: true
+      target_field: "@metadata._id"
+      encoding: base64
   - script:
       lang: javascript
       id: my_filter

x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml

@@ -41,10 +41,21 @@ processors:
         - threatintel.misp.Galaxy
         - threatintel.misp.Attribute.Galaxy
         - threatintel.misp.Attribute.ShadowAttribute
-        - threatintel.misp.Object
         - threatintel.misp.EventReport
+        - threatintel.misp.Object.Attribute.Galaxy
+        - threatintel.misp.Object.Attribute.ShadowAttribute
         - message
       ignore_missing: true
+  - remove:
+      field:
+        - threatintel.misp.Attribute
+      ignore_missing: true
+      if: ctx?.threatintel?.misp?.Attribute.size() == 0
+  - remove:
+      field:
+        - threatintel.misp.Object
+      ignore_missing: true
+      if: ctx?.threatintel?.misp?.Object.size() == 0
   - date:
       field: threatintel.misp.timestamp
       formats:
@@ -54,6 +65,14 @@ processors:
       field: threatintel.misp.Attribute
       target_field: threatintel.misp.attribute
       ignore_missing: true
+  - rename:
+      field: threatintel.misp.Object
+      target_field: threatintel.misp.object
+      ignore_missing: true
+  - rename:
+      field: threatintel.misp.object.Attribute
+      target_field: threatintel.misp.object.attribute
+      ignore_missing: true
   - rename:
       field: threatintel.misp.Orgc
       target_field: threatintel.misp.orgc
@@ -62,15 +81,25 @@ processors:
       field: threatintel.misp.Org
       target_field: threatintel.misp.org
       ignore_missing: true
-  - rename:
-      field: threatintel.misp.Attribute
-      target_field: threatintel.misp.attribute
-      ignore_missing: true
   - rename:
       field: threatintel.misp.Tag
       target_field: threatintel.misp.tag
       ignore_missing: true
 
+  # # Dance around issue of not being able to split the document into two.
+  # # Make the Object.Attribute field primary if it exists, but keep the
+  # # outer Attribute as context.
+  - rename:
+      field: threatintel.misp.attribute
+      target_field: threatintel.misp.context.attribute
+      ignore_missing: true
+      if: ctx?.threatintel?.misp?.object != null
+  - rename:
+      field: threatintel.misp.object.attribute
+      target_field: threatintel.misp.attribute
+      ignore_missing: true
+      if: ctx?.threatintel?.misp?.object != null
+
   #####################
   # Threat ECS Fields #
   #####################
@@ -96,7 +125,7 @@ processors:
   - set:
       field: threatintel.indicator.type
       value: file
-      if: "['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.threatintel?.misp?.attribute?.type) || ctx.threatintel?.misp?.attribute?.type.startsWith('filename')"
+      if: "ctx?.threatintel?.misp?.attribute?.type != null && (['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.threatintel?.misp?.attribute?.type) || ctx.threatintel?.misp?.attribute?.type.startsWith('filename'))"
   - rename:
       field: threatintel.misp.attribute.value
       target_field: "threatintel.indicator.file.hash.{{threatintel.misp.attribute.type}}"
@@ -112,17 +141,17 @@ processors:
       patterns:
         - "%{WORD}\\|%{WORD:_tmp.hashtype}"
       ignore_missing: true
-      if: ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|')
+      if: ctx?.threatintel?.misp?.attribute?.type != null && ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|')
   - grok:
       field: threatintel.misp.attribute.value
       patterns:
         - "%{DATA:threatintel.indicator.file.name}\\|%{GREEDYDATA:_tmp.hashvalue}"
       ignore_missing: true
-      if: ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|')
+      if: ctx?.threatintel?.misp?.attribute?.type != null && ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|')
   - set:
       field: threatintel.indicator.file.hash.{{_tmp.hashtype}}
       value: "{{_tmp.hashvalue}}"
-      if: "ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|') && ctx?._tmp?.hashvalue != null && ctx?._tmp?.hashtype != null"
+      if: "ctx?.threatintel?.misp?.attribute?.type != null && ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|') && ctx?._tmp?.hashvalue != null && ctx?._tmp?.hashtype != null"
 
   ## URL/URI indicator operations
   - set:
@@ -218,6 +247,14 @@ processors:
       target_field: threatintel.indicator.email.address
       ignore_missing: true
       if: ctx?.threatintel?.indicator?.type == 'email-addr'
+  - rename:
+      field: threatintel.misp.event_creator_email
+      target_field: user.email
+      ignore_missing: true
+  - append:
+      field: user.roles
+      value: "reporting_user"
+      if: ctx?.user?.email != null
 
   ## MAC Address indicator operations
   - set:
@@ -288,6 +325,14 @@ processors:
         - threatintel.misp.attribute.value
       ignore_missing: true
       if: ctx?.threatintel?.indicator?.type != 'unknown'
+  - remove:
+      field:
+        # This removes a number of fields that may be wanted in the future when
+        # threatintel.misp.attribute and threatintel.misp.object.attribute can
+        # be separated. At the root of .object are fields that mirror fields at
+        # the root of threatintel.misp.
+        - threatintel.misp.object
+      ignore_missing: true
   - remove:
       field:
         - threatintel.misp.Attribute.timestamp