x-pack/filebeat/module/threatintel/misp: add support for secondary ob…

…ject attribute handling (#28124)

Co-authored-by: Marius Iversen <[email protected]>

CHANGELOG.next.asciidoc

@@ -744,6 +744,7 @@ https:/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Make aws-cloudwatch input GA. {pull}28161[28161]
 - Move processing to ingest node for AWS vpcflow fileset. {pull}28168[28168]
 - Release zoom module as GA. {pull}28106[28106]
+- Add support for secondary object attribute handling in ThreatIntel MISP module {pull}28124[28124]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -151364,6 +151364,156 @@ type: keyword
 The value of the attribute, depending on the type like "url, sha1, email-src".
 
 
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.id`*::
++
+--
+The ID of the secondary attribute related to the event object.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.type`*::
++
+--
+The type of the secondary attribute related to the event object. For example email, ipv4, sha1 and such.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.category`*::
++
+--
+The category of the secondary attribute related to the event object. For example "Network Activity".
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.to_ids`*::
++
+--
+If the secondary attribute should be automatically synced with an IDS.
+
+
+type: boolean
+
+--
+
+*`threatintel.misp.context.attribute.uuid`*::
++
+--
+The UUID of the secondary attribute related to the event.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.event_id`*::
++
+--
+The local event ID of the secondary attribute related to the event.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.distribution`*::
++
+--
+How the secondary attribute has been distributed, represented by integer numbers.
+
+
+type: long
+
+--
+
+*`threatintel.misp.context.attribute.timestamp`*::
++
+--
+The timestamp in which the secondary attribute was attached to the event object.
+
+
+type: date
+
+--
+
+*`threatintel.misp.context.attribute.comment`*::
++
+--
+Comments made to the secondary attribute itself.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.sharing_group_id`*::
++
+--
+The group ID of the sharing group related to the specific secondary attribute.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.deleted`*::
++
+--
+If the secondary attribute has been removed from the event object.
+
+
+type: boolean
+
+--
+
+*`threatintel.misp.context.attribute.disable_correlation`*::
++
+--
+If correlation has been enabled on the secondary attribute related to the event object.
+
+
+type: boolean
+
+--
+
+*`threatintel.misp.context.attribute.object_id`*::
++
+--
+The ID of the Object in which the secondary attribute is attached.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.object_relation`*::
++
+--
+The type of relation the secondary attribute has with the event object itself.
+
+
+type: keyword
+
+--
+
+*`threatintel.misp.context.attribute.value`*::
++
+--
+The value of the attribute, depending on the type like "url, sha1, email-src".
+
+
 type: keyword
 
 --

x-pack/filebeat/module/threatintel/misp/_meta/fields.yml

@@ -163,3 +163,63 @@
  type: keyword
  description: >
  The value of the attribute, depending on the type like "url, sha1, email-src".
+ - name: context.attribute.id
+ type: keyword
+ description: >
+ The ID of the secondary attribute related to the event object.
+ - name: context.attribute.type
+ type: keyword
+ description: >
+ The type of the secondary attribute related to the event object. For example email, ipv4, sha1 and such.
+ - name: context.attribute.category
+ type: keyword
+ description: >
+ The category of the secondary attribute related to the event object. For example "Network Activity".
+ - name: context.attribute.to_ids
+ type: boolean
+ description: >
+ If the secondary attribute should be automatically synced with an IDS.
+ - name: context.attribute.uuid
+ type: keyword
+ description: >
+ The UUID of the secondary attribute related to the event.
+ - name: context.attribute.event_id
+ type: keyword
+ description: >
+ The local event ID of the secondary attribute related to the event.
+ - name: context.attribute.distribution
+ type: long
+ description: >
+ How the secondary attribute has been distributed, represented by integer numbers.
+ - name: context.attribute.timestamp
+ type: date
+ description: >
+ The timestamp in which the secondary attribute was attached to the event object.
+ - name: context.attribute.comment
+ type: keyword
+ description: >
+ Comments made to the secondary attribute itself.
+ - name: context.attribute.sharing_group_id
+ type: keyword
+ description: >
+ The group ID of the sharing group related to the specific secondary attribute.
+ - name: context.attribute.deleted
+ type: boolean
+ description: >
+ If the secondary attribute has been removed from the event object.
+ - name: context.attribute.disable_correlation
+ type: boolean
+ description: >
+ If correlation has been enabled on the secondary attribute related to the event object.
+ - name: context.attribute.object_id
+ type: keyword
+ description: >
+ The ID of the Object in which the secondary attribute is attached.
+ - name: context.attribute.object_relation
+ type: keyword
+ description: >
+ The type of relation the secondary attribute has with the event object itself.
+ - name: context.attribute.value
+ type: keyword
+ description: >
+ The value of the attribute, depending on the type like "url, sha1, email-src".

x-pack/filebeat/module/threatintel/misp/config/config.yml

@@ -33,12 +33,20 @@ response.split:
  target: body.response
  split:
  target: body.Event.Attribute
+ ignore_empty_value: true
  keep_parent: true
+ split:
+ target: body.Event.Object
+ keep_parent: true
+ split:
+ target: body.Event.Object.Attribute
+ keep_parent: true
 response.request_body_on_pagination: true
 response.pagination:
 - set:
  target: body.page
  value: '[[add .last_response.page 1]]'
+ fail_on_template_error: true
 cursor:
  timestamp:
  value: '[[.last_event.Event.timestamp]]'
@@ -61,8 +69,14 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 processors:
  - decode_json_fields:
  fields: [message]
- document_id: Event.Attribute.uuid
  target: json
+ - fingerprint:
+ fields:
+ - json.Event.Attribute.uuid
+ - json.Event.Object.Attribute.uuid
+ ignore_missing: true
+ target_field: "@metadata._id"
+ encoding: base64
  - script:
  lang: javascript
  id: my_filter

x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml

@@ -41,10 +41,21 @@ processors:
  - threatintel.misp.Galaxy
  - threatintel.misp.Attribute.Galaxy
  - threatintel.misp.Attribute.ShadowAttribute
- - threatintel.misp.Object
  - threatintel.misp.EventReport
+ - threatintel.misp.Object.Attribute.Galaxy
+ - threatintel.misp.Object.Attribute.ShadowAttribute
  - message
  ignore_missing: true
+ - remove:
+ field:
+ - threatintel.misp.Attribute
+ ignore_missing: true
+ if: ctx?.threatintel?.misp?.Attribute.size() == 0
+ - remove:
+ field:
+ - threatintel.misp.Object
+ ignore_missing: true
+ if: ctx?.threatintel?.misp?.Object.size() == 0
  - date:
  field: threatintel.misp.timestamp
  formats:
@@ -54,6 +65,14 @@ processors:
  field: threatintel.misp.Attribute
  target_field: threatintel.misp.attribute
  ignore_missing: true
+ - rename:
+ field: threatintel.misp.Object
+ target_field: threatintel.misp.object
+ ignore_missing: true
+ - rename:
+ field: threatintel.misp.object.Attribute
+ target_field: threatintel.misp.object.attribute
+ ignore_missing: true
  - rename:
  field: threatintel.misp.Orgc
  target_field: threatintel.misp.orgc
@@ -62,15 +81,25 @@ processors:
  field: threatintel.misp.Org
  target_field: threatintel.misp.org
  ignore_missing: true
- - rename:
- field: threatintel.misp.Attribute
- target_field: threatintel.misp.attribute
- ignore_missing: true
  - rename:
  field: threatintel.misp.Tag
  target_field: threatintel.misp.tag
  ignore_missing: true
 
+ # # Dance around issue of not being able to split the document into two.
+ # # Make the Object.Attribute field primary if it exists, but keep the
+ # # outer Attribute as context.
+ - rename:
+ field: threatintel.misp.attribute
+ target_field: threatintel.misp.context.attribute
+ ignore_missing: true
+ if: ctx?.threatintel?.misp?.object != null
+ - rename:
+ field: threatintel.misp.object.attribute
+ target_field: threatintel.misp.attribute
+ ignore_missing: true
+ if: ctx?.threatintel?.misp?.object != null
+
  #####################
  # Threat ECS Fields #
  #####################
@@ -96,7 +125,7 @@ processors:
  - set:
  field: threatintel.indicator.type
  value: file
- if: "['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.threatintel?.misp?.attribute?.type) || ctx.threatintel?.misp?.attribute?.type.startsWith('filename')"
+ if: "ctx?.threatintel?.misp?.attribute?.type != null && (['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.threatintel?.misp?.attribute?.type) || ctx.threatintel?.misp?.attribute?.type.startsWith('filename'))"
  - rename:
  field: threatintel.misp.attribute.value
  target_field: "threatintel.indicator.file.hash.{{threatintel.misp.attribute.type}}"
@@ -112,17 +141,17 @@ processors:
  patterns:
  - "%{WORD}\\|%{WORD:_tmp.hashtype}"
  ignore_missing: true
- if: ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|')
+ if: ctx?.threatintel?.misp?.attribute?.type != null && ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|')
  - grok:
  field: threatintel.misp.attribute.value
  patterns:
  - "%{DATA:threatintel.indicator.file.name}\\|%{GREEDYDATA:_tmp.hashvalue}"
  ignore_missing: true
- if: ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|')
+ if: ctx?.threatintel?.misp?.attribute?.type != null && ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|')
  - set:
  field: threatintel.indicator.file.hash.{{_tmp.hashtype}}
  value: "{{_tmp.hashvalue}}"
- if: "ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|') && ctx?._tmp?.hashvalue != null && ctx?._tmp?.hashtype != null"
+ if: "ctx?.threatintel?.misp?.attribute?.type != null && ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|') && ctx?._tmp?.hashvalue != null && ctx?._tmp?.hashtype != null"
 
  ## URL/URI indicator operations
  - set:
@@ -218,6 +247,14 @@ processors:
  target_field: threatintel.indicator.email.address
  ignore_missing: true
  if: ctx?.threatintel?.indicator?.type == 'email-addr'
+ - rename:
+ field: threatintel.misp.event_creator_email
+ target_field: user.email
+ ignore_missing: true
+ - append:
+ field: user.roles
+ value: "reporting_user"
+ if: ctx?.user?.email != null
 
  ## MAC Address indicator operations
  - set:
@@ -288,6 +325,14 @@ processors:
  - threatintel.misp.attribute.value
  ignore_missing: true
  if: ctx?.threatintel?.indicator?.type != 'unknown'
+ - remove:
+ field:
+ # This removes a number of fields that may be wanted in the future when
+ # threatintel.misp.attribute and threatintel.misp.object.attribute can
+ # be separated. At the root of .object are fields that mirror fields at
+ # the root of threatintel.misp.
+ - threatintel.misp.object
+ ignore_missing: true
  - remove:
  field:
  - threatintel.misp.Attribute.timestamp