bug: fix ctip parser handling JSON Payload content

CHANGELOG.md

@@ -58,6 +58,7 @@ CHANGELOG
 - `intelmq.bots.parsers.microsoft.parser_ctip`:
  - New parameter `overwrite` (PR#2112 by Sebastian Wagner, fixes #2022).
  - Fix handling of field `Payload.domain` if it contains the same IP address as `Payload.serverIp` (PR#2144 by Mikk Margus Möll and Sebastian Wagner).
+ - Handle Payload field with non-base64-encoded JSON content and numbered dictionaries (PR#2193 by Sebastian Wagner)
 - `intelmq.bot.parsers.shodan.parser` (PR#2117 by Mikk Margus Möll):
  - Instead of keeping track of `extra.ftp.<something>.parameters`, FTP parameters are collected together into `extra.ftp.features` as a list of said features, reducing field count.
  - Shodan field `rsync.modules` is collected.

intelmq/bots/parsers/microsoft/parser_ctip.py

@@ -59,7 +59,7 @@
  "CustomField4": "",
  "CustomField5": ""
  },
- "Payload": base64 encoded json
+ "Payload": base64 encoded json with meaningful dictionary keys or JSON-string with numbered dictionary keys
  }
 
 """
@@ -263,14 +263,23 @@ def parse_azure(self, line, report):
 
  for key, value in line.copy().items():
  if key == 'Payload':
+ # empty
  if value == 'AA==': # NULL
  del line[key]
  continue
- try:
- value = json.loads(utils.base64_decode(value))
- # continue unpacking in next loop
- except json.decoder.JSONDecodeError:
- line[key] = utils.base64_decode(value)
+
+ # JSON string
+ if value.startswith('{'):
+ for payload_key, payload_value in json.loads(value).items():
+ event[f'extra.payload.{payload_key}'] = payload_value
+ del line[key]
+ else:
+ # base64-encoded JSON
+ try:
+ value = json.loads(utils.base64_decode(value))
+ # continue unpacking in next loop
+ except json.decoder.JSONDecodeError:
+ line[key] = utils.base64_decode(value)
  elif key == 'TLP' and value.lower() == 'unknown':
  del line[key]
  if isinstance(value, dict):

intelmq/etc/feeds.yaml

@@ -1756,7 +1756,7 @@ providers:
  parser:
  module: intelmq.bots.parsers.microsoft.parser_ctip
  parameters:
- revision: 2020-05-29
+ revision: 2022-06-01
  documentation: https://docs.microsoft.com/en-us/security/gsp/informationsharingandexchange http://www.dcuctip.com/
  public: false
  CTIP C2 via Azure:

intelmq/tests/bots/parsers/microsoft/ctip_azure.txt

@@ -3,3 +3,4 @@
 {"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Gov.0001","DateTimeReceivedUtc":132622667720000000,"DateTimeReceivedUtcTxt":"Wednesday April 07 2021 10:59:32.0000","Malware":"Emotet","ThreatCode":"B77-GV","ThreatConfidence":"High","TotalEncounters":1,"TLP":"Unknown","SourceIp":"224.0.5.8","SourcePort":33587,"DestinationIp":"10.0.0.1","DestinationPort":8080,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"Styria","SourceIpCity":"Graz","SourceIpPostalCode":"8042","SourceIpLatitude":47.1298,"SourceIpLongitude":15.466,"SourceIpMetroCode":0,"SourceIpAreaCode":6,"SourceIpConnectionType":"","SourceIpv4Int":0},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"bot-id-data","CustomField2":"comp-name","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0aW1lc3RhbXBfdXRjIjoiMjAyMS0wNC0wN1QxMDo1OTozMiIsInNvdXJjZV9pcCI6IjEwLjAuMC4xIiwic291cmNlX3BvcnQiOiIzMzU4NyIsImRlc3RpbmF0aW9uX2lwIjoiMTAuMC4wLjEiLCJkZXN0aW5hdGlvbl9wb3J0IjoiODA4MCIsImNvbXB1dGVyX25hbWUiOiJjb21wLW5hbWUiLCJib3RfaWQiOiJib3QtaWQtZGF0YSJ9"}
 {"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132651352622420000,"DateTimeReceivedUtcTxt":"Monday May 10 2021 15:47:42.2420","Malware":"Avalanche","ThreatCode":"B67-SS-Gamarue","ThreatConfidence":"Low","TotalEncounters":2,"TLP":"Green","SourceIp":"224.0.5.8","SourcePort":28285,"DestinationIp":"10.0.0.1","DestinationPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cellular","SourceIpv4Int":3758097672},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"andromeda210","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTYyMDY2MTY2Mi4yNDIzMTYsImlwIjoiMjI0LjAuNS44IiwicG9ydCI6MjgyODUsInNlcnZlcklwIjoiMTAuMC4wLjEiLCJzZXJ2ZXJQb3J0Ijo4MCwiZG9tYWluIjoiZXhhbXBsZS5jb20iLCJmYW1pbHkiOiJhbmRyb21lZGEiLCJtYWx3YXJlIjp7fSwicmVzcG9uc2UiOiJIdHRwT2siLCJoYW5kbGVyIjoiaGFuZGxlcjEiLCJ0eXBlIjoiSHR0cCJ9"}
 {"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132651352622420000,"DateTimeReceivedUtcTxt":"Monday May 10 2021 15:47:42.2420","Malware":"Avalanche","ThreatCode":"B67-SS-Gamarue","ThreatConfidence":"Low","TotalEncounters":2,"TLP":"Green","SourceIp":"224.0.5.8","SourcePort":28285,"DestinationIp":"10.0.0.1","DestinationPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cellular","SourceIpv4Int":3758097672},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"andromeda210","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTYyMDY2MTY2Mi4yNDIzMTYsImlwIjoiMjI0LjAuNS44IiwicG9ydCI6MjgyODUsInNlcnZlcklwIjoiMTAuMC4wLjEiLCJzZXJ2ZXJQb3J0Ijo4MCwiZG9tYWluIjoiMTAuMC4wLjEiLCJmYW1pbHkiOiJhbmRyb21lZGEiLCJtYWx3YXJlIjp7fSwicmVzcG9uc2UiOiJIdHRwT2siLCJoYW5kbGVyIjoiaGFuZGxlcjEiLCJ0eXBlIjoiSHR0cCJ9"}
+{"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132990083418030000,"DateTimeReceivedUtcTxt":"Wednesday June 01 2022 13:33:13.3713","Malware":"Malware","ThreatCode":"B00-Leet","ThreatConfidence":"High","TotalEncounters":137,"TLP":"Green","SourceIp":"10.0.0.15","SourcePort":10000,"DestinationIp":"10.0.0.2","DestinationPort":443,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"My ISP","SourceIpCountryCode":"DE","SourceIpRegion":"Saarland","SourceIpCity":"Saarbrücken","SourceIpPostalCode":"66111","SourceIpLatitude":49.2367,"SourceIpLongitude":6.9794,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cable/DSL","SourceIpv4Int":167772175},"HttpInfo":{"HttpHost":"example.com","HttpRequest":"/index.php","HttpMethod":"POST","HttpReferrer":"","HttpUserAgent":"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36","HttpVersion":"HTTP/1.1"},"CustomInfo":{"CustomField1":"v1.6","CustomField2":"14758f1afd44c09b7992073ccf00b43d","CustomField3":"my PC name","CustomField4":"personal","CustomField5":""},"Payload":"{\"10001\":\"my PC name\",\"10002\":\"personal\",\"10022\":\"00000000\",\"10029\":157,\"10006\":\"00\"}"}

intelmq/tests/bots/parsers/microsoft/test_parser_ctip_azure.py

@@ -189,6 +189,46 @@
  'tlp': 'GREEN',
  'extra.source.connection_type': 'Cellular',
  },
+ {'__type': 'Event',
+ 'classification.type': 'infected-system',
+ 'destination.ip': '10.0.0.2',
+ 'destination.port': 443,
+ 'event_description.text': 'Microsoft.DCU.CTIP.Sinkhole',
+ 'extra.custom_field1': 'v1.6',
+ 'extra.custom_field2': '14758f1afd44c09b7992073ccf00b43d',
+ 'extra.custom_field3': 'my PC name',
+ 'extra.custom_field4': 'personal',
+ 'extra.http.host': 'example.com',
+ 'extra.http.method': 'POST',
+ 'extra.http.request': '/index.php',
+ 'extra.http.version': 'HTTP/1.1',
+ 'extra.malware': 'Malware',
+ 'extra.payload.10001': 'my PC name',
+ 'extra.payload.10002': 'personal',
+ 'extra.payload.10006': '00',
+ 'extra.payload.10022': '00000000',
+ 'extra.payload.10029': 157,
+ 'extra.source.connection_type': 'Cable/DSL',
+ 'extra.source.geolocation.postal_code': '66111',
+ 'extra.total_encounters': 137,
+ 'extra.user_agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) '
+ 'AppleWebKit/537.36 (KHTML, like Gecko) '
+ 'Chrome/79.0.3945.88 Safari/537.36',
+ 'feed.accuracy': 100.0,
+ 'feed.name': 'ctip',
+ 'malware.name': 'b00-leet',
+ 'raw': base64_encode(EXAMPLE_LINES[5]),
+ 'source.as_name': 'My ISP',
+ 'source.asn': 64496,
+ 'source.geolocation.cc': 'DE',
+ 'source.geolocation.city': 'Saarbrücken',
+ 'source.geolocation.latitude': 49.2367,
+ 'source.geolocation.longitude': 6.9794,
+ 'source.geolocation.region': 'Saarland',
+ 'source.ip': '10.0.0.15',
+ 'source.port': 10000,
+ 'time.source': '2022-06-06T16:59:01.802999+00:00',
+ 'tlp': 'GREEN'},
 ]