Parsers for Bambenek Consulting and Netlab 360 OSINT feeds

intelmq/bots/BOTS

@@ -180,6 +180,21 @@
  "module": "intelmq.bots.parsers.autoshun.parser",
  "parameters": {}
  },
+ "Bambenek C2 Domain Feed": {
+ "description": "Bambenek C2 Domain parser is the bot responsible to parse and and sanatize the information.",
+ "module": "intelmq.bots.parsers.bambenek.parser_c2dommasterlist",
+ "parameters": {}
+ },
+ "Bambenek C2 IP Feed": {
+ "description": "Bambenek C2 IP parser is the bot responsible to parse and and sanatize the information.",
+ "module": "intelmq.bots.parsers.bambenek.parser_c2ipmasterlist",
+ "parameters": {}
+ },
+ "Bambenek DGA Domain Feed": {
+ "description": "Bambenek DGA Domain parser is the bot responsible to parse and and sanatize the information.",
+ "module": "intelmq.bots.parsers.bambenek.parser_dgafeed",
+ "parameters": {}
+ },
  "BitSight Ciberfeed Stream": {
  "description": "Bitsight Ciberfeed parser is the bot reponsible to parse and sanitize the information.",
  "module": "intelmq.bots.parsers.bitsight.parser",
@@ -316,6 +331,16 @@
  "module": "intelmq.bots.parsers.n6.parser_n6stomp",
  "parameters": {}
  },
+ "Netlab 360 DGA": {
+ "description": "Netlab 360 DGA Feed Parser is the bot responsible to parse the report and sanitize the information.",
+ "module": "intelmq.bots.parsers.netlab_360.parser_dga",
+ "parameters": {}
+ },
+ "Netlab 360 Magnitude": {
+ "description": "Netlab 360 Magnitude EK Feed Parser is the bot responsible to parse the report and sanitize the information.",
+ "module": "intelmq.bots.parsers.netlab_360.parser_magnitude",
+ "parameters": {}
+ },
  "OpenBL": {
  "description": "OpenBL Parser is the bot responsible to parse the report and sanitize the information.",
  "module": "intelmq.bots.parsers.openbl.parser",

intelmq/bots/parsers/bambenek/parser_c2dommasterlist.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+"""
+http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt
+format:
+ooxnererwyatanb.com,Domain used by banjori,2016-11-10 15:04,http://osint.bambenekconsulting.com/manual/banjori.txt
+destination.fqdn,event_description.text,time.source,event_description.url
+"""
+
+import sys
+from intelmq.lib import utils
+from intelmq.lib.bot import Bot
+from intelmq.lib.message import Event
+
+
+class Bambenekc2dommasterlistParserBot(Bot):
+
+ def process(self):
+ report = self.receive_message()
+ raw_report = utils.base64_decode(report.get("raw"))
+
+ for row in raw_report.splitlines():
+ if row.startswith('#'):
+ continue
+
+ row_split = row.split(',')
+
+ event = Event(report)
+
+ event.add('destination.fqdn', row_split[0])
+ event.add('event_description.text', row_split[1])
+ event.add('time.source', row_split[2] + " UTC")
+ event.add('event_description.url', row_split[3])
+ event.add('classification.type', 'c&c')
+ event.add('status', 'online')
+ event.add('raw', row)
+
+ self.send_message(event)
+ self.acknowledge_message()
+
+if __name__ == "__main__":
+ bot = Bambenekc2dommasterlistParserBot(sys.argv[1])
+ bot.start()

intelmq/bots/parsers/bambenek/parser_c2ipmasterlist.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+"""
+http://osint.bambenekconsulting.com/feeds/c2-ipmasterlist.txt
+format:
+31.170.178.179,IP used by beebone C&C,2016-11-12 14:09,http://osint.bambenekconsulting.com/manual/beebone.txt
+destination.ip,event_description.text,time.source,event_description.url
+"""
+
+import sys
+from intelmq.lib import utils
+from intelmq.lib.bot import Bot
+from intelmq.lib.message import Event
+
+
+class Bambenekc2ipmasterlistParserBot(Bot):
+
+ def process(self):
+ report = self.receive_message()
+ raw_report = utils.base64_decode(report.get("raw"))
+
+ for row in raw_report.splitlines():
+ if row.startswith('#'):
+ continue
+
+ row_split = row.split(',')
+
+ event = Event(report)
+
+ event.add('destination.ip', row_split[0])
+ event.add('event_description.text', row_split[1])
+ event.add('time.source', row_split[2] + " UTC")
+ event.add('event_description.url', row_split[3])
+ event.add('classification.type', 'c&c')
+ event.add('status', 'online')
+ event.add('raw', row)
+
+ self.send_message(event)
+ self.acknowledge_message()
+
+if __name__ == "__main__":
+ bot = Bambenekc2ipmasterlistParserBot(sys.argv[1])
+ bot.start()

intelmq/bots/parsers/bambenek/parser_dgafeed.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+"""
+feed:
+http://osint.bambenekconsulting.com/feeds/dga-feed.txt
+format:
+xqmclnusaswvof.com,Domain used by Cryptolocker - Flashback DGA for 10 Nov 2016,2016-11-10,
+http://osint.bambenekconsulting.com/manual/cl.txt
+destination.fqdn,event_description.text,time.source,event_description.url
+"""
+
+import sys
+from intelmq.lib import utils
+from intelmq.lib.bot import Bot
+from intelmq.lib.message import Event
+
+
+class BambenekDGAfeedParserBot(Bot):
+
+ def process(self):
+ report = self.receive_message()
+ raw_report = utils.base64_decode(report.get("raw"))
+
+ for row in raw_report.splitlines():
+ if row.startswith('#'):
+ continue
+
+ row_split = row.split(',')
+
+ event = Event(report)
+
+ event.add('destination.fqdn', row_split[0])
+ event.add('event_description.text', row_split[1])
+ event.add('time.source', row_split[2] + " 00:00 UTC")
+ event.add('event_description.url', row_split[3])
+ event.add('classification.type', 'ransomware')
+ event.add('raw', row)
+
+ self.send_message(event)
+ self.acknowledge_message()
+
+if __name__ == "__main__":
+ bot = BambenekDGAfeedParserBot(sys.argv[1])
+ bot.start()

intelmq/bots/parsers/netlab_360/parser_dga.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+"""
+feed:
+http://data.netlab.360.com/feeds/dga/dga.txt
+format:
+suppobox difficultdress.net 2016-11-12 11:58:56 2016-11-13 00:04:15
+reference:
+http://data.netlab.360.com/dga
+"""
+
+import sys
+from intelmq.lib import utils
+from intelmq.lib.bot import Bot
+from intelmq.lib.message import Event
+
+
+class Netlab360DGAParserBot(Bot):
+
+ def process(self):
+ report = self.receive_message()
+ raw_report = utils.base64_decode(report.get("raw"))
+
+ for row in raw_report.splitlines():
+ if row.startswith('#') or len(row) == 0:
+ continue
+
+ row_split = row.split('\t')
+
+ event = Event(report)
+
+ event.add('classification.identifier', row_split[0].lower())
+ event.add('time.source', row_split[3] + " UTC")
+ event.add('destination.fqdn', row_split[1])
+ event.add('raw', row)
+ event.add('classification.type', 'malware')
+ event.add('event_description.url', 'http://data.netlab.360.com/dga')
+
+ self.send_message(event)
+ self.acknowledge_message()
+
+if __name__ == "__main__":
+ bot = Netlab360DGAParserBot(sys.argv[1])
+ bot.start()

intelmq/bots/parsers/netlab_360/parser_magnitude.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""
+feed:
+http://data.netlab.360.com/feeds/ek/magnitude.txt
+format:
+Magnitude 1478939365 178.32.227.12 f57idayaa1.notsite.faith http://f57idayaa1.notsite.faith/1c472e11922f6a0a6c20fa5996aa35f4
+Magnitude 1478907804 178.32.227.12 3w212q83hay0ao51fi.courtall.party N/A
+reference:
+http://data.netlab.360.com/ek
+"""
+
+import sys
+from datetime import datetime
+from intelmq.lib import utils
+from intelmq.lib.bot import Bot
+from intelmq.lib.message import Event
+
+
+class Netlab360MagnitudeParserBot(Bot):
+
+ def process(self):
+ report = self.receive_message()
+ raw_report = utils.base64_decode(report.get("raw"))
+
+ for row in raw_report.splitlines():
+ if row.startswith('#') or len(row) == 0:
+ continue
+
+ row_split = row.split('\t')
+
+ event = Event(report)
+
+ event.add('classification.identifier', row_split[0].lower())
+ event.add('time.source', datetime.utcfromtimestamp(int(row_split[1])).strftime('%Y-%m-%dT%H:%M:%S+00:00'))
+ event.add('destination.ip', row_split[2])
+ event.add('destination.fqdn', row_split[3])
+ event.add('destination.url', row_split[4])
+ event.add('raw', row)
+ event.add('classification.type', 'exploit')
+ event.add('event_description.url', 'http://data.netlab.360.com/ek')
+
+ self.send_message(event)
+ self.acknowledge_message()
+
+if __name__ == "__main__":
+ bot = Netlab360MagnitudeParserBot(sys.argv[1])
+ bot.start()

intelmq/tests/bots/parsers/bambenek/c2-dommasterlist.txt

@@ -0,0 +1,17 @@
+#############################################################
+## Master Feed of known, active and non-sinkholed C&Cs domain
+## names
+## 
+## Feed generated at: 2016-11-12 15:16 
+##
+## Feed Provided By: John Bambenek of Bambenek Consulting
+## [email protected] // http://bambenekconsulting.com
+## Use of this feed is governed by the license here: 
+## http://osint.bambenekconsulting.com/license.txt 
+##
+## For more information on this feed go to: 
+## http://osint.bambenekconsulting.com/manual/c2-dommasterlist.txt
+## 
+## All times are in UTC
+#############################################################
+sfkh21okrrnlu.com,Domain used by shiotob/urlzone/bebloh,2016-11-12 15:02,http://osint.bambenekconsulting.com/manual/bebloh.txt

intelmq/tests/bots/parsers/bambenek/c2-ipmasterlist.txt

@@ -0,0 +1,17 @@
+#############################################################
+## Master Feed of known, active and non-sinkholed C&Cs IP 
+## addresses
+## 
+## Feed generated at: 2016-11-12 18:17 
+##
+## Feed Provided By: John Bambenek of Bambenek Consulting
+## [email protected] // http://bambenekconsulting.com
+## Use of this feed is governed by the license here: 
+## http://osint.bambenekconsulting.com/license.txt 
+##
+## For more information on this feed go to: 
+## http://osint.bambenekconsulting.com/manual/c2-ipmasterlist.txt
+## 
+## All times are in UTC
+#############################################################
+213.247.47.190,IP used by shiotob/urlzone/bebloh C&C,2016-11-12 18:02,http://osint.bambenekconsulting.com/manual/bebloh.txt

intelmq/tests/bots/parsers/bambenek/dga-feed.txt

@@ -0,0 +1,15 @@
+#############################################################
+## Domain feed of known DGA domains from -2 to +3 days
+## 
+## Feed generated at: Sat Nov 12 00:15:01 UTC 2016 
+##
+## Feed Provided By: John Bambenek of Bambenek Consulting
+## [email protected] // http://bambenekconsulting.com
+##
+## Use of this feed is governed by the license here: 
+## http://osint.bambenekconsulting.com/license.txt 
+## For more information on this feed go to: 
+## http://osint.bambenekconsulting.com/manual/dga-feed.txt
+## 
+#############################################################
+xqmclnusaswvof.com,Domain used by Cryptolocker - Flashback DGA for 10 Nov 2016,2016-11-10,http://osint.bambenekconsulting.com/manual/cl.txt

intelmq/tests/bots/parsers/bambenek/test_parser_c2dommasterlist.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+import os
+import unittest
+
+import intelmq.lib.test as test
+import intelmq.lib.utils as utils
+from intelmq.bots.parsers.bambenek.parser_c2dommasterlist import Bambenekc2dommasterlistParserBot
+
+with open(os.path.join(os.path.dirname(__file__), 'c2-dommasterlist.txt')) as handle:
+ EXAMPLE_FILE = handle.read()
+
+EXAMPLE_REPORT = {"feed.name": "Bambenek C2 Domain Feed",
+ "feed.url": "http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt",
+ "raw": utils.base64_encode(EXAMPLE_FILE),
+ "__type": "Report",
+ "time.observation": "2016-01-01T00:00:00+00:00",
+ }
+EVENTS = [{"feed.name": "Bambenek C2 Domain Feed",
+ "feed.url": "http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt",
+ "__type": "Event",
+ "time.source": "2016-11-12T15:02:00+00:00",
+ "destination.fqdn": "sfkh21okrrnlu.com",
+ "classification.type": "c&c",
+ "status": "online",
+ "time.observation": "2016-01-01T00:00:00+00:00",
+ "event_description.text": "Domain used by shiotob/urlzone/bebloh",
+ "event_description.url": "http://osint.bambenekconsulting.com/manual/bebloh.txt",
+ "raw": "c2ZraDIxb2tycm5sdS5jb20sRG9tYWluIHVzZWQgYnkgc2hpb3RvYi91cmx6b25lL2JlYmxvaCwyMDE2LTExLTEyIDE1OjAyLGh0dHA6Ly9vc2ludC5iYW1iZW5la2NvbnN1bHRpbmcuY29tL21hbnVhbC9iZWJsb2gudHh0",
+ }]
+
+class TestBambenekc2dommasterlistParserBot(test.BotTestCase, unittest.TestCase):
+ """
+ A TestCase for Bambenekc2dommasterlistParserBot
+ """
+
+ @classmethod
+ def set_bot(cls):
+ cls.bot_reference = Bambenekc2dommasterlistParserBot
+ cls.default_input_message = EXAMPLE_REPORT
+
+ def test_event(self):
+ """ Test if correct Event has been produced. """
+ self.run_bot()
+ self.assertMessageEqual(0, EVENTS[0])
+
+if __name__ == '__main__':
+ unittest.main()