Merge pull request #298 from robcza/alienvault-otx

AlienVault OTX API

intelmq/bots/BOTS

@@ -216,6 +216,15 @@
  "url": "https://dshield.org/asdetailsascii.html?as=<AS Number>"
  }
  },
+ "AlienVault OTX": {
+ "module": "intelmq.bots.collectors.alienvault_otx.collector",
+ "description": "AlienVault OTX Collector is the bot responsible to get the report through the API. Report could vary according to subscriptions.",
+ "parameters": {
+ "api_key": "<insert your api key>",
+ "feed": "AlienVault OTX",
+ "rate_limit": "3600"
+ }
+ },
  "DShield Block": {
  "description": "DShield Block Collector is the bot responsible to get the report from source of information.",
  "module": "intelmq.bots.collectors.http.collector_http",
@@ -654,6 +663,11 @@
  "module": "intelmq.bots.parsers.alienvault.parser",
  "parameters": {}
  },
+ "AlienVault OTX": {
+ "description": "AlienVault Parser is the bot responsible to parse the report and sanitize the information.",
+ "module": "intelmq.bots.parsers.alienvault.parser_otx",
+ "parameters": {}
+ },
  "Arbor": {
  "description": "Arbor Parser is the bot responsible to parse the report and sanitize the information.",
  "module": "intelmq.bots.parsers.arbor.parser",

intelmq/bots/collectors/alienvault_otx/COPYRIGHT.md

@@ -0,0 +1,2 @@
+Contents of the file "OTXv2.py" are licensed under the Apache license and originally taken from AlienVault Labs: https:/AlienVault-Labs/OTX-Python-SDK
+Complete license TERMS AND CONDITIONS: https:/AlienVault-Labs/OTX-Python-SDK/blob/master/LICENSE

intelmq/bots/collectors/alienvault_otx/OTXv2.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+
+import httplib
+import urlparse
+import urllib
+import urllib2
+import simplejson as json
+import time
+import re
+import logging
+import datetime
+
+logger = logging.getLogger("OTXv2")
+
+
+class InvalidAPIKey(Exception):
+
+ def __init__(self, value):
+ self.value = value
+
+ def __str__(self):
+ return repr(self.value)
+
+
+class BadRequest(Exception):
+
+ def __init__(self, value):
+ self.value = value
+
+ def __str__(self):
+ return repr(self.value)
+
+
+class OTXv2(object):
+
+ def __init__(self, key, server="http://otx.alienvault.com"):
+ self.key = key
+ self.server = server
+
+ def get(self, url):
+ request = urllib2.build_opener()
+ request.addheaders = [('X-OTX-API-KEY', self.key)]
+ response = None
+ try:
+ response = request.open(url)
+ except urllib2.URLError as e:
+ if e.code == 403:
+ raise InvalidAPIKey("Invalid API Key")
+ elif e.code == 400:
+ raise BadRequest("Bad Request")
+ data = response.read()
+ json_data = json.loads(data)
+ return json_data
+
+ def getall(self, limit=20):
+ pulses = []
+ next = "%s/api/v1/pulses/subscribed?limit=%d" % (self.server, limit)
+ while next:
+ json_data = self.get(next)
+ for r in json_data["results"]:
+ pulses.append(r)
+ next = json_data["next"]
+ return pulses
+
+ def getall_iter(self, limit=20):
+ pulses = []
+ next = "%s/api/v1/pulses/subscribed?limit=%d" % (self.server, limit)
+ while next:
+ json_data = self.get(next)
+ for r in json_data["results"]:
+ yield r
+ next = json_data["next"]
+
+ def getsince(self, mytimestamp, limit=20):
+ pulses = []
+ next = "%s/api/v1/pulses/subscribed?limit=%d&modified_since=%s" % (
+ self.server, limit, mytimestamp)
+ while next:
+ json_data = self.get(next)
+ for r in json_data["results"]:
+ pulses.append(r)
+ next = json_data["next"]
+ return pulses
+
+ def getsince_iter(self, mytimestamp, limit=20):
+ pulses = []
+ next = "%s/api/v1/pulses/subscribed?limit=%d&modified_since=%s" % (
+ self.server, limit, mytimestamp)
+ while next:
+ json_data = self.get(next)
+ for r in json_data["results"]:
+ yield r
+ next = json_data["next"]
+
+ def getevents_since(self, mytimestamp, limit=20):
+ events = []
+ next = "%s/api/v1/pulses/events?limit=%d&since=%s" % (
+ self.server, limit, mytimestamp)
+ while next:
+ json_data = self.get(next)
+ for r in json_data["results"]:
+ events.append(r)
+ next = json_data["next"]
+ return events

intelmq/bots/collectors/alienvault_otx/README.md

@@ -0,0 +1,4 @@
+- Collector for: https://otx.alienvault.com
+- Needs this script to be run: https:/AlienVault-Labs/OTX-Python-SDK/blob/master/OTXv2.py
+- The runtime.conf parameter "api_key" has to be set (register on the website to get one)
+

intelmq/bots/collectors/alienvault_otx/collector.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import sys
+from OTXv2 import OTXv2
+import json
+
+from intelmq.lib.bot import Bot
+from intelmq.lib.message import Report
+from intelmq.lib.harmonization import DateTime
+
+
+class AlienVaultOTXCollectorBot(Bot):
+
+ def process(self):
+ self.logger.info("Downloading report through API")
+ otx = OTXv2(self.parameters.api_key)
+ pulses = otx.getall()
+ self.logger.info("Report downloaded.")
+
+ report = Report()
+ report.add("raw", json.dumps(pulses), sanitize=True)
+ report.add("feed.name", self.parameters.feed, sanitize=True)
+ time_observation = DateTime().generate_datetime_now()
+ report.add('time.observation', time_observation, sanitize=True)
+ self.send_message(report)
+
+
+if __name__ == "__main__":
+ bot = AlienVaultOTXCollectorBot(sys.argv[1])
+ bot.start()

intelmq/bots/parsers/alienvault/parser_otx.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+"""
+Events are gathered based on user subscriptions in AlienVault OTX
+The data structure is described in detail here:
+https:/AlienVault-Labs/OTX-Python-SDK/blob/master/
+howto_use_python_otx_api.ipynb
+"""
+from __future__ import unicode_literals
+
+import json
+import sys
+
+from intelmq.lib import utils
+from intelmq.lib.bot import Bot
+from intelmq.lib.message import Event
+
+HASHES = {
+ 'FileHash-SHA256': '$5$',
+ 'FileHash-SHA1': '$sha1$',
+ 'FileHash-MD5': '$1$'
+}
+
+
+class AlienVaultOTXParserBot(Bot):
+
+ def process(self):
+ report = self.receive_message()
+ if report is None or not report.contains("raw"):
+ self.acknowledge_message()
+ return
+
+ raw_report = utils.base64_decode(report.value("raw"))
+
+ for pulse in json.loads(raw_report):
+ additional = json.dumps({"author": pulse['author_name'],
+ "pulse": pulse['name']},
+ sort_keys=True)
+ for indicator in pulse["indicators"]:
+ event = Event(report)
+ # hashes
+ if indicator["type"] in HASHES.keys():
+ event.add('malware.hash', HASHES[indicator["type"]] +
+ indicator["indicator"])
+ # fqdn
+ if indicator["type"] in ['hostname', 'domain']:
+ event.add('source.fqdn',
+ indicator["indicator"], sanitize=True)
+ # IP addresses
+ elif indicator["type"] in ['IPv4', 'IPv6']:
+ event.add('source.ip',
+ indicator["indicator"], sanitize=True)
+ # emails
+ elif indicator["type"] == 'email':
+ event.add('source.account',
+ indicator["indicator"], sanitize=True)
+ # URLs
+ elif indicator["type"] in ['URL', 'URI']:
+ event.add('source.url',
+ indicator["indicator"], sanitize=True)
+ # CIDR
+ elif indicator["type"] in ['CIDR']:
+ event.add('source.network',
+ indicator["indicator"], sanitize=True)
+ # FilePath, Mutex, CVE - TODO: process these IoCs as well
+ else:
+ continue
+
+ event.add('comment', pulse['description'])
+ event.add('extra', additional, sanitize=True)
+ event.add('classification.type', 'blacklist', sanitize=True)
+ event.add('time.source', indicator["created"][:-4] + "+00:00",
+ sanitize=True)
+ event.add("raw", json.dumps(indicator, sort_keys=True),
+ sanitize=True)
+ self.send_message(event)
+ self.acknowledge_message()
+
+if __name__ == "__main__":
+ bot = AlienVaultOTXParserBot(sys.argv[1])
+ bot.start()

intelmq/conf/harmonization.conf

@@ -148,7 +148,7 @@
  "type": "URL"
  },
  "malware.hash": {
- "description": "A string depicting a checksum for a file, be it a malware sample for example.",
+ "description": "A string depicting a checksum for a file, be it a malware sample for example. Includes hash type according to https://en.wikipedia.org/wiki/Crypt_%28C%29",
  "regex": "^[a-fA-F0-9]+$",
  "type": "String"
  },