From 1162c36fa973dcc73a6102bb509835ed97a98b85 Mon Sep 17 00:00:00 2001 From: robcza Date: Tue, 25 Aug 2015 16:38:09 +0200 Subject: [PATCH 01/12] alienvault-otx api collector and parser --- intelmq/bots/BOTS | 16 +++++ .../collectors/alienvault-otx/__init__.py | 0 .../collectors/alienvault-otx/collector.py | 28 ++++++++ intelmq/bots/parsers/alienvault/parser-otx.py | 64 +++++++++++++++++++ 4 files changed, 108 insertions(+) create mode 100644 intelmq/bots/collectors/alienvault-otx/__init__.py create mode 100644 intelmq/bots/collectors/alienvault-otx/collector.py create mode 100644 intelmq/bots/parsers/alienvault/parser-otx.py diff --git a/intelmq/bots/BOTS b/intelmq/bots/BOTS index dd6bf1a6f..05970a624 100644 --- a/intelmq/bots/BOTS +++ b/intelmq/bots/BOTS @@ -288,6 +288,15 @@ "rate_limit": "3600" } }, + "AlienVault OTX": { + "module": "intelmq.bots.collectors.alienvault-otx.collector", + "description": "AlienVault OTX Collector is the bot responsible to get the report through the API. Report could vary according to subscriptions.", + "parameters": { + "api_key": "", + "feed": "AlienVault OTX", + "rate_limit": "3600" + } + }, "Danger Rulez": { "module": "intelmq.bots.collectors.http.collector_http", "description": "Danger Rulez Collector is the bot responsible to get the report from source of information.", @@ -602,6 +611,13 @@ "error_log_message": false } }, + "AlienVault OTX": { + "module": "intelmq.bots.parsers.alienvault.parser-otx", + "description": "AlienVault Parser is the bot responsible to parse the report and sanitize the information.", + "parameters": { + "error_log_message": false + } + }, "Danger Rulez": { "module": "intelmq.bots.parsers.danger_rulez.parser", "description": "Danger Rulez Parser is the bot responsible to parse the report and sanitize the information.", diff --git a/intelmq/bots/collectors/alienvault-otx/__init__.py b/intelmq/bots/collectors/alienvault-otx/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/intelmq/bots/collectors/alienvault-otx/collector.py b/intelmq/bots/collectors/alienvault-otx/collector.py new file mode 100644 index 000000000..e36f8371d --- /dev/null +++ b/intelmq/bots/collectors/alienvault-otx/collector.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +import sys +from OTXv2 import OTXv2 +import json + +from intelmq.bots.collectors.http.lib import fetch_url +from intelmq.lib.bot import Bot +from intelmq.lib.message import Report + + +class AlienVaultOTXCollectorBot(Bot): + + def process(self): + self.logger.info("Downloading report through API") + otx = OTXv2(self.parameters.api_key) + pulses = otx.getall() + self.logger.info("Report downloaded.") + + report = Report() + report.add("raw", json.dumps(pulses), sanitize=True) + report.add("feed.name", self.parameters.feed, sanitize=True) + self.send_message(report) + + +if __name__ == "__main__": + bot = AlienVaultOTXCollectorBot(sys.argv[1]) + bot.start() diff --git a/intelmq/bots/parsers/alienvault/parser-otx.py b/intelmq/bots/parsers/alienvault/parser-otx.py new file mode 100644 index 000000000..b91c5e173 --- /dev/null +++ b/intelmq/bots/parsers/alienvault/parser-otx.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +import sys +import json + +from intelmq.lib import utils +from intelmq.lib.bot import Bot +from intelmq.lib.harmonization import DateTime +from intelmq.lib.message import Event + +HASHES = { + 'FileHash-SHA256' : 'SHA-256', + 'FileHash-SHA1' : 'SHA-1', + 'FileHash-MD5' : 'MD5' +} + + +class AlienVaultOTXParserBot(Bot): + + def process(self): + report = self.receive_message() + if (report is None or not report.contains("raw") or + len(report.value("raw").strip()) == 0): + self.acknowledge_message() + return + + time_observation = DateTime().generate_datetime_now() + raw_report = utils.base64_decode(report.value("raw")) + + for pulse in json.loads(raw_report): + comment = "author: " + pulse['author_name'] + "; name: " + pulse['name'] + "; description: " + pulse['description'] + for indicator in pulse["indicators"]: + event = Event() + #hashes don't work - invalid keys + #if indicator["type"] in ['FileHash-SHA256', 'FileHash-SHA1', 'FileHash-MD5']: + # event.add('source.artifact_hash', indicator["indicator"]) + # event.add('source.artifact_hash_type', HASHES[indicator["type"]], sanitize = True) + #fqdn + if indicator["type"] in ['hostname', 'domain']: + event.add('source.fqdn', indicator["indicator"], sanitize = True) + # IP addresses + elif indicator["type"] in ['IPv4', 'IPv6']: + event.add('source.ip', indicator["indicator"], sanitize = True) + #emails + elif indicator["type"] == 'email': + event.add('source.email_address', indicator["indicator"], sanitize = True) + #URLs + elif indicator["type"] in ['URL', 'URI']: + event.add('source.url', indicator["indicator"], sanitize = True) + #CIDR, FilePath, Mutex, CVE + else: + continue + + event.add('comment', comment) + event.add('classification.type', 'blacklist', sanitize = True) + event.add('time.observation', time_observation, sanitize=True) + event.add('feed.name', report.value("feed.name")) + event.add("raw", json.dumps(indicator), sanitize=True) + self.send_message(event) + self.acknowledge_message() + +if __name__ == "__main__": + bot = AlienVaultOTXParserBot(sys.argv[1]) + bot.start() From e8ae727a1ab4ffda92137a0bdb6b0f0b4842e0f4 Mon Sep 17 00:00:00 2001 From: robcza Date: Tue, 25 Aug 2015 16:50:33 +0200 Subject: [PATCH 02/12] OTX SDK added --- .../bots/collectors/alienvault-otx/OTXv2.py | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 intelmq/bots/collectors/alienvault-otx/OTXv2.py diff --git a/intelmq/bots/collectors/alienvault-otx/OTXv2.py b/intelmq/bots/collectors/alienvault-otx/OTXv2.py new file mode 100644 index 000000000..1db3d6c13 --- /dev/null +++ b/intelmq/bots/collectors/alienvault-otx/OTXv2.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python + +import httplib +import urlparse +import urllib +import urllib2 +import simplejson as json +import time +import re +import logging +import datetime + +logger = logging.getLogger("OTXv2") + +class InvalidAPIKey(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + +class BadRequest(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + +class OTXv2(object): + def __init__(self, key, server="http://otx.alienvault.com"): + self.key = key + self.server = server + + def get(self, url): + request = urllib2.build_opener() + request.addheaders = [('X-OTX-API-KEY', self.key)] + response = None + try: + response = request.open(url) + except urllib2.URLError, e: + if e.code == 403: + raise InvalidAPIKey("Invalid API Key") + elif e.code == 400: + raise BadRequest("Bad Request") + data = response.read() + json_data = json.loads(data) + return json_data + + def getall(self, limit=20): + pulses = [] + next = "%s/api/v1/pulses/subscribed?limit=%d" % (self.server, limit) + while next: + json_data = self.get(next) + for r in json_data["results"]: + pulses.append(r) + next = json_data["next"] + return pulses + + def getall_iter(self, limit=20): + pulses = [] + next = "%s/api/v1/pulses/subscribed?limit=%d" % (self.server, limit) + while next: + json_data = self.get(next) + for r in json_data["results"]: + yield r + next = json_data["next"] + + def getsince(self, mytimestamp, limit=20): + pulses = [] + next = "%s/api/v1/pulses/subscribed?limit=%d&modified_since=%s" % (self.server, limit, mytimestamp) + while next: + json_data = self.get(next) + for r in json_data["results"]: + pulses.append(r) + next = json_data["next"] + return pulses + + def getsince_iter(self, mytimestamp, limit=20): + pulses = [] + next = "%s/api/v1/pulses/subscribed?limit=%d&modified_since=%s" % (self.server, limit, mytimestamp) + while next: + json_data = self.get(next) + for r in json_data["results"]: + yield r + next = json_data["next"] + + + def getevents_since(self, mytimestamp, limit=20): + events = [] + next = "%s/api/v1/pulses/events?limit=%d&since=%s" % (self.server, limit, mytimestamp) + while next: + json_data = self.get(next) + for r in json_data["results"]: + events.append(r) + next = json_data["next"] + return events + + + + + + From 0f7d12f1ec8ed0a71e278daa4fab19dba5f976e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20=C5=A0efr?= Date: Tue, 25 Aug 2015 16:52:15 +0200 Subject: [PATCH 03/12] Create README.md --- intelmq/bots/collectors/alienvault-otx/README.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 intelmq/bots/collectors/alienvault-otx/README.md diff --git a/intelmq/bots/collectors/alienvault-otx/README.md b/intelmq/bots/collectors/alienvault-otx/README.md new file mode 100644 index 000000000..c3691f4fa --- /dev/null +++ b/intelmq/bots/collectors/alienvault-otx/README.md @@ -0,0 +1,4 @@ +- Collector for: https://otx.alienvault.com +- Needs this script to be run: https://github.com/AlienVault-Labs/OTX-Python-SDK/blob/master/OTXv2.py +- The runtime.conf parameter "api_key" has to be set (register on the website to get one) + From e961c00c5ee3b251ad1306145ecd9755759c4316 Mon Sep 17 00:00:00 2001 From: robcza Date: Tue, 25 Aug 2015 17:33:17 +0200 Subject: [PATCH 04/12] pep8 compliance fixed --- intelmq/bots/parsers/alienvault/parser-otx.py | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/intelmq/bots/parsers/alienvault/parser-otx.py b/intelmq/bots/parsers/alienvault/parser-otx.py index b91c5e173..9fcb2f5bc 100644 --- a/intelmq/bots/parsers/alienvault/parser-otx.py +++ b/intelmq/bots/parsers/alienvault/parser-otx.py @@ -9,9 +9,9 @@ from intelmq.lib.message import Event HASHES = { - 'FileHash-SHA256' : 'SHA-256', - 'FileHash-SHA1' : 'SHA-1', - 'FileHash-MD5' : 'MD5' + 'FileHash-SHA256': 'SHA-256', + 'FileHash-SHA1': 'SHA-1', + 'FileHash-MD5': 'MD5' } @@ -23,36 +23,45 @@ def process(self): len(report.value("raw").strip()) == 0): self.acknowledge_message() return - + time_observation = DateTime().generate_datetime_now() raw_report = utils.base64_decode(report.value("raw")) for pulse in json.loads(raw_report): - comment = "author: " + pulse['author_name'] + "; name: " + pulse['name'] + "; description: " + pulse['description'] + comment = "author: " + pulse['author_name'] + "; name: " + \ + pulse['name'] + "; description: " + pulse['description'] for indicator in pulse["indicators"]: event = Event() - #hashes don't work - invalid keys - #if indicator["type"] in ['FileHash-SHA256', 'FileHash-SHA1', 'FileHash-MD5']: - # event.add('source.artifact_hash', indicator["indicator"]) - # event.add('source.artifact_hash_type', HASHES[indicator["type"]], sanitize = True) - #fqdn + # fqdn if indicator["type"] in ['hostname', 'domain']: - event.add('source.fqdn', indicator["indicator"], sanitize = True) + event.add( + 'source.fqdn', + indicator["indicator"], + sanitize=True) # IP addresses elif indicator["type"] in ['IPv4', 'IPv6']: - event.add('source.ip', indicator["indicator"], sanitize = True) - #emails + event.add( + 'source.ip', + indicator["indicator"], + sanitize=True) + # emails elif indicator["type"] == 'email': - event.add('source.email_address', indicator["indicator"], sanitize = True) - #URLs + event.add( + 'source.email_address', + indicator["indicator"], + sanitize=True) + # URLs elif indicator["type"] in ['URL', 'URI']: - event.add('source.url', indicator["indicator"], sanitize = True) - #CIDR, FilePath, Mutex, CVE + event.add( + 'source.url', + indicator["indicator"], + sanitize=True) + #CIDR, FilePath, Mutex, CVE, hashes else: continue event.add('comment', comment) - event.add('classification.type', 'blacklist', sanitize = True) + event.add('classification.type', 'blacklist', sanitize=True) event.add('time.observation', time_observation, sanitize=True) event.add('feed.name', report.value("feed.name")) event.add("raw", json.dumps(indicator), sanitize=True) From edf97f32e228658ff3829fc309f18700c3ed37e6 Mon Sep 17 00:00:00 2001 From: robcza Date: Wed, 2 Sep 2015 14:19:03 +0200 Subject: [PATCH 05/12] time observation in the collector --- intelmq/bots/collectors/alienvault-otx/collector.py | 3 +++ intelmq/bots/parsers/alienvault/parser-otx.py | 6 ++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/intelmq/bots/collectors/alienvault-otx/collector.py b/intelmq/bots/collectors/alienvault-otx/collector.py index e36f8371d..f31dbe562 100644 --- a/intelmq/bots/collectors/alienvault-otx/collector.py +++ b/intelmq/bots/collectors/alienvault-otx/collector.py @@ -7,6 +7,7 @@ from intelmq.bots.collectors.http.lib import fetch_url from intelmq.lib.bot import Bot from intelmq.lib.message import Report +from intelmq.lib.harmonization import DateTime class AlienVaultOTXCollectorBot(Bot): @@ -20,6 +21,8 @@ def process(self): report = Report() report.add("raw", json.dumps(pulses), sanitize=True) report.add("feed.name", self.parameters.feed, sanitize=True) + time_observation = DateTime().generate_datetime_now() + report.add('time.observation', time_observation, sanitize=True) self.send_message(report) diff --git a/intelmq/bots/parsers/alienvault/parser-otx.py b/intelmq/bots/parsers/alienvault/parser-otx.py index dbd51dd9d..872895a0b 100644 --- a/intelmq/bots/parsers/alienvault/parser-otx.py +++ b/intelmq/bots/parsers/alienvault/parser-otx.py @@ -23,9 +23,6 @@ def process(self): self.acknowledge_message() return - time_observation = DateTime().generate_datetime_now() - raw_report = utils.base64_decode(report.value("raw")) - for pulse in json.loads(raw_report): additional_information = json.dumps( {'author':pulse['author_name'], @@ -74,7 +71,8 @@ def process(self): event.add('comment', pulse['description']) event.add('additional_information', additional_information) event.add('classification.type', 'blacklist', sanitize=True) - event.add('time.observation', time_observation, sanitize=True) + event.add('time.observation', report.value( + 'time.observation'), sanitize=True) event.add('time.source', indicator["created"], sanitize=True) event.add('feed.name', report.value("feed.name")) event.add("raw", json.dumps(indicator), sanitize=True) From 2322d00029e33c1682ae34169c3c5760505312c3 Mon Sep 17 00:00:00 2001 From: robcza Date: Wed, 2 Sep 2015 14:24:09 +0200 Subject: [PATCH 06/12] syntax error fixes --- intelmq/bots/parsers/alienvault/parser-otx.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/intelmq/bots/parsers/alienvault/parser-otx.py b/intelmq/bots/parsers/alienvault/parser-otx.py index 872895a0b..ac0f84073 100644 --- a/intelmq/bots/parsers/alienvault/parser-otx.py +++ b/intelmq/bots/parsers/alienvault/parser-otx.py @@ -19,7 +19,7 @@ class AlienVaultOTXParserBot(Bot): def process(self): report = self.receive_message() - if (report is None or not report.contains("raw"): + if (report is None or not report.contains("raw")): self.acknowledge_message() return @@ -63,7 +63,6 @@ def process(self): 'source.network', indicator["indicator"], sanitize=True) - elif indicator["type"] #FilePath, Mutex, CVE, hashes - TODO: process these IoCs as well else: continue From 844653a65e68d2bdb4b9a183e719b869b8ca9627 Mon Sep 17 00:00:00 2001 From: robcza Date: Wed, 2 Sep 2015 17:34:13 +0200 Subject: [PATCH 07/12] working collector + bot, test data provided however there is an issue with test itself --- intelmq/bots/BOTS | 4 +- .../OTXv2.py | 0 .../README.md | 0 .../__init__.py | 0 .../collector.py | 0 .../{parser-otx.py => parser_otx.py} | 40 ++- .../parsers/alienvault/test_parser_otx.data | 268 ++++++++++++++++++ .../parsers/alienvault/test_parser_otx.py | 51 ++++ 8 files changed, 350 insertions(+), 13 deletions(-) rename intelmq/bots/collectors/{alienvault-otx => alienvault_otx}/OTXv2.py (100%) rename intelmq/bots/collectors/{alienvault-otx => alienvault_otx}/README.md (100%) rename intelmq/bots/collectors/{alienvault-otx => alienvault_otx}/__init__.py (100%) rename intelmq/bots/collectors/{alienvault-otx => alienvault_otx}/collector.py (100%) rename intelmq/bots/parsers/alienvault/{parser-otx.py => parser_otx.py} (68%) create mode 100644 intelmq/tests/bots/parsers/alienvault/test_parser_otx.data create mode 100644 intelmq/tests/bots/parsers/alienvault/test_parser_otx.py diff --git a/intelmq/bots/BOTS b/intelmq/bots/BOTS index 2b1f5df32..b5e36e045 100644 --- a/intelmq/bots/BOTS +++ b/intelmq/bots/BOTS @@ -217,7 +217,7 @@ } }, "AlienVault OTX": { - "module": "intelmq.bots.collectors.alienvault-otx.collector", + "module": "intelmq.bots.collectors.alienvault_otx.collector", "description": "AlienVault OTX Collector is the bot responsible to get the report through the API. Report could vary according to subscriptions.", "parameters": { "api_key": "", @@ -645,7 +645,7 @@ }, "AlienVault OTX": { "description": "AlienVault Parser is the bot responsible to parse the report and sanitize the information.", - "module": "intelmq.bots.parsers.alienvault.parser-otx", + "module": "intelmq.bots.parsers.alienvault.parser_otx", "parameters": {} }, "Arbor": { diff --git a/intelmq/bots/collectors/alienvault-otx/OTXv2.py b/intelmq/bots/collectors/alienvault_otx/OTXv2.py similarity index 100% rename from intelmq/bots/collectors/alienvault-otx/OTXv2.py rename to intelmq/bots/collectors/alienvault_otx/OTXv2.py diff --git a/intelmq/bots/collectors/alienvault-otx/README.md b/intelmq/bots/collectors/alienvault_otx/README.md similarity index 100% rename from intelmq/bots/collectors/alienvault-otx/README.md rename to intelmq/bots/collectors/alienvault_otx/README.md diff --git a/intelmq/bots/collectors/alienvault-otx/__init__.py b/intelmq/bots/collectors/alienvault_otx/__init__.py similarity index 100% rename from intelmq/bots/collectors/alienvault-otx/__init__.py rename to intelmq/bots/collectors/alienvault_otx/__init__.py diff --git a/intelmq/bots/collectors/alienvault-otx/collector.py b/intelmq/bots/collectors/alienvault_otx/collector.py similarity index 100% rename from intelmq/bots/collectors/alienvault-otx/collector.py rename to intelmq/bots/collectors/alienvault_otx/collector.py diff --git a/intelmq/bots/parsers/alienvault/parser-otx.py b/intelmq/bots/parsers/alienvault/parser_otx.py similarity index 68% rename from intelmq/bots/parsers/alienvault/parser-otx.py rename to intelmq/bots/parsers/alienvault/parser_otx.py index ac0f84073..93d511d61 100644 --- a/intelmq/bots/parsers/alienvault/parser-otx.py +++ b/intelmq/bots/parsers/alienvault/parser_otx.py @@ -1,4 +1,9 @@ # -*- coding: utf-8 -*- +""" +; Events are gathered based on user subscriptions in AlienVault OTX +; The data structure is described in detail here: +; https://github.com/AlienVault-Labs/OTX-Python-SDK/blob/master/howto_use_python_otx_api.ipynb +""" from __future__ import unicode_literals import sys import json @@ -23,15 +28,23 @@ def process(self): self.acknowledge_message() return + raw_report = utils.base64_decode(report.value("raw")) + for pulse in json.loads(raw_report): - additional_information = json.dumps( - {'author':pulse['author_name'], - 'pulse':pulse['name']}) + additional = json.dumps( + {'author': pulse['author_name'], + 'pulse': pulse['name']}) for indicator in pulse["indicators"]: event = Event() - #hashes - if indicator["type"] in ['FileHash-SHA256', 'FileHash-SHA1', 'FileHash-MD5']: - event.add('malware.hash', indicator["indicator"], sanitize = True) + # hashes + if indicator["type"] in [ + 'FileHash-SHA256', + 'FileHash-SHA1', + 'FileHash-MD5']: + event.add( + 'malware.hash', + indicator["indicator"], + sanitize=True) # event.add('malware.hash_type', HASHES[indicator["type"]], sanitize = True) # fqdn if indicator["type"] in ['hostname', 'domain']: @@ -48,7 +61,7 @@ def process(self): # emails elif indicator["type"] == 'email': event.add( - 'source.email_address', + 'source.account', indicator["indicator"], sanitize=True) # URLs @@ -57,22 +70,27 @@ def process(self): 'source.url', indicator["indicator"], sanitize=True) - #CIDR + # CIDR elif indicator["type"] in ['CIDR']: event.add( 'source.network', indicator["indicator"], sanitize=True) - #FilePath, Mutex, CVE, hashes - TODO: process these IoCs as well + # FilePath, Mutex, CVE, hashes - TODO: process these IoCs as + # well else: continue event.add('comment', pulse['description']) - event.add('additional_information', additional_information) + #event.add('additional', additional) event.add('classification.type', 'blacklist', sanitize=True) event.add('time.observation', report.value( 'time.observation'), sanitize=True) - event.add('time.source', indicator["created"], sanitize=True) + event.add( + 'time.source', + indicator["created"][ + :-4] + "+00:00", + sanitize=True) event.add('feed.name', report.value("feed.name")) event.add("raw", json.dumps(indicator), sanitize=True) self.send_message(event) diff --git a/intelmq/tests/bots/parsers/alienvault/test_parser_otx.data b/intelmq/tests/bots/parsers/alienvault/test_parser_otx.data new file mode 100644 index 000000000..a8b5dbb65 --- /dev/null +++ b/intelmq/tests/bots/parsers/alienvault/test_parser_otx.data @@ -0,0 +1,268 @@ +[{ + "description": "Our findings show that Rocket Kitten is still active, retains a growing level of persistence, and acts ever more aggressively in terms of attack method. We also found that recent publications on the group\u2019s activity have done nothing to change their behavior or reduce their activity. They don\u2019t seem to bother to have to \u201cdisappear.\u201d With this paper, we feel fairly certain that Rocket Kitten\u2019s prime targets are not companies and political organizations as entire bodies but individuals that operate in strategically interesting fields such as diplomacy, foreign policy research, and defense-related businesses. We believe the espionage factor and political context make their attacks unique and very different from traditional targeted attacks.", + "created": "2015-09-02T09:21:53.093000", + "tags": [ + "spy kittens", + "rocket kitten", + "ghole", + "spearphishing", + "Social engineering", + "TSPY_WOOLERG", + "apt", + "trendmicro" + ], + "modified": "2015-09-02T09:22:22.976000", + "author_name": "AlienVault", + "references": [ + "http://www.trendmicro.com/cloud-content/us/pdfs/security-intelligence/white-papers/wp-the-spy-kittens-are-back.pdf" + ], + "indicators": [ + { + "indicator": "http://107.6.172.54/woolen/", + "_id": "55e6bfb14637f22cb6057466", + "type": "URL", + "description": "", + "created": "2015-09-02T09:21:53.093" + }, + { + "indicator": "af364ff503da71875b6d7c401a1e98e31450a561", + "_id": "55e6bfb14637f22cb6057467", + "type": "FileHash-SHA1", + "description": "", + "created": "2015-09-02T09:21:53.093" + }, + { + "indicator": "64ba130e627dd85c85d6534e769d239080e068dd", + "_id": "55e6bfb14637f22cb6057468", + "type": "FileHash-SHA1", + "description": "", + "created": "2015-09-02T09:21:53.093" + }, + { + "indicator": "46a995df8d9918ca0793404110904479b6adcb9f", + "_id": "55e6bfb14637f22cb6057469", + "type": "FileHash-SHA1", + "description": "", + "created": "2015-09-02T09:21:53.093" + }, + { + "indicator": "457f54e9a0f32f2648f95a8e339d9fd9aed23fa7", + "_id": "55e6bfb14637f22cb605746a", + "type": "FileHash-SHA1", + "description": "", + "created": "2015-09-02T09:21:53.093" + }, + { + "indicator": "84.11.146.62", + "_id": "55e6bfb14637f22cb605746b", + "type": "IPv4", + "description": "", + "created": "2015-09-02T09:21:53.093" + }, + { + "indicator": "107.6.172.54", + "_id": "55e6bfb14637f22cb605746c", + "type": "IPv4", + "description": "", + "created": "2015-09-02T09:21:53.093" + }, + { + "indicator": "107.6.181.116", + "_id": "55e6bfb14637f22cb605746d", + "type": "IPv4", + "description": "", + "created": "2015-09-02T09:21:53.093" + }, + { + "indicator": "29968b0c4157f226761073333ff2e82b588ddf8e", + "_id": "55e6bfce67db8c2dec9c1c89", + "type": "FileHash-SHA1", + "description": "", + "created": "2015-09-02T09:22:22.976" + }, + { + "indicator": "eeb67e663b2fa980c6b228fc2e04304c8992401d", + "_id": "55e6bfce67db8c2dec9c1c8a", + "type": "FileHash-SHA1", + "description": "", + "created": "2015-09-02T09:22:22.976" + }, + { + "indicator": "db2b8f49b4e76c2f538a3a6b222c35547c802cef", + "_id": "55e6bfce67db8c2dec9c1c8b", + "type": "FileHash-SHA1", + "description": "", + "created": "2015-09-02T09:22:22.976" + } + ], + "revision": 2.0, + "id": "55e6bfb14637f22cb605746e", + "name": "The Spy Kittens Are Back: Rocket Kitten 2" + }, + { + "description": "Myanmar is a country currently engaged in an important political process. A pro-democracy reform took place \nin 2011 which has helped the government create an atmopshere conducive to investor interest. The country is \nresource rich, with a variety of natural resources and a steady labor supply. Despite recent progress, the \ncountry is subject to ongoing conflict with ethnic rebels and an ongoing civil war. Analysts suggest that both \nChina and the United States are vying for greater influence in Myanmar, with China in particular having \ngeopolitical interest due to sea passages, port deals, and fuel pipelines that are important to its goals. \nGeopolitical analysts have suggested that the United States may have its own interests that involve thwarting \nChinese ambitions in the region. APT groups from multiple countries - including China - have been known to target organizations of strategic interest with aggressive malware-based espionage campaigns.", + "created": "2015-09-01T07:47:06.073000", + "tags": [ + "plugx", + "Myanmar", + "rat", + "Strategic\tWeb\tCompromise" + ], + "modified": "2015-09-01T07:47:06.073000", + "author_name": "AlienVault", + "references": [ + "http://pages.arbornetworks.com/rs/082-KNA-087/images/ASERT%20Threat%20Intelligence%20Brief%202015-05%20PlugX%20Threat%20Activity%20in%20Myanmar.pdf" + ], + "indicators": [ + { + "indicator": "http://www.uecmyanmar.org/dmdocuments/invitations.rar", + "_id": "55e557fa4637f21c54c1baf8", + "type": "URL", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "http://the-casgroup.com/Document/doc.zip", + "_id": "55e557fa4637f21c54c1baf9", + "type": "URL", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "http://www.uecmyanmar.org/dmdocuments/PlanProposal.rar", + "_id": "55e557fa4637f21c54c1bafa", + "type": "URL", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "http://thecasgroup.com/Document/doc/dxls.exe", + "_id": "55e557fa4637f21c54c1bafb", + "type": "URL", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "appeur.gnway.cc", + "_id": "55e557fa4637f21c54c1bafc", + "type": "hostname", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "webhttps.websecexp.com", + "_id": "55e557fa4637f21c54c1bafd", + "type": "hostname", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "usafbi.websecexp.com", + "_id": "55e557fa4637f21c54c1bafe", + "type": "hostname", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "usacia.websecexp.com", + "_id": "55e557fa4637f21c54c1baff", + "type": "hostname", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "e2eddf6e7233ab52ad29d8f63b1727cd", + "_id": "55e557fa4637f21c54c1bb00", + "type": "FileHash-MD5", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "78a9897344d756701d4674c7f559610a", + "_id": "55e557fa4637f21c54c1bb01", + "type": "FileHash-MD5", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "eeb631127f1b9fb3d13d209d8e675634", + "_id": "55e557fa4637f21c54c1bb02", + "type": "FileHash-MD5", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "5ee5df9a5f4d16de3f880740db884f69", + "_id": "55e557fa4637f21c54c1bb03", + "type": "FileHash-MD5", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "1e36a853bc0b1d111ce726a508bc1a86", + "_id": "55e557fa4637f21c54c1bb04", + "type": "FileHash-MD5", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "a1c0c364e02b3b1e0e7b8ce89b611b53", + "_id": "55e557fa4637f21c54c1bb05", + "type": "FileHash-MD5", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "69754b86021d3daa658da15579b8f08a", + "_id": "55e557fa4637f21c54c1bb06", + "type": "FileHash-MD5", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "a30262bf36b3023ef717b6e23e21bd30", + "_id": "55e557fa4637f21c54c1bb07", + "type": "FileHash-MD5", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "9aceefb76c2e227c651ef6a035461b5c", + "_id": "55e557fa4637f21c54c1bb08", + "type": "FileHash-MD5", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "d0c5410140c15c8d148437f0f7eabcf7", + "_id": "55e557fa4637f21c54c1bb09", + "type": "FileHash-MD5", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "d055518ad14f3d6c40aa6ced6a2d05f2", + "_id": "55e557fa4637f21c54c1bb0a", + "type": "FileHash-MD5", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "532f4c671a19145cf19c34d18138da63", + "_id": "55e557fa4637f21c54c1bb0b", + "type": "FileHash-MD5", + "description": "", + "created": "2015-09-01T07:47:06.073" + }, + { + "indicator": "809976f3aa0ffd6860056be3b66d5092", + "_id": "55e557fa4637f21c54c1bb0c", + "type": "FileHash-MD5", + "description": "", + "created": "2015-09-01T07:47:06.073" + } + ], + "revision": 1.0, + "id": "55e557fa4637f21c54c1bb0d", + "name": "PlugX Threat\tActivity in Myanmar" +}] diff --git a/intelmq/tests/bots/parsers/alienvault/test_parser_otx.py b/intelmq/tests/bots/parsers/alienvault/test_parser_otx.py new file mode 100644 index 000000000..6c41ae637 --- /dev/null +++ b/intelmq/tests/bots/parsers/alienvault/test_parser_otx.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import json +import os +import unittest + +import intelmq.lib.test as test +import intelmq.lib.utils as utils +from intelmq.bots.parsers.alienvault.parser_otx import \ + AlienVaultOTXParserBot + +with open(os.path.join(os.path.dirname(__file__), 'test_parser_otx.data')) as handle: + EXAMPLE_FILE = handle.read() + +EXAMPLE_REPORT = {"feed.name": "AlienVault OTX", + "raw": utils.base64_encode(EXAMPLE_FILE), + "__type": "Report", + "time.observation": "2015-09-02T14:17:58+00:00" + } +EXAMPLE_EVENT = { + "comment": "Our findings show that Rocket Kitten is still active, retains a growing level of persistence, and acts ever more aggressively in terms of attack method. We also found that recent publications on the group’s activity have done nothing to change their behavior or reduce their activity. They don’t seem to bother to have to “disappear.” With this paper, we feel fairly certain that Rocket Kitten’s prime targets are not companies and political organizations as entire bodies but individuals that operate in strategically interesting fields such as diplomacy, foreign policy research, and defense-related businesses. We believe the espionage factor and political context make their attacks unique and very different from traditional targeted attacks.", + "feed": { + "name": "AlienVault OTX"}, + "classification": { + "type": "blacklist"}, + "source": { + "url": "http://107.6.172.54/woolen/"}, + "raw": "eyJpbmRpY2F0b3IiOiAiaHR0cDovLzEwNy42LjE3Mi41NC93b29sZW4vIiwgIl9pZCI6ICI1NWU2YmZiMTQ2MzdmMjJjYjYwNTc0NjYiLCAidHlwZSI6ICJVUkwiLCAiZGVzY3JpcHRpb24iOiAiIiwgImNyZWF0ZWQiOiAiMjAxNS0wOS0wMlQwOToyMTo1My4wOTMifQ==", + "time": { + "source": "2015-09-02T09:21:53+00:00", + "observation": "2015-09-02T14:17:58+00:00"}} + + +class TestAlienVaultOTXParserBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for AlienVaultOTXParserBot. + """ + + @classmethod + def set_bot(self): + self.bot_reference = AlienVaultOTXParserBot + self.default_input_message = json.dumps(EXAMPLE_REPORT) + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() + self.assertMessageEqual(0, EXAMPLE_EVENT) + +if __name__ == '__main__': + unittest.main() From b9c2e16af4ac3740c30e5b9f0086e7c868e0274f Mon Sep 17 00:00:00 2001 From: robcza Date: Wed, 2 Sep 2015 17:39:59 +0200 Subject: [PATCH 08/12] license for the AlienVault OTX SDK --- intelmq/bots/collectors/alienvault_otx/COPYRIGHT.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 intelmq/bots/collectors/alienvault_otx/COPYRIGHT.md diff --git a/intelmq/bots/collectors/alienvault_otx/COPYRIGHT.md b/intelmq/bots/collectors/alienvault_otx/COPYRIGHT.md new file mode 100644 index 000000000..0cb85f3c9 --- /dev/null +++ b/intelmq/bots/collectors/alienvault_otx/COPYRIGHT.md @@ -0,0 +1,2 @@ +Contents of the file "OTXv2.py" are licensed under the Apache license and originally taken from AlienVault Labs: https://github.com/AlienVault-Labs/OTX-Python-SDK +Complete license TERMS AND CONDITIONS: https://github.com/AlienVault-Labs/OTX-Python-SDK/blob/master/LICENSE From cadc76a9563715da15c97be57ba61edc59e3635a Mon Sep 17 00:00:00 2001 From: robcza Date: Wed, 9 Sep 2015 15:49:04 +0200 Subject: [PATCH 09/12] fixed alienvault otx parser + tests --- intelmq/bots/parsers/alienvault/parser_otx.py | 5 ++--- .../parsers/alienvault/test_parser_otx.data | 2 +- .../bots/parsers/alienvault/test_parser_otx.py | 17 ++++++++--------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/intelmq/bots/parsers/alienvault/parser_otx.py b/intelmq/bots/parsers/alienvault/parser_otx.py index 93d511d61..ef43de11d 100644 --- a/intelmq/bots/parsers/alienvault/parser_otx.py +++ b/intelmq/bots/parsers/alienvault/parser_otx.py @@ -32,8 +32,7 @@ def process(self): for pulse in json.loads(raw_report): additional = json.dumps( - {'author': pulse['author_name'], - 'pulse': pulse['name']}) + {"author": pulse['author_name'], "pulse": pulse['name']}) for indicator in pulse["indicators"]: event = Event() # hashes @@ -82,7 +81,7 @@ def process(self): continue event.add('comment', pulse['description']) - #event.add('additional', additional) + event.add('additional', additional, sanitize=True) event.add('classification.type', 'blacklist', sanitize=True) event.add('time.observation', report.value( 'time.observation'), sanitize=True) diff --git a/intelmq/tests/bots/parsers/alienvault/test_parser_otx.data b/intelmq/tests/bots/parsers/alienvault/test_parser_otx.data index a8b5dbb65..73b53607f 100644 --- a/intelmq/tests/bots/parsers/alienvault/test_parser_otx.data +++ b/intelmq/tests/bots/parsers/alienvault/test_parser_otx.data @@ -1,5 +1,5 @@ [{ - "description": "Our findings show that Rocket Kitten is still active, retains a growing level of persistence, and acts ever more aggressively in terms of attack method. We also found that recent publications on the group\u2019s activity have done nothing to change their behavior or reduce their activity. They don\u2019t seem to bother to have to \u201cdisappear.\u201d With this paper, we feel fairly certain that Rocket Kitten\u2019s prime targets are not companies and political organizations as entire bodies but individuals that operate in strategically interesting fields such as diplomacy, foreign policy research, and defense-related businesses. We believe the espionage factor and political context make their attacks unique and very different from traditional targeted attacks.", + "description": "Our findings show that Rocket Kitten is still active, retains a growing level of persistence, and acts ever more aggressively in terms of attack method. We also found that recent publications on the group’s activity have done nothing to change their behavior or reduce their activity. They don’t seem to bother to have to “disappear.” With this paper, we feel fairly certain that Rocket Kitten’s prime targets are not companies and political organizations as entire bodies but individuals that operate in strategically interesting fields such as diplomacy, foreign policy research, and defense-related businesses. We believe the espionage factor and political context make their attacks unique and very different from traditional targeted attacks.", "created": "2015-09-02T09:21:53.093000", "tags": [ "spy kittens", diff --git a/intelmq/tests/bots/parsers/alienvault/test_parser_otx.py b/intelmq/tests/bots/parsers/alienvault/test_parser_otx.py index 6c41ae637..3360c5cc2 100644 --- a/intelmq/tests/bots/parsers/alienvault/test_parser_otx.py +++ b/intelmq/tests/bots/parsers/alienvault/test_parser_otx.py @@ -19,17 +19,16 @@ "time.observation": "2015-09-02T14:17:58+00:00" } EXAMPLE_EVENT = { + "__type": "Event", + "additional": '{"pulse": "The Spy Kittens Are Back: Rocket Kitten 2", "author": "AlienVault"}', "comment": "Our findings show that Rocket Kitten is still active, retains a growing level of persistence, and acts ever more aggressively in terms of attack method. We also found that recent publications on the group’s activity have done nothing to change their behavior or reduce their activity. They don’t seem to bother to have to “disappear.” With this paper, we feel fairly certain that Rocket Kitten’s prime targets are not companies and political organizations as entire bodies but individuals that operate in strategically interesting fields such as diplomacy, foreign policy research, and defense-related businesses. We believe the espionage factor and political context make their attacks unique and very different from traditional targeted attacks.", - "feed": { - "name": "AlienVault OTX"}, - "classification": { - "type": "blacklist"}, - "source": { - "url": "http://107.6.172.54/woolen/"}, + "feed.name": "AlienVault OTX", + "classification.type": "blacklist", + "source.url": "http://107.6.172.54/woolen/", "raw": "eyJpbmRpY2F0b3IiOiAiaHR0cDovLzEwNy42LjE3Mi41NC93b29sZW4vIiwgIl9pZCI6ICI1NWU2YmZiMTQ2MzdmMjJjYjYwNTc0NjYiLCAidHlwZSI6ICJVUkwiLCAiZGVzY3JpcHRpb24iOiAiIiwgImNyZWF0ZWQiOiAiMjAxNS0wOS0wMlQwOToyMTo1My4wOTMifQ==", - "time": { - "source": "2015-09-02T09:21:53+00:00", - "observation": "2015-09-02T14:17:58+00:00"}} + "time.source": "2015-09-02T09:21:53+00:00", + "time.observation": "2015-09-02T14:17:58+00:00" +} class TestAlienVaultOTXParserBot(test.BotTestCase, unittest.TestCase): From 7e3c9cb89a9fa67fcfbd07a79cf853117cc520b4 Mon Sep 17 00:00:00 2001 From: robcza Date: Wed, 9 Sep 2015 16:04:03 +0200 Subject: [PATCH 10/12] malware.hash_type added to harmonization --- intelmq/bots/parsers/alienvault/parser_otx.py | 4 +++- intelmq/conf/harmonization.conf | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/intelmq/bots/parsers/alienvault/parser_otx.py b/intelmq/bots/parsers/alienvault/parser_otx.py index ef43de11d..ec2043ee7 100644 --- a/intelmq/bots/parsers/alienvault/parser_otx.py +++ b/intelmq/bots/parsers/alienvault/parser_otx.py @@ -44,7 +44,9 @@ def process(self): 'malware.hash', indicator["indicator"], sanitize=True) - # event.add('malware.hash_type', HASHES[indicator["type"]], sanitize = True) + event.add( + 'malware.hash_type', HASHES[ + indicator["type"]], sanitize=True) # fqdn if indicator["type"] in ['hostname', 'domain']: event.add( diff --git a/intelmq/conf/harmonization.conf b/intelmq/conf/harmonization.conf index aba2ba3d8..2014e8bf7 100644 --- a/intelmq/conf/harmonization.conf +++ b/intelmq/conf/harmonization.conf @@ -148,6 +148,10 @@ "regex": "^[a-fA-F0-9]+$", "type": "String" }, + "malware.hash_type": { + "description": "hashing algorithm used to produce the malware.hash", + "type": "String" + }, "malware.name": { "description": "A malware family name in lower case.", "regex": "[a-z ]+", From 34b9628b7b43be124499022add4e20e2a68717af Mon Sep 17 00:00:00 2001 From: robcza Date: Wed, 9 Sep 2015 20:06:00 +0200 Subject: [PATCH 11/12] malware.hash harmonization and several fixes proposed by @sebix --- .../collectors/alienvault_otx/collector.py | 1 - intelmq/bots/parsers/alienvault/parser_otx.py | 57 ++++++------------- intelmq/conf/harmonization.conf | 6 +- 3 files changed, 19 insertions(+), 45 deletions(-) diff --git a/intelmq/bots/collectors/alienvault_otx/collector.py b/intelmq/bots/collectors/alienvault_otx/collector.py index f31dbe562..a0189623e 100644 --- a/intelmq/bots/collectors/alienvault_otx/collector.py +++ b/intelmq/bots/collectors/alienvault_otx/collector.py @@ -4,7 +4,6 @@ from OTXv2 import OTXv2 import json -from intelmq.bots.collectors.http.lib import fetch_url from intelmq.lib.bot import Bot from intelmq.lib.message import Report from intelmq.lib.harmonization import DateTime diff --git a/intelmq/bots/parsers/alienvault/parser_otx.py b/intelmq/bots/parsers/alienvault/parser_otx.py index ec2043ee7..792c2c600 100644 --- a/intelmq/bots/parsers/alienvault/parser_otx.py +++ b/intelmq/bots/parsers/alienvault/parser_otx.py @@ -14,12 +14,11 @@ from intelmq.lib.message import Event HASHES = { - 'FileHash-SHA256': 'SHA-256', - 'FileHash-SHA1': 'SHA-1', - 'FileHash-MD5': 'MD5' + 'FileHash-SHA256': '$5$', + 'FileHash-SHA1': '$sha1$', + 'FileHash-MD5': '$1$' } - class AlienVaultOTXParserBot(Bot): def process(self): @@ -36,49 +35,30 @@ def process(self): for indicator in pulse["indicators"]: event = Event() # hashes - if indicator["type"] in [ - 'FileHash-SHA256', - 'FileHash-SHA1', - 'FileHash-MD5']: - event.add( - 'malware.hash', - indicator["indicator"], - sanitize=True) - event.add( - 'malware.hash_type', HASHES[ - indicator["type"]], sanitize=True) + if indicator["type"] in HASHES.keys(): + event.add('malware.hash', + HASHES[indicator["type"]] + indicator["indicator"]) # fqdn if indicator["type"] in ['hostname', 'domain']: - event.add( - 'source.fqdn', - indicator["indicator"], - sanitize=True) + event.add('source.fqdn', + indicator["indicator"], sanitize=True) # IP addresses elif indicator["type"] in ['IPv4', 'IPv6']: - event.add( - 'source.ip', - indicator["indicator"], - sanitize=True) + event.add('source.ip', + indicator["indicator"], sanitize=True) # emails elif indicator["type"] == 'email': - event.add( - 'source.account', - indicator["indicator"], - sanitize=True) + event.add('source.account', + indicator["indicator"], sanitize=True) # URLs elif indicator["type"] in ['URL', 'URI']: - event.add( - 'source.url', - indicator["indicator"], - sanitize=True) + event.add('source.url', + indicator["indicator"], sanitize=True) # CIDR elif indicator["type"] in ['CIDR']: - event.add( - 'source.network', - indicator["indicator"], - sanitize=True) - # FilePath, Mutex, CVE, hashes - TODO: process these IoCs as - # well + event.add('source.network', + indicator["indicator"], sanitize=True) + # FilePath, Mutex, CVE - TODO: process these IoCs as well else: continue @@ -89,8 +69,7 @@ def process(self): 'time.observation'), sanitize=True) event.add( 'time.source', - indicator["created"][ - :-4] + "+00:00", + indicator["created"][:-4] + "+00:00", sanitize=True) event.add('feed.name', report.value("feed.name")) event.add("raw", json.dumps(indicator), sanitize=True) diff --git a/intelmq/conf/harmonization.conf b/intelmq/conf/harmonization.conf index 2014e8bf7..854140150 100644 --- a/intelmq/conf/harmonization.conf +++ b/intelmq/conf/harmonization.conf @@ -144,14 +144,10 @@ "type": "URL" }, "malware.hash": { - "description": "A string depicting a checksum for a file, be it a malware sample for example.", + "description": "A string depicting a checksum for a file, be it a malware sample for example. Includes hash type according to https://en.wikipedia.org/wiki/Crypt_%28C%29", "regex": "^[a-fA-F0-9]+$", "type": "String" }, - "malware.hash_type": { - "description": "hashing algorithm used to produce the malware.hash", - "type": "String" - }, "malware.name": { "description": "A malware family name in lower case.", "regex": "[a-z ]+", From d2f8e3db822d6df198abd3e1ea29e3b523a7ba6e Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Fri, 11 Sep 2015 09:26:37 +0200 Subject: [PATCH 12/12] BUG: Adapt alienvault otx to current master Signed-off-by: Sebastian Wagner --- intelmq/bots/parsers/alienvault/parser_otx.py | 39 +++++++++---------- .../parsers/alienvault/test_parser_otx.py | 24 ++++++++++-- 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/intelmq/bots/parsers/alienvault/parser_otx.py b/intelmq/bots/parsers/alienvault/parser_otx.py index 792c2c600..ebfd68db1 100644 --- a/intelmq/bots/parsers/alienvault/parser_otx.py +++ b/intelmq/bots/parsers/alienvault/parser_otx.py @@ -1,16 +1,17 @@ # -*- coding: utf-8 -*- """ -; Events are gathered based on user subscriptions in AlienVault OTX -; The data structure is described in detail here: -; https://github.com/AlienVault-Labs/OTX-Python-SDK/blob/master/howto_use_python_otx_api.ipynb +Events are gathered based on user subscriptions in AlienVault OTX +The data structure is described in detail here: +https://github.com/AlienVault-Labs/OTX-Python-SDK/blob/master/ +howto_use_python_otx_api.ipynb """ from __future__ import unicode_literals -import sys + import json +import sys from intelmq.lib import utils from intelmq.lib.bot import Bot -from intelmq.lib.harmonization import DateTime from intelmq.lib.message import Event HASHES = { @@ -19,25 +20,27 @@ 'FileHash-MD5': '$1$' } + class AlienVaultOTXParserBot(Bot): def process(self): report = self.receive_message() - if (report is None or not report.contains("raw")): + if report is None or not report.contains("raw"): self.acknowledge_message() return raw_report = utils.base64_decode(report.value("raw")) for pulse in json.loads(raw_report): - additional = json.dumps( - {"author": pulse['author_name'], "pulse": pulse['name']}) + additional = json.dumps({"author": pulse['author_name'], + "pulse": pulse['name']}, + sort_keys=True) for indicator in pulse["indicators"]: - event = Event() + event = Event(report) # hashes if indicator["type"] in HASHES.keys(): - event.add('malware.hash', - HASHES[indicator["type"]] + indicator["indicator"]) + event.add('malware.hash', HASHES[indicator["type"]] + + indicator["indicator"]) # fqdn if indicator["type"] in ['hostname', 'domain']: event.add('source.fqdn', @@ -63,16 +66,12 @@ def process(self): continue event.add('comment', pulse['description']) - event.add('additional', additional, sanitize=True) + event.add('extra', additional, sanitize=True) event.add('classification.type', 'blacklist', sanitize=True) - event.add('time.observation', report.value( - 'time.observation'), sanitize=True) - event.add( - 'time.source', - indicator["created"][:-4] + "+00:00", - sanitize=True) - event.add('feed.name', report.value("feed.name")) - event.add("raw", json.dumps(indicator), sanitize=True) + event.add('time.source', indicator["created"][:-4] + "+00:00", + sanitize=True) + event.add("raw", json.dumps(indicator, sort_keys=True), + sanitize=True) self.send_message(event) self.acknowledge_message() diff --git a/intelmq/tests/bots/parsers/alienvault/test_parser_otx.py b/intelmq/tests/bots/parsers/alienvault/test_parser_otx.py index 3360c5cc2..6f00efb47 100644 --- a/intelmq/tests/bots/parsers/alienvault/test_parser_otx.py +++ b/intelmq/tests/bots/parsers/alienvault/test_parser_otx.py @@ -10,7 +10,9 @@ from intelmq.bots.parsers.alienvault.parser_otx import \ AlienVaultOTXParserBot -with open(os.path.join(os.path.dirname(__file__), 'test_parser_otx.data')) as handle: + +with open(os.path.join(os.path.dirname(__file__), + 'test_parser_otx.data')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_REPORT = {"feed.name": "AlienVault OTX", @@ -20,12 +22,26 @@ } EXAMPLE_EVENT = { "__type": "Event", - "additional": '{"pulse": "The Spy Kittens Are Back: Rocket Kitten 2", "author": "AlienVault"}', - "comment": "Our findings show that Rocket Kitten is still active, retains a growing level of persistence, and acts ever more aggressively in terms of attack method. We also found that recent publications on the group’s activity have done nothing to change their behavior or reduce their activity. They don’t seem to bother to have to “disappear.” With this paper, we feel fairly certain that Rocket Kitten’s prime targets are not companies and political organizations as entire bodies but individuals that operate in strategically interesting fields such as diplomacy, foreign policy research, and defense-related businesses. We believe the espionage factor and political context make their attacks unique and very different from traditional targeted attacks.", + "extra": '{"author": "AlienVault", "pulse": "The Spy Kittens Are Back: ' + 'Rocket Kitten 2"}', + "comment": """Our findings show that Rocket Kitten is still active, retains +a growing level of persistence, and acts ever more aggressively in terms of +attack method. We also found that recent publications on the group’s activity +have done nothing to change their behavior or reduce their activity. They don’t +seem to bother to have to “disappear.” With this paper, we feel fairly certain +that Rocket Kitten’s prime targets are not companies and political +organizations as entire bodies but individuals that operate in strategically +interesting fields such as diplomacy, foreign policy research, and +defense-related businesses. We believe the espionage factor and political +context make their attacks unique and very different from traditional targeted +attacks.""".replace('\n', ' '), "feed.name": "AlienVault OTX", "classification.type": "blacklist", "source.url": "http://107.6.172.54/woolen/", - "raw": "eyJpbmRpY2F0b3IiOiAiaHR0cDovLzEwNy42LjE3Mi41NC93b29sZW4vIiwgIl9pZCI6ICI1NWU2YmZiMTQ2MzdmMjJjYjYwNTc0NjYiLCAidHlwZSI6ICJVUkwiLCAiZGVzY3JpcHRpb24iOiAiIiwgImNyZWF0ZWQiOiAiMjAxNS0wOS0wMlQwOToyMTo1My4wOTMifQ==", + "raw": "eyJfaWQiOiAiNTVlNmJmYjE0NjM3ZjIyY2I2MDU3NDY2IiwgImNyZWF0ZWQiOiAiMj" + "AxNS0wOS0wMlQwOToyMTo1My4wOTMiLCAiZGVzY3JpcHRpb24iOiAiIiwgImluZGlj" + "YXRvciI6ICJodHRwOi8vMTA3LjYuMTcyLjU0L3dvb2xlbi8iLCAidHlwZSI6ICJVUk" + "wifQ==", "time.source": "2015-09-02T09:21:53+00:00", "time.observation": "2015-09-02T14:17:58+00:00" }