Merge pull request #651 from sebix/collectorbot

Create and Use CollectorBot

intelmq/bin/intelmqctl.py

@@ -333,7 +333,7 @@ def bot_run(self, bot_id):
  botname = [name for name in dir(module)
  if hasattr(getattr(module, name), 'process') and
  name.endswith('Bot') and
- name != 'ParserBot'][0]
+ name not in ['CollectorBot', 'ParserBot']][0]
  bot = getattr(module, botname)
  instance = bot(bot_id)
  instance.start()

intelmq/bots/collectors/alienvault_otx/collector.py

@@ -2,13 +2,13 @@
 import json
 import sys
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import CollectorBot
 from intelmq.lib.message import Report
 
 from .OTXv2 import OTXv2
 
 
-class AlienVaultOTXCollectorBot(Bot):
+class AlienVaultOTXCollectorBot(CollectorBot):
 
  def process(self):
  self.logger.info("Downloading report through API")
@@ -19,8 +19,6 @@ def process(self):
 
  report = Report()
  report.add("raw", json.dumps(pulses))
- report.add("feed.name", self.parameters.feed)
- report.add("feed.accuracy", self.parameters.accuracy)
  self.send_message(report)
 
 

intelmq/bots/collectors/bitsight/collector.py

@@ -1,13 +1,13 @@
 # -*- coding: utf-8 -*-
 import sys
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import CollectorBot
 from intelmq.lib.message import Report
 
 import pycurl
 
 
-class BitsightCollectorBot(Bot):
+class BitsightCollectorBot(CollectorBot):
 
  def init(self):
  self.logger.info("Connecting to BitSightTech stream server")
@@ -35,8 +35,6 @@ def on_receive(self, data):
 
  report = Report()
  report.add("raw", line)
- report.add("feed.name", self.parameters.feed)
- report.add("feed.accuracy", self.parameters.accuracy)
  report.add("feed.url", self.parameters.http_url)
 
  self.send_message(report)

intelmq/bots/collectors/blueliv/collector_crimeserver.py

@@ -3,12 +3,12 @@
 import logging
 import sys
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import CollectorBot
 from intelmq.lib.message import Report
 from sdk.blueliv_api import BluelivAPI
 
 
-class BluelivCrimeserverCollectorBot(Bot):
+class BluelivCrimeserverCollectorBot(CollectorBot):
 
  def process(self):
  self.logger.info("Downloading report through API")
@@ -28,8 +28,6 @@ def process(self):
 
  report = Report()
  report.add("raw", json.dumps([item for item in response.items]))
- report.add("feed.name", self.parameters.feed)
- report.add("feed.accuracy", self.parameters.accuracy)
  self.send_message(report)
 
 

intelmq/bots/collectors/file/collector_file.py

@@ -21,11 +21,11 @@
 import sys
 
 import intelmq.lib.exceptions as exceptions
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import CollectorBot
 from intelmq.lib.message import Report
 
 
-class FileCollectorBot(Bot):
+class FileCollectorBot(CollectorBot):
 
  def init(self):
  # Test if path is a directory
@@ -59,9 +59,7 @@ def process(self):
 
  report = Report()
  report.add("raw", f.read())
- report.add("feed.name", self.parameters.feed)
  report.add("feed.url", "file://localhost%s" % filename)
- report.add("feed.accuracy", self.parameters.accuracy)
  self.send_message(report)
 
  if self.parameters.delete_file:

intelmq/bots/collectors/ftp/collector_ftp.py

@@ -13,17 +13,15 @@
 from __future__ import unicode_literals
 import sys
 from ftplib import FTP
-import socket
 import zipfile
 import io
 import fnmatch
 
-from intelmq.lib.bot import Bot
-from intelmq.lib.harmonization import DateTime
+from intelmq.lib.bot import CollectorBot
 from intelmq.lib.message import Report
 
 
-class FTPCollectorBot(Bot):
+class FTPCollectorBot(CollectorBot):
  def process(self):
  self.logger.info("Downloading report from %s" %
  self.parameters.ftp_host + ':' +
@@ -77,10 +75,8 @@ def process(self):
  for raw_report in raw_reports:
  report = Report()
  report.add("raw", raw_report, sanitize=True)
- report.add("feed.name", self.parameters.feed, sanitize=True)
  report.add("feed.url", 'ftp://' + self.parameters.ftp_host + ':' +
  str(self.parameters.ftp_port), sanitize=True)
- report.add("feed.accuracy", self.parameters.accuracy, sanitize=True)
  self.send_message(report)
 
 

intelmq/bots/collectors/ftp/collector_ftps.py

@@ -20,8 +20,7 @@
 import io
 import fnmatch
 
-from intelmq.lib.bot import Bot
-from intelmq.lib.harmonization import DateTime
+from intelmq.lib.bot import CollectorBot
 from intelmq.lib.message import Report
 
 
@@ -56,7 +55,7 @@ def connect(self, host='', port=0, timeout=-999):
 # END content from Stack Overflow
 
 
-class FTPSCollectorBot(Bot):
+class FTPSCollectorBot(CollectorBot):
  def process(self):
  self.logger.info("Downloading report from %s" %
  self.parameters.ftps_host + ':' +
@@ -112,10 +111,8 @@ def process(self):
  for raw_report in raw_reports:
  report = Report()
  report.add("raw", raw_report, sanitize=True)
- report.add("feed.name", self.parameters.feed, sanitize=True)
  report.add("feed.url", 'ftps://' + self.parameters.ftps_host + ':' +
  str(self.parameters.ftps_port), sanitize=True)
- report.add("feed.accuracy", self.parameters.accuracy, sanitize=True)
  self.send_message(report)
 
 

intelmq/bots/collectors/http/collector_http.py

@@ -18,11 +18,11 @@
 
 import requests
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import CollectorBot
 from intelmq.lib.message import Report
 
 
-class HTTPCollectorBot(Bot):
+class HTTPCollectorBot(CollectorBot):
 
  def init(self):
  self.http_header = getattr(self.parameters, 'http_header', {})
@@ -73,9 +73,7 @@ def process(self):
  for raw_report in raw_reports:
  report = Report()
  report.add("raw", raw_report)
- report.add("feed.name", self.parameters.feed)
  report.add("feed.url", self.parameters.http_url)
- report.add("feed.accuracy", self.parameters.accuracy)
  self.send_message(report)
 
 

intelmq/bots/collectors/http/collector_http_stream.py

@@ -2,11 +2,11 @@
 import sys
 
 import pycurl
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import CollectorBot
 from intelmq.lib.message import Report
 
 
-class HTTPStreamCollectorBot(Bot):
+class HTTPStreamCollectorBot(CollectorBot):
 
  def init(self):
  self.conn = pycurl.Curl()
@@ -25,8 +25,6 @@ def on_receive(self, data):
 
  report = Report()
  report.add("raw", str(line))
- report.add("feed.name", self.parameters.feed)
- report.add("feed.accuracy", self.parameters.accuracy)
  self.send_message(report)
 
 

intelmq/bots/collectors/mail/collector_mail_attach.py

@@ -7,13 +7,12 @@
  import imbox
 except ImportError:
  imbox = None
-import requests
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import CollectorBot
 from intelmq.lib.message import Report
 
 
-class MailAttachCollectorBot(Bot):
+class MailAttachCollectorBot(CollectorBot):
 
  def init(self):
  if imbox is None:
@@ -55,8 +54,6 @@ def process(self):
 
  report = Report()
  report.add("raw", raw_report)
- report.add("feed.name", self.parameters.feed)
- report.add("feed.accuracy", self.parameters.accuracy)
 
  self.send_message(report)
 

intelmq/bots/collectors/mail/collector_mail_url.py

@@ -9,11 +9,11 @@
  imbox = None
 import requests
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import CollectorBot
 from intelmq.lib.message import Report
 
 
-class MailURLCollectorBot(Bot):
+class MailURLCollectorBot(CollectorBot):
 
  def init(self):
  if imbox is None:
@@ -79,9 +79,6 @@ def process(self):
 
  report = Report()
  report.add("raw", resp.content)
- report.add("feed.name",
- self.parameters.feed)
- report.add("feed.accuracy", self.parameters.accuracy)
  self.send_message(report)
 
  # Only mark read if message relevant to this instance,

intelmq/bots/collectors/misp/collector.py

@@ -16,11 +16,11 @@
 
 from pymisp import PyMISP
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import CollectorBot
 from intelmq.lib.message import Report
 
 
-class MISPCollectorBot(Bot):
+class MISPCollectorBot(CollectorBot):
 
  def init(self):
  # Initialise MISP connection
@@ -50,9 +50,7 @@ def process(self):
  # Send the results to the parser
  report = Report()
  report.add('raw', json.dumps(misp_event, sort_keys=True))
- report.add('feed.name', self.parameters.feed)
  report.add('feed.url', self.parameters.misp_url)
- report.add('feed.accuracy', self.parameters.accuracy)
  self.send_message(report)
 
  # Finally, update the tags on the MISP events.

intelmq/bots/collectors/n6/collector_stomp.py

@@ -2,7 +2,7 @@
 import os.path
 import sys
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import CollectorBot
 from intelmq.lib.message import Report
 
 try:
@@ -32,7 +32,6 @@ def on_message(self, headers, message):
  '{!r}...'.format(message[:500]))
  report = Report()
  report.add("raw", message.rstrip())
- report.add("feed.name", self.n6stomper.parameters.feed)
  report.add("feed.url", "stomp://" +
  self.n6stomper.parameters.server +
  ":" + str(self.n6stomper.parameters.port) +
@@ -41,7 +40,7 @@ def on_message(self, headers, message):
  self.n6stomper.logger.debug('Receiving Message.')
 
 
-class n6stompCollectorBot(Bot):
+class n6stompCollectorBot(CollectorBot):
  """ main class for the n6 STOMP protocol collector """
 
  def init(self):

intelmq/bots/collectors/rt/collector_rt.py

@@ -5,13 +5,13 @@
 import sys
 import zipfile
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import CollectorBot
 from intelmq.lib.message import Report
 
 import rt
 
 
-class RTCollectorBot(Bot):
+class RTCollectorBot(CollectorBot):
 
  def init(self):
  self.http_header = getattr(self.parameters, 'http_header', {})
@@ -84,9 +84,6 @@ def process(self):
  report = Report()
  report.add("raw", raw, sanitize=True)
  report.add("rtir_id", ticket_id, sanitize=True)
- report.add("feed.name", self.parameters.feed, sanitize=True)
- report.add("feed.accuracy", self.parameters.accuracy,
- sanitize=True)
  report.add("time.observation", created + ' UTC', force=True)
  self.send_message(report)
 

intelmq/bots/collectors/xmpp/collector.py

@@ -3,15 +3,15 @@
 import json
 import sys
 
-from intelmq.lib.bot import Bot
+from intelmq.lib.bot import CollectorBot
 from intelmq.bots.outputs.xmpp.output import XMPPBot
 try:
  import sleekxmpp
 except ImportError:
  sleekxmpp = None
 
 
-class XMPPCollectorBot(Bot):
+class XMPPCollectorBot(CollectorBot):
 
  xmpp = None
 

intelmq/bots/parsers/shadowserver/parser.py

@@ -43,6 +43,9 @@ def init(self):
  if self.parameters.override:
  self.override = True
 
+ # Already warned about deprecation
+ self.depr_warning = False
+
  def parse(self, report):
  raw_report = utils.base64_decode(report["raw"])
  csvr = csv.DictReader(io.StringIO(raw_report))
@@ -156,6 +159,13 @@ def parse_line(self, row, report):
 
  # Now add additional constant fields.
  dict.update(event, conf.get('constant_fields', {})) # TODO: rewrite in 1.0
+ if 'feed.code' in conf.get('constant_fields', {}).keys() and not self.depr_warning:
+ self.depr_warning = True
+ # could not get this working with logging.captureWarnings(True) :(
+ # TODO: remove from config 1.0
+ self.logger.warn('`feed.code` will be removed from the constant_fields in '
+ 'default config in favor of the `code` parameter in collectors '
+ 'in 1.0!')
 
  self.logger.debug("Raw_line: {!r}.".format(row))
  event.add('raw', self.recover_line(row))