Merge pull request #7 from wagner-certat/my-CZ-NIC_mail_collector_patch

ENH+DOC: mail attach collector: filename parsing
CZ-NIC · Dec 11, 2017 · 9fbc916 · 9fbc916
2 parents ffe6688 + 126ddc7
commit 9fbc916
Show file tree

Hide file tree

Showing 9 changed files with 34 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -65,6 +65,9 @@ Support for Python 3.3 has been dropped, it reached its end of life.
 - warnings of bots are catched by the logger (#1074)
 
 ### Bots
+#### Collectors
+- bots.collectors.mail.collector_mail_attach: Support attachment file parsing for imbox versions newer than 0.9.5
+
 #### Parsers
 - All CSV parsers ignore NULL-bytes now, because the csv-library cannot handle it (#967)
 - Modify Bot default ruleset: changed conficker rule to catch more spellings

diff --git a/intelmq/bots/collectors/mail/collector_mail_attach.py b/intelmq/bots/collectors/mail/collector_mail_attach.py
@@ -1,4 +1,9 @@
 # -*- coding: utf-8 -*-
+"""
+In Version 0.9.5 the attachment filename is no longer surrounded by double quotes, see for the discussion:
+https:/certtools/intelmq/pull/1134
+https:/martinrusev/imbox/commit/7c6cc2fb5f7e39c1496d68f3d432eec19517bf8e#diff-1ae09572064c2e7c225de54ad5b49154
+"""
 import re
 import zipfile
 
@@ -38,7 +43,11 @@ def process(self):
  if not attach:
  continue
 
- if re.search(self.parameters.attach_regex, attach['filename']):
+ attach_filename = attach['filename']
+ if attach_filename.startswith('"'): # for imbox versions older than 0.9.5, see also above
+ attach_filename = attach_filename[1:-1]
+
+ if re.search(self.parameters.attach_regex, attach_filename):
 
  if self.parameters.attach_unzip:
  zipped = zipfile.ZipFile(attach['content'])

diff --git a/intelmq/bots/outputs/restapi/output.py b/intelmq/bots/outputs/restapi/output.py
@@ -34,6 +34,8 @@ def process(self):
  cert=self.ssl_client_cert,
  timeout=self.http_timeout_sec,
  **kwargs)
+ if not r.ok:
+ self.logger.debug("Error during message sending with response body: %r.", r.text)
  r.raise_for_status()
  self.logger.debug('Sent message.')
  self.acknowledge_message()

diff --git a/intelmq/bots/parsers/cleanmx/parser.py b/intelmq/bots/parsers/cleanmx/parser.py
@@ -6,7 +6,9 @@
 PHISHING = OrderedDict([
  ("line", "__IGNORE__"),
  ("id", "extra"),
+ ("first", "__IGNORE__"),
  ("firsttime", "time.source"),
+ ("last", "__IGNORE__"),
  ("lasttime", "__IGNORE__"),
  ("phishtank", "extra"),
  ("virusname", "event_description.target"),
@@ -76,9 +78,13 @@ def parse_line(self, row, report):
 
  extra = {}
  for key, value in row.items():
+
  if not value:
  continue
 
+ if value == 'undef':
+ continue
+
  if key is None:
  self.logger.warning('Value without key found, skipping the'
  ' value: %r', value)
@@ -110,9 +116,6 @@ def parse_line(self, row, report):
  elif value == 'up':
  value = 'online'
 
- if key_orig == 'scanner' and value == 'undef':
- continue
-
  if key == 'extra':
  extra[key_orig] = value
  continue

diff --git a/intelmq/lib/test.py b/intelmq/lib/test.py
@@ -272,10 +272,13 @@ def run_bot(self, iterations: int=1, error_on_pipeline: bool=False, prepare=True
  self.assertLoglineEqual(-1, "Bot stopped.", "INFO")
  self.assertNotRegexpMatchesLog("(ERROR.*?){%d}" % (self.allowed_error_count + 1))
  self.assertNotRegexpMatchesLog("CRITICAL")
- """ If no error happened (incl. tracebacks, we can check for formatting) """
- if not self.allowed_error_count: # This would fail for tracebacks currently
+ """ If no error happened (incl. tracebacks) we can check for formatting """
+ if not self.allowed_error_count:
  for logline in self.loglines:
  fields = utils.parse_logline(logline)
+ if not isinstance(fields, dict):
+ # Traceback
+ continue
  self.assertTrue(fields['message'][-1] in '.:?!',
  msg='Logline {!r} does not end with .? or !.'
  ''.format(fields['message']))

diff --git a/intelmq/tests/bots/experts/cymru_whois/test_expert.py b/intelmq/tests/bots/experts/cymru_whois/test_expert.py
@@ -71,8 +71,8 @@
 EXAMPLE_6TO4_OUTPUT = {"__type": "Event",
  "source.ip": "2002:3ee0:3972:0001::1",
  "source.network": "2002::/16",
- "source.asn": 1103,
- "source.as_name": "SURFNET-NL SURFnet, The Netherlands, NL",
+ "source.asn": 6939,
+ "source.as_name": "HURRICANE - Hurricane Electric, Inc., US",
  "time.observation": "2015-01-01T00:00:00+00:00",
  }
 

diff --git a/intelmq/tests/bots/parsers/cleanmx/test_parser.py b/intelmq/tests/bots/parsers/cleanmx/test_parser.py
@@ -33,7 +33,7 @@
  'extra.review': '198.18.0.1',
  'feed.name': 'CleanMX Phishing',
  'feed.url': 'http://support.clean-mx.de/clean-mx/xmlphishing?response=alive&format=csv&domain=',
- 'raw': 'bGluZSxpZCxmaXJzdHRpbWUsbGFzdHRpbWUscGhpc2h0YW5rLHZpcnVzbmFtZSx1cmwscmVjZW50LHJlc3BvbnNlLGlwLHJldmlldyxkb21haW4sY291bnRyeSxzb3VyY2UsZW1haWwsaW5ldG51bSxuZXRuYW1lLGRkZXNjcixuczEsbnMyLG5zMyxuczQsbnM1DQoxLDkzNzcxNDIsMjAxNi0xMS0yOSAxMDozMTo0NSwxOTcwLTAxLTAxIDAxOjAwOjAwLDQ2NDczNDUsREhMLGh0dHA6Ly9leGFtcGxlLmNvbS9kZWhsJTIwcGFja2FnZS9jb25maXJtLyxkb3duLGFsaXZlLDE5OC4xOC4wLjEsMTk4LjE4LjAuMSwxOTguMTguMC4xLFVTLEFSSU4sYWJ1c2VAZXhhbXBsZS5jb20sMTk4LjE4LjAuMCAtIDE5OC4xOS4yNTUuMjU1LEVYQU1QTEUtTkVUV09SSy0xNSxFeGFtcGxlIExheWVyLG5zMi5leGFtcGxlLmNvbSxuczEuZXhhbXBsZS5jb20sLCwNCg==',
+ 'raw': 'bGluZSxpZCxmaXJzdCxmaXJzdHRpbWUsbGFzdCxsYXN0dGltZSxwaGlzaHRhbmssdmlydXNuYW1lLHVybCxyZWNlbnQscmVzcG9uc2UsaXAscmV2aWV3LGRvbWFpbixjb3VudHJ5LHNvdXJjZSxlbWFpbCxpbmV0bnVtLG5ldG5hbWUsZGRlc2NyLG5zMSxuczIsbnMzLG5zNCxuczUNCjEsOTM3NzE0MiwxNDc4ODYwMzA1LDIwMTYtMTEtMjkgMTA6MzE6NDUsMCwxOTcwLTAxLTAxIDAwOjAwOjAwLDQ2NDczNDUsREhMLGh0dHA6Ly9leGFtcGxlLmNvbS9kZWhsJTIwcGFja2FnZS9jb25maXJtLyxkb3duLGFsaXZlLDE5OC4xOC4wLjEsMTk4LjE4LjAuMSwxOTguMTguMC4xLFVTLEFSSU4sYWJ1c2VAZXhhbXBsZS5jb20sMTk4LjE4LjAuMCAtIDE5OC4xOS4yNTUuMjU1LEVYQU1QTEUtTkVUV09SSy0xNSxFeGFtcGxlIExheWVyLG5zMi5leGFtcGxlLmNvbSxuczEuZXhhbXBsZS5jb20sLCwNCg==',
  'source.abuse_contact': '[email protected]',
  'source.geolocation.cc': 'US',
  'source.ip': '198.18.0.1',
@@ -56,7 +56,7 @@
  'extra.ns1': 'ns-de.example.com',
  'feed.name': 'CleanMX Phishing',
  'feed.url': 'http://support.clean-mx.de/clean-mx/xmlphishing?response=alive&format=csv&domain=',
- 'raw': 'bGluZSxpZCxmaXJzdHRpbWUsbGFzdHRpbWUscGhpc2h0YW5rLHZpcnVzbmFtZSx1cmwscmVjZW50LHJlc3BvbnNlLGlwLHJldmlldyxkb21haW4sY291bnRyeSxzb3VyY2UsZW1haWwsaW5ldG51bSxuZXRuYW1lLGRkZXNjcixuczEsbnMyLG5zMyxuczQsbnM1DQo3LDkzNzcxMzYsMjAxNi0xMS0yOSAxMDoxNzozOCwxOTcwLTAxLTAxIDAxOjAwOjAwLDQ2NDc0MTIsRnJlZSxodHRwOi8vZXhhbXBsZS5uZXQvRnIvNWI4Y2EzY2FmODlmNWNkNjI0YzJiNjkyYjk5NzFjY2MvLHVwLGFsaXZlLDE5OC4xOC4wLjcsMTk4LjE4LjAuNyxleGFtcGxlLm5ldCxQTCxSSVBFLGFidXNlQGV4YW1wbGUubmV0LDE5OC4xOC4wLjAgLSAxOTguMTkuMjU1LjI1NSxFWEFNUExFLCxucy1kZS5leGFtcGxlLmNvbSxucy1kZS5leGFtcGxlLm5ldCxucy1kZS5leGFtcGxlLmNvbSxucy1kZS5leGFtcGxlLm9yZywNCg==',
+ 'raw': 'bGluZSxpZCxmaXJzdCxmaXJzdHRpbWUsbGFzdCxsYXN0dGltZSxwaGlzaHRhbmssdmlydXNuYW1lLHVybCxyZWNlbnQscmVzcG9uc2UsaXAscmV2aWV3LGRvbWFpbixjb3VudHJ5LHNvdXJjZSxlbWFpbCxpbmV0bnVtLG5ldG5hbWUsZGRlc2NyLG5zMSxuczIsbnMzLG5zNCxuczUNCjcsOTM3NzEzNiwxNDc4ODU5NDU4LDIwMTYtMTEtMjkgMTA6MTc6MzgsMCwxOTcwLTAxLTAxIDAwOjAwOjAwLDQ2NDc0MTIsRnJlZSxodHRwOi8vZXhhbXBsZS5uZXQvRnIvNWI4Y2EzY2FmODlmNWNkNjI0YzJiNjkyYjk5NzFjY2MvLHVwLGFsaXZlLDE5OC4xOC4wLjcsMTk4LjE4LjAuNyxleGFtcGxlLm5ldCxQTCxSSVBFLGFidXNlQGV4YW1wbGUubmV0LDE5OC4xOC4wLjAgLSAxOTguMTkuMjU1LjI1NSxFWEFNUExFLCxucy1kZS5leGFtcGxlLmNvbSxucy1kZS5leGFtcGxlLm5ldCxucy1kZS5leGFtcGxlLmNvbSxucy1kZS5leGFtcGxlLm9yZywNCg==',
  'source.abuse_contact': '[email protected]',
  'source.fqdn': 'example.net',
  'source.geolocation.cc': 'PL',

diff --git a/intelmq/tests/bots/parsers/cleanmx/xmlphishing b/intelmq/tests/bots/parsers/cleanmx/xmlphishing
@@ -1,3 +1,3 @@
-"line","id","firsttime","lasttime","phishtank","virusname","url","recent","response","ip","review","domain","country","source","email","inetnum","netname","ddescr","ns1","ns2","ns3","ns4","ns5"
-"1","9377142","2016-11-29 10:31:45","1970-01-01 01:00:00","4647345","DHL","http://example.com/dehl%20package/confirm/","down","alive","198.18.0.1","198.18.0.1","198.18.0.1","US","ARIN","[email protected]","198.18.0.0 - 198.19.255.255","EXAMPLE-NETWORK-15","Example Layer","ns2.example.com","ns1.example.com","","",""
-"7","9377136","2016-11-29 10:17:38","1970-01-01 01:00:00","4647412","Free","http://example.net/Fr/5b8ca3caf89f5cd624c2b692b9971ccc/","up","alive","198.18.0.7","198.18.0.7","example.net","PL","RIPE","[email protected]","198.18.0.0 - 198.19.255.255","EXAMPLE","","ns-de.example.com","ns-de.example.net","ns-de.example.com","ns-de.example.org",""
+"line","id","first","firsttime","last","lasttime","phishtank","virusname","url","recent","response","ip","review","domain","country","source","email","inetnum","netname","ddescr","ns1","ns2","ns3","ns4","ns5"
+"1","9377142","1478860305","2016-11-29 10:31:45","0","1970-01-01 00:00:00","4647345","DHL","http://example.com/dehl%20package/confirm/","down","alive","198.18.0.1","198.18.0.1","198.18.0.1","US","ARIN","[email protected]","198.18.0.0 - 198.19.255.255","EXAMPLE-NETWORK-15","Example Layer","ns2.example.com","ns1.example.com","","",""
+"7","9377136","1478859458","2016-11-29 10:17:38","0","1970-01-01 00:00:00","4647412","Free","http://example.net/Fr/5b8ca3caf89f5cd624c2b692b9971ccc/","up","alive","198.18.0.7","198.18.0.7","example.net","PL","RIPE","[email protected]","198.18.0.0 - 198.19.255.255","EXAMPLE","","ns-de.example.com","ns-de.example.net","ns-de.example.com","ns-de.example.org",""
diff --git a/setup.py b/setup.py
@@ -56,6 +56,7 @@
  version=__version__,
  maintainer='Sebastian Wagner',
  maintainer_email='[email protected]',
+ python_requires='>=3.3',
  install_requires=REQUIRES,
  test_suite='intelmq.tests',
  packages=find_packages(),