DOC+TST: shadowserver parser: poodle and helpers

CHANGELOG.md

@@ -228,7 +228,9 @@ CHANGELOG
 - `bots.collectors.rt.collector_rt`: Log ticket id for downloaded reports.
 
 #### Parsers
-- `bots.parsers.shadowserver`: if required fields do not exist in data, an exception is raised, so the line will be dumped and not further processed.
+- `bots.parsers.shadowserver`:
+ - if required fields do not exist in data, an exception is raised, so the line will be dumped and not further processed.
+ - fix a bug in the parsing of column `cipher_suite` in ssl poodle reports (#1288).
 
 #### Experts
 - Reverse DNS Expert: ignore all invalid results and use first valid one (#1264).
@@ -246,6 +248,9 @@ CHANGELOG
 - Drop tests for Python 3.3 for the mode with all requirements, as some optional dependencies do not support Python 3.3 anymore.
 - `lib.test`: Add parameter `compare_raw` (default: `True`) to `assertMessageEqual`, to optionally skip the comparison of the raw field.
 - Add tests for RT collector.
+- Add tests for Shadowserver Parser:
+ - SSL Poodle Reports.
+ - Helper functions.
 
 ### Tools
 - `intelmqctl list` now sorts the output of bots and queues (#1262).

intelmq/bots/parsers/shadowserver/config.py

@@ -111,14 +111,14 @@ def add_UTC_to_timestamp(value):
 
 
 def convert_bool(value):
- if value.lower() in ('y', 'yes', 'true', 'enabled'):
+ if value.lower() in ('y', 'yes', 'true', 'enabled', '1'):
  return True
- elif value.lower() in ('n', 'no', 'false', 'disabled'):
+ elif value.lower() in ('n', 'no', 'false', 'disabled', '0'):
  return False
 
 
 def validate_to_none(value):
- if not len(value) or value in ['0', 'unknown']:
+ if not len(value) or value in ('0', 'unknown'):
  return None
  return value
 

intelmq/tests/bots/parsers/shadowserver/ssl_poodle.csv

@@ -0,0 +1,2 @@
+"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","ssl_poodle","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error"
+"2018-08-08 00:51:42","203.0.113.85",8443,"example.com","ssl-poodle","TLSv1.0",65540,"AT","WIEN","VIENNA","TLS_RSA_WITH_RC4_128_SHA","Y",1024,"usg20_107BEF394BA5","usg20_107BEF394BA5","2014-06-25 00:00:42","2034-06-20 00:00:42","04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3","53AA112A",2,"sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,0,0,,"16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E","0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE","33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC","HTTP/1.1",200,"OK","text/html",,,,"",,"chunked","Wed, 08 Aug 2018 00:51:44 GMT","Y","Y","N","N","unknown","x509: unknown error"

intelmq/tests/bots/parsers/shadowserver/test_helpers.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Aug 9 15:18:24 2018
+
+@author: sebastian
+"""
+
+import unittest
+from intelmq.bots.parsers.shadowserver.config import validate_to_none, convert_bool
+
+
+class TestShadowserverHelpers(unittest.TestCase):
+
+ def test_none(self):
+ self.assertEqual(None, validate_to_none(''))
+ self.assertEqual(None, validate_to_none('0'))
+ self.assertEqual(None, validate_to_none('0'))
+ self.assertEqual('1', validate_to_none('1'))
+ self.assertEqual('foobar', validate_to_none('foobar'))
+
+ def test_bool(self):
+ self.assertEqual(True, convert_bool('true'))
+ self.assertEqual(True, convert_bool('y'))
+ self.assertEqual(True, convert_bool('yes'))
+ self.assertEqual(True, convert_bool('enabled'))
+ self.assertEqual(True, convert_bool('1'))
+ self.assertEqual(False, convert_bool('false'))
+ self.assertEqual(False, convert_bool('n'))
+ self.assertEqual(False, convert_bool('no'))
+ self.assertEqual(False, convert_bool('disabled'))
+ self.assertEqual(False, convert_bool('0'))

intelmq/tests/bots/parsers/shadowserver/test_ssl_poodle.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+
+import os
+import unittest
+
+import intelmq.lib.test as test
+import intelmq.lib.utils as utils
+from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
+
+with open(os.path.join(os.path.dirname(__file__), 'ssl_poodle.csv')) as handle:
+ EXAMPLE_FILE = handle.read()
+EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
+
+EXAMPLE_REPORT = {"feed.name": "ShadowServer SSL POODLE",
+ "raw": utils.base64_encode(EXAMPLE_FILE),
+ "__type": "Report",
+ "time.observation": "2015-01-01T00:00:00+00:00",
+ }
+EVENTS = [{'classification.taxonomy': 'vulnerable',
+ 'classification.type': 'vulnerable service',
+ 'classification.identifier': 'ssl-poodle',
+ 'extra.browser_error': 'x509: unknown error',
+ 'extra.browser_trusted': False,
+ 'extra.cert_expiration_date': '2034-06-20 00:00:42',
+ 'extra.cert_expired': False,
+ 'extra.cert_issue_date': '2014-06-25 00:00:42',
+ 'extra.cert_length': '1024',
+ 'extra.cert_serial_number': '53AA112A',
+ 'extra.cert_valid': True,
+ 'extra.cipher_suite': 'TLS_RSA_WITH_RC4_128_SHA',
+ 'extra.content_type': 'text/html',
+ 'extra.handshake': 'TLSv1.0',
+ 'extra.http_code': 200,
+ 'extra.http_date': '2018-08-08T00:51:44+00:00',
+ 'extra.http_reason': 'OK',
+ 'extra.http_response_type': 'HTTP/1.1',
+ 'extra.issuer_common_name': 'usg20_107BEF394BA5',
+ 'extra.key_algorithm': 'rsaEncryption',
+ 'extra.md5_fingerprint': '33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC',
+ 'extra.self_signed': True,
+ 'extra.sha1_fingerprint': '04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3',
+ 'extra.sha256_fingerprint': '16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E',
+ 'extra.sha512_fingerprint': '0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE',
+ 'extra.signature_algorithm': 'sha1WithRSAEncryption',
+ 'extra.ssl_poodle': True,
+ 'extra.ssl_version': '2',
+ 'extra.subject_common_name': 'usg20_107BEF394BA5',
+ 'extra.tag': 'ssl-poodle',
+ 'extra.transfer_encoding': 'chunked',
+ 'feed.name': 'ShadowServer SSL POODLE',
+ 'protocol.application': 'https',
+ 'source.asn': 65540,
+ 'source.geolocation.cc': 'AT',
+ 'source.geolocation.city': 'VIENNA',
+ 'source.geolocation.region': 'WIEN',
+ 'source.ip': '203.0.113.85',
+ 'source.port': 8443,
+ 'source.reverse_dns': 'example.com',
+ 'time.source': '2018-08-08T00:51:42+00:00',
+ 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0],
+ EXAMPLE_LINES[1]])),
+ '__type': 'Event',
+ },
+ ]
+
+
+class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase):
+ """
+ A TestCase for a ShadowserverParserBot.
+ """
+
+ @classmethod
+ def set_bot(cls):
+ cls.bot_reference = ShadowserverParserBot
+ cls.default_input_message = EXAMPLE_REPORT
+ cls.sysconfig = {'feedname': 'SSL-POODLE-Vulnerable-Servers'}
+
+ def test_event(self):
+ """ Test if correct Event has been produced. """
+ self.run_bot()
+ for i, EVENT in enumerate(EVENTS):
+ self.assertMessageEqual(i, EVENT)
+
+
+if __name__ == '__main__': # pragma: no cover
+ unittest.main()