diff --git a/CHANGELOG.md b/CHANGELOG.md index e146c03f9..8b46af35e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -228,7 +228,9 @@ CHANGELOG - `bots.collectors.rt.collector_rt`: Log ticket id for downloaded reports. #### Parsers -- `bots.parsers.shadowserver`: if required fields do not exist in data, an exception is raised, so the line will be dumped and not further processed. +- `bots.parsers.shadowserver`: + - if required fields do not exist in data, an exception is raised, so the line will be dumped and not further processed. + - fix a bug in the parsing of column `cipher_suite` in ssl poodle reports (#1288). #### Experts - Reverse DNS Expert: ignore all invalid results and use first valid one (#1264). @@ -246,6 +248,9 @@ CHANGELOG - Drop tests for Python 3.3 for the mode with all requirements, as some optional dependencies do not support Python 3.3 anymore. - `lib.test`: Add parameter `compare_raw` (default: `True`) to `assertMessageEqual`, to optionally skip the comparison of the raw field. - Add tests for RT collector. +- Add tests for Shadowserver Parser: + - SSL Poodle Reports. + - Helper functions. ### Tools - `intelmqctl list` now sorts the output of bots and queues (#1262). diff --git a/intelmq/bots/parsers/shadowserver/config.py b/intelmq/bots/parsers/shadowserver/config.py index 0a7940698..da40f328f 100644 --- a/intelmq/bots/parsers/shadowserver/config.py +++ b/intelmq/bots/parsers/shadowserver/config.py @@ -111,14 +111,14 @@ def add_UTC_to_timestamp(value): def convert_bool(value): - if value.lower() in ('y', 'yes', 'true', 'enabled'): + if value.lower() in ('y', 'yes', 'true', 'enabled', '1'): return True - elif value.lower() in ('n', 'no', 'false', 'disabled'): + elif value.lower() in ('n', 'no', 'false', 'disabled', '0'): return False def validate_to_none(value): - if not len(value) or value in ['0', 'unknown']: + if not len(value) or value in ('0', 'unknown'): return None return value diff --git a/intelmq/tests/bots/parsers/shadowserver/ssl_poodle.csv b/intelmq/tests/bots/parsers/shadowserver/ssl_poodle.csv new file mode 100644 index 000000000..9f90e8f2c --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/ssl_poodle.csv @@ -0,0 +1,2 @@ +"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","ssl_poodle","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error" +"2018-08-08 00:51:42","203.0.113.85",8443,"example.com","ssl-poodle","TLSv1.0",65540,"AT","WIEN","VIENNA","TLS_RSA_WITH_RC4_128_SHA","Y",1024,"usg20_107BEF394BA5","usg20_107BEF394BA5","2014-06-25 00:00:42","2034-06-20 00:00:42","04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3","53AA112A",2,"sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,0,0,,"16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E","0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE","33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC","HTTP/1.1",200,"OK","text/html",,,,"",,"chunked","Wed, 08 Aug 2018 00:51:44 GMT","Y","Y","N","N","unknown","x509: unknown error" diff --git a/intelmq/tests/bots/parsers/shadowserver/test_helpers.py b/intelmq/tests/bots/parsers/shadowserver/test_helpers.py new file mode 100644 index 000000000..b7f4fa2ac --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_helpers.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Thu Aug 9 15:18:24 2018 + +@author: sebastian +""" + +import unittest +from intelmq.bots.parsers.shadowserver.config import validate_to_none, convert_bool + + +class TestShadowserverHelpers(unittest.TestCase): + + def test_none(self): + self.assertEqual(None, validate_to_none('')) + self.assertEqual(None, validate_to_none('0')) + self.assertEqual(None, validate_to_none('0')) + self.assertEqual('1', validate_to_none('1')) + self.assertEqual('foobar', validate_to_none('foobar')) + + def test_bool(self): + self.assertEqual(True, convert_bool('true')) + self.assertEqual(True, convert_bool('y')) + self.assertEqual(True, convert_bool('yes')) + self.assertEqual(True, convert_bool('enabled')) + self.assertEqual(True, convert_bool('1')) + self.assertEqual(False, convert_bool('false')) + self.assertEqual(False, convert_bool('n')) + self.assertEqual(False, convert_bool('no')) + self.assertEqual(False, convert_bool('disabled')) + self.assertEqual(False, convert_bool('0')) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_ssl_poodle.py b/intelmq/tests/bots/parsers/shadowserver/test_ssl_poodle.py new file mode 100644 index 000000000..e943c78ad --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_ssl_poodle.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +import os +import unittest + +import intelmq.lib.test as test +import intelmq.lib.utils as utils +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot + +with open(os.path.join(os.path.dirname(__file__), 'ssl_poodle.csv')) as handle: + EXAMPLE_FILE = handle.read() +EXAMPLE_LINES = EXAMPLE_FILE.splitlines() + +EXAMPLE_REPORT = {"feed.name": "ShadowServer SSL POODLE", + "raw": utils.base64_encode(EXAMPLE_FILE), + "__type": "Report", + "time.observation": "2015-01-01T00:00:00+00:00", + } +EVENTS = [{'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable service', + 'classification.identifier': 'ssl-poodle', + 'extra.browser_error': 'x509: unknown error', + 'extra.browser_trusted': False, + 'extra.cert_expiration_date': '2034-06-20 00:00:42', + 'extra.cert_expired': False, + 'extra.cert_issue_date': '2014-06-25 00:00:42', + 'extra.cert_length': '1024', + 'extra.cert_serial_number': '53AA112A', + 'extra.cert_valid': True, + 'extra.cipher_suite': 'TLS_RSA_WITH_RC4_128_SHA', + 'extra.content_type': 'text/html', + 'extra.handshake': 'TLSv1.0', + 'extra.http_code': 200, + 'extra.http_date': '2018-08-08T00:51:44+00:00', + 'extra.http_reason': 'OK', + 'extra.http_response_type': 'HTTP/1.1', + 'extra.issuer_common_name': 'usg20_107BEF394BA5', + 'extra.key_algorithm': 'rsaEncryption', + 'extra.md5_fingerprint': '33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC', + 'extra.self_signed': True, + 'extra.sha1_fingerprint': '04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3', + 'extra.sha256_fingerprint': '16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E', + 'extra.sha512_fingerprint': '0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE', + 'extra.signature_algorithm': 'sha1WithRSAEncryption', + 'extra.ssl_poodle': True, + 'extra.ssl_version': '2', + 'extra.subject_common_name': 'usg20_107BEF394BA5', + 'extra.tag': 'ssl-poodle', + 'extra.transfer_encoding': 'chunked', + 'feed.name': 'ShadowServer SSL POODLE', + 'protocol.application': 'https', + 'source.asn': 65540, + 'source.geolocation.cc': 'AT', + 'source.geolocation.city': 'VIENNA', + 'source.geolocation.region': 'WIEN', + 'source.ip': '203.0.113.85', + 'source.port': 8443, + 'source.reverse_dns': 'example.com', + 'time.source': '2018-08-08T00:51:42+00:00', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], + EXAMPLE_LINES[1]])), + '__type': 'Event', + }, + ] + + +class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for a ShadowserverParserBot. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.default_input_message = EXAMPLE_REPORT + cls.sysconfig = {'feedname': 'SSL-POODLE-Vulnerable-Servers'} + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() + for i, EVENT in enumerate(EVENTS): + self.assertMessageEqual(i, EVENT) + + +if __name__ == '__main__': # pragma: no cover + unittest.main()