From 07fea1cc3f0024c035a14dc4da53c824f086874d Mon Sep 17 00:00:00 2001 From: Mateo Date: Wed, 5 Jan 2022 12:46:47 -0300 Subject: [PATCH 1/4] Added new shadowserver files to _config.py Added new shadowserver event types and files to _config.py: - Accesible AMQP - Device Identification - SSL Poodle IPv6 --- intelmq/bots/parsers/shadowserver/_config.py | 77 +++++++++++++++++++- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index fa85b58fe..e1ddce735 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -1015,8 +1015,8 @@ def force_base64(value: Optional[str]) -> Optional[str]: }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Scan -ssl_poodle_vulnerable_servers = { +# https://www.shadowserver.org/what-we-do/network-reporting/ssl-poodle-report/ +ssl_poodle46_vulnerable_servers = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), ('source.ip', 'ip'), @@ -2903,10 +2903,78 @@ def scan_exchange_identifier(field): } } +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-amqp-report/ +accessible_amqp = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip'), + ('source.port', 'port') + ], + 'optional_fields': [ + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'tag'), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'channel', convert_int), + ('extra.', 'message_length', convert_int), + ('extra.', 'class', convert_int), + ('extra.', 'method', convert_int), + ('extra.', 'version_major', validate_to_none), + ('extra.', 'version_minor', validate_to_none), + ('extra.', 'capabilities', validate_to_none), + ('extra.', 'cluster_name', validate_to_none), + ('extra.', 'platform', validate_to_none), + ('extra.', 'product', validate_to_none), + ('extra.', 'product_version', validate_to_none), + ('extra.', 'mechanisms', validate_to_none), + ('extra.', 'locales', validate_to_none), + ], + 'constant_fields': { + 'classification.taxonomy': 'other', + 'classification.type': 'other', + 'classification.identifier': 'accessible-amqp', + } +} + +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-amqp-report/ +device_id = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip'), + ('source.port', 'port') + ], + 'optional_fields': [ + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'tag'), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), + ], + 'constant_fields': { + 'classification.taxonomy': 'other', + 'classification.type': 'other', + 'classification.identifier': 'device-id', + } +} + mapping = ( # feed name, file name, function ('Accessible-ADB', 'scan_adb', accessible_adb), ('Accessible-AFP', 'scan_afp', accessible_afp), + ('Accessible-AMQP', 'scan_amqp', accessible_amqp), ('Accessible-ARD', 'scan_ard', accessible_ard), ('Accessible-CoAP', 'scan_coap', accessible_coap), ('Accessible-CWMP', 'scan_cwmp', accessible_cwmp), @@ -2930,6 +2998,8 @@ def scan_exchange_identifier(field): ('Compromised-Website', 'compromised_website', compromised_website), ('DNS-Open-Resolvers', 'scan_dns', dns_open_resolvers), ('Darknet', 'darknet', darknet), # legacy (replaced by event4_honeypot_darknet) + ('Device-Identification IPv4', 'device_id', device_id), + ('Device-Identification IPv6', 'device_id6', device_id), ('Drone', 'botnet_drone', drone), # legacy (replaced by event4_sinkhole, event4_honeypot_darknet and event46_sinkhole_http) ('Drone-Brute-Force', 'drone_brute_force', drone_brute_force), # legacy (replaced by honeypot_brute_force) ('HTTP-Scanners', 'hp_http_scan', http_scanners), @@ -2970,7 +3040,8 @@ def scan_exchange_identifier(field): ('Outdated-DNSSEC-Key', 'outdated_dnssec_key', outdated_dnssec_key), ('Outdated-DNSSEC-Key-IPv6', 'outdated_dnssec_key_v6', outdated_dnssec_key), ('SSL-FREAK-Vulnerable-Servers', 'scan_ssl_freak', ssl_freak_vulnerable_servers), - ('SSL-POODLE-Vulnerable-Servers', 'scan_ssl_poodle', ssl_poodle_vulnerable_servers), + ('SSL-POODLE-Vulnerable-Servers IPv4', 'scan_ssl_poodle', ssl_poodle46_vulnerable_servers), + ('SSL-POODLE-Vulnerable-Servers IPv6', 'scan6_ssl_poodle', ssl_poodle46_vulnerable_servers), ('Sandbox-URL', 'cwsandbox_url', sandbox_url), ('Sinkhole-DNS', 'sinkhole_dns', sinkhole_dns), ('Sinkhole-Events', 'event4_sinkhole', event46_sinkhole), From 88e6161003d28f93c603ffc21765164a9ce7975b Mon Sep 17 00:00:00 2001 From: Mateo Date: Wed, 5 Jan 2022 18:02:28 -0300 Subject: [PATCH 2/4] Update url comment intelmq/bots/parsers/shadowserver/_config.py Co-authored-by: Sebastian --- intelmq/bots/parsers/shadowserver/_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index e1ddce735..f108268d5 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -2941,7 +2941,7 @@ def scan_exchange_identifier(field): } } -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-amqp-report/ +# https://www.shadowserver.org/what-we-do/network-reporting/device-identification-report/ device_id = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), From bee364110f098a5b69165c04cdc23b9bc7b57e05 Mon Sep 17 00:00:00 2001 From: Mateo Date: Wed, 5 Jan 2022 18:17:39 -0300 Subject: [PATCH 3/4] Update accessible_amqp classification type Co-authored-by: Sebastian --- intelmq/bots/parsers/shadowserver/_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index f108268d5..38e76bb45 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -2935,8 +2935,8 @@ def scan_exchange_identifier(field): ('extra.', 'locales', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', 'classification.identifier': 'accessible-amqp', } } From 747100f6ee6519a44cd157fe0b6c98f4b3585821 Mon Sep 17 00:00:00 2001 From: Mateo Date: Wed, 5 Jan 2022 18:25:26 -0300 Subject: [PATCH 4/4] Update device_id class.type to undetermined --- intelmq/bots/parsers/shadowserver/_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 38e76bb45..6a4521145 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -2965,7 +2965,7 @@ def scan_exchange_identifier(field): ], 'constant_fields': { 'classification.taxonomy': 'other', - 'classification.type': 'other', + 'classification.type': 'undetermined', 'classification.identifier': 'device-id', } }