From c777e84167fba7b7659bd271211de032676dd3a1 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 30 Jun 2021 11:37:14 +0200 Subject: [PATCH 1/2] ENH: harm: simplify TLP sanitiation the value is upper-case only already, so remove the case-insensitive code --- intelmq/lib/harmonization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/intelmq/lib/harmonization.py b/intelmq/lib/harmonization.py index db1415c3d..19af0588f 100644 --- a/intelmq/lib/harmonization.py +++ b/intelmq/lib/harmonization.py @@ -1160,7 +1160,7 @@ class TLP(UppercaseString): Accepted for sanitation are different cases and the prefix 'tlp:'. """ enum = ['WHITE', 'GREEN', 'AMBER', 'RED'] - prefix_pattern = re.compile(r'^(TLP:?)?\s*', flags=re.IGNORECASE) + prefix_pattern = re.compile(r'^(TLP:?)?\s*') @staticmethod def is_valid(value: str, sanitize: bool = False) -> bool: @@ -1180,6 +1180,6 @@ def sanitize(value: str) -> Optional[str]: value = UppercaseString.sanitize(value) if value: value = TLP.prefix_pattern.sub('', value) - if value.lower() == 'yellow': + if value == 'YELLOW': value = 'AMBER' return value From 18706cde0f5640fc2e39d8273ea8cf5264eda13b Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 30 Jun 2021 11:42:14 +0200 Subject: [PATCH 2/2] BUG: ctip parser: handle TLP value 'unknown' fixes certtools/intelmq#2008 --- intelmq/bots/parsers/microsoft/parser_ctip.py | 2 ++ intelmq/tests/bots/parsers/microsoft/ctip_azure.txt | 2 +- intelmq/tests/bots/parsers/microsoft/test_parser_ctip_azure.py | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/intelmq/bots/parsers/microsoft/parser_ctip.py b/intelmq/bots/parsers/microsoft/parser_ctip.py index 55c8f665f..2ab66a622 100644 --- a/intelmq/bots/parsers/microsoft/parser_ctip.py +++ b/intelmq/bots/parsers/microsoft/parser_ctip.py @@ -265,6 +265,8 @@ def parse_azure(self, line, report): # continue unpacking in next loop except json.decoder.JSONDecodeError: line[key] = utils.base64_decode(value) + elif key == 'TLP' and value.lower() == 'unknown': + del line[key] if isinstance(value, dict): for subkey, subvalue in value.items(): line['%s.%s' % (key, subkey)] = subvalue diff --git a/intelmq/tests/bots/parsers/microsoft/ctip_azure.txt b/intelmq/tests/bots/parsers/microsoft/ctip_azure.txt index 07c47929c..e2a8edc1e 100644 --- a/intelmq/tests/bots/parsers/microsoft/ctip_azure.txt +++ b/intelmq/tests/bots/parsers/microsoft/ctip_azure.txt @@ -1,4 +1,4 @@ {"DataFeed":"CTIP-Infected","SourcedFrom":"SinkHoleMessage","DateTimeReceivedUtc":132348339284870000,"DateTimeReceivedUtcTxt":"Sunday May 24 2020 22:45:28.4870","Malware":"Avalanche","ThreatCode":"B67-SS-TINBA","ThreatConfidence":"Low","TotalEncounters":3,"TLP":"Amber","SourceIp":"224.0.5.8","SourcePort":65116,"DestinationIp":"198.18.18.18","DestinationPort":80,"TargetIp":"203.0.113.45","TargetPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS 1","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":""},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"tinba","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTU5MDM2MDMyOC40ODc0MiwiaXAiOiIxMjcuMC4wLjEiLCJwb3J0Ijo2NTExNiwic2VydmVySXAiOiIxOTguMTguMTg1LjE2MiIsInNlcnZlclBvcnQiOjgwLCJkb21haW4iOiJleGFtcGxlLmNvbSIsImZhbWlseSI6InRpbmJhIiwibWFsd2FyZSI6e30sInJlc3BvbnNlIjoiUmVzcG9uc2UiLCJoYW5kbGVyIjoidGluYmEiLCJ0eXBlIjoiSHR0cCJ9"} {"DataFeed":"CTIP-Infected","SourcedFrom":"SinkHoleMessage","DateTimeReceivedUtc":132348340630510000,"DateTimeReceivedUtcTxt":"Sunday May 24 2020 22:47:43.0510","Malware":"Avalanche","ThreatCode":"B67-SS-MATSNU","ThreatConfidence":"High","TotalEncounters":5,"TLP":"YELLOW","SourceIp":"224.0.5.8","SourcePort":49296,"DestinationIp":"198.18.18.18","DestinationPort":80,"TargetIp":"203.0.113.45","TargetPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64497","SourceIpAsnOrgName":"Example AS 2","SourceIpCountryCode":"AT","SourceIpRegion":"Vienna","SourceIpCity":"Vienna","SourceIpPostalCode":"1060","SourceIpLatitude":48.1951,"SourceIpLongitude":16.3483,"SourceIpMetroCode":0,"SourceIpAreaCode":9,"SourceIpConnectionType":""},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"matsnu5","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"dGhpcyBpcyBqdXN0IHNvbWUgdGV4dA=="} -{"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Gov.0001","DateTimeReceivedUtc":132622667720000000,"DateTimeReceivedUtcTxt":"Wednesday April 07 2021 10:59:32.0000","Malware":"Emotet","ThreatCode":"B77-GV","ThreatConfidence":"High","TotalEncounters":1,"TLP":"Green","SourceIp":"224.0.5.8","SourcePort":33587,"DestinationIp":"10.0.0.1","DestinationPort":8080,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"Styria","SourceIpCity":"Graz","SourceIpPostalCode":"8042","SourceIpLatitude":47.1298,"SourceIpLongitude":15.466,"SourceIpMetroCode":0,"SourceIpAreaCode":6,"SourceIpConnectionType":"","SourceIpv4Int":0},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"bot-id-data","CustomField2":"comp-name","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0aW1lc3RhbXBfdXRjIjoiMjAyMS0wNC0wN1QxMDo1OTozMiIsInNvdXJjZV9pcCI6IjEwLjAuMC4xIiwic291cmNlX3BvcnQiOiIzMzU4NyIsImRlc3RpbmF0aW9uX2lwIjoiMTAuMC4wLjEiLCJkZXN0aW5hdGlvbl9wb3J0IjoiODA4MCIsImNvbXB1dGVyX25hbWUiOiJjb21wLW5hbWUiLCJib3RfaWQiOiJib3QtaWQtZGF0YSJ9"} +{"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Gov.0001","DateTimeReceivedUtc":132622667720000000,"DateTimeReceivedUtcTxt":"Wednesday April 07 2021 10:59:32.0000","Malware":"Emotet","ThreatCode":"B77-GV","ThreatConfidence":"High","TotalEncounters":1,"TLP":"Unknown","SourceIp":"224.0.5.8","SourcePort":33587,"DestinationIp":"10.0.0.1","DestinationPort":8080,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"Styria","SourceIpCity":"Graz","SourceIpPostalCode":"8042","SourceIpLatitude":47.1298,"SourceIpLongitude":15.466,"SourceIpMetroCode":0,"SourceIpAreaCode":6,"SourceIpConnectionType":"","SourceIpv4Int":0},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"bot-id-data","CustomField2":"comp-name","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0aW1lc3RhbXBfdXRjIjoiMjAyMS0wNC0wN1QxMDo1OTozMiIsInNvdXJjZV9pcCI6IjEwLjAuMC4xIiwic291cmNlX3BvcnQiOiIzMzU4NyIsImRlc3RpbmF0aW9uX2lwIjoiMTAuMC4wLjEiLCJkZXN0aW5hdGlvbl9wb3J0IjoiODA4MCIsImNvbXB1dGVyX25hbWUiOiJjb21wLW5hbWUiLCJib3RfaWQiOiJib3QtaWQtZGF0YSJ9"} {"DataFeed":"Microsoft.DCU.CTIP.Infected","SourcedFrom":"Microsoft.DCU.CTIP.Sinkhole","DateTimeReceivedUtc":132651352622420000,"DateTimeReceivedUtcTxt":"Monday May 10 2021 15:47:42.2420","Malware":"Avalanche","ThreatCode":"B67-SS-Gamarue","ThreatConfidence":"Low","TotalEncounters":2,"TLP":"Green","SourceIp":"224.0.5.8","SourcePort":28285,"DestinationIp":"10.0.0.1","DestinationPort":80,"SourceIpInfo":{"SourceIpAsnNumber":"64496","SourceIpAsnOrgName":"Example AS","SourceIpCountryCode":"AT","SourceIpRegion":"","SourceIpCity":"","SourceIpPostalCode":"","SourceIpLatitude":48.2,"SourceIpLongitude":16.3667,"SourceIpMetroCode":0,"SourceIpAreaCode":0,"SourceIpConnectionType":"Cellular","SourceIpv4Int":3758097672},"HttpInfo":{"HttpHost":"","HttpRequest":"","HttpMethod":"","HttpReferrer":"","HttpUserAgent":"","HttpVersion":""},"CustomInfo":{"CustomField1":"andromeda210","CustomField2":"","CustomField3":"","CustomField4":"","CustomField5":""},"Payload":"eyJ0cyI6MTYyMDY2MTY2Mi4yNDIzMTYsImlwIjoiMjI0LjAuNS44IiwicG9ydCI6MjgyODUsInNlcnZlcklwIjoiMTAuMC4wLjEiLCJzZXJ2ZXJQb3J0Ijo4MCwiZG9tYWluIjoiZXhhbXBsZS5jb20iLCJmYW1pbHkiOiJhbmRyb21lZGEiLCJtYWx3YXJlIjp7fSwicmVzcG9uc2UiOiJIdHRwT2siLCJoYW5kbGVyIjoiaGFuZGxlcjEiLCJ0eXBlIjoiSHR0cCJ9"} \ No newline at end of file diff --git a/intelmq/tests/bots/parsers/microsoft/test_parser_ctip_azure.py b/intelmq/tests/bots/parsers/microsoft/test_parser_ctip_azure.py index f8412c81b..3dbba4c94 100644 --- a/intelmq/tests/bots/parsers/microsoft/test_parser_ctip_azure.py +++ b/intelmq/tests/bots/parsers/microsoft/test_parser_ctip_azure.py @@ -119,7 +119,6 @@ 'source.ip': '224.0.5.8', 'source.port': 33587, 'time.source': '2021-04-07T10:59:32+00:00', - 'tlp': 'GREEN', 'source.geolocation.cc': 'AT', }, {